{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 2.0, "eval_steps": 200, "global_step": 21754, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 299.6928446771379, "eval_completions/max_terminated_length": 298.3961605584642, "eval_completions/mean_length": 209.70615183246073, "eval_completions/mean_terminated_length": 209.30374606200746, "eval_completions/min_length": 135.17102966841188, "eval_completions/min_terminated_length": 135.17102966841188, "eval_entropy": 1.0814493743745028, "eval_frac_reward_zero_std": 0.42408376963350786, "eval_kl": 0.0, "eval_loss": -3.307464430690743e-05, "eval_num_tokens": 0.0, "eval_reward": -0.15597731239092497, "eval_reward_std": 0.4992521557352306, "eval_rewards/reward_fn/mean": -0.15597731239092497, "eval_rewards/reward_fn/std": 0.49925217217071727, "eval_runtime": 1746.5295, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.2849910983240418, "eval_sampling/importance_sampling_ratio/mean": 0.9999885321911717, "eval_sampling/importance_sampling_ratio/min": 0.7644410355969041, "eval_sampling/sampling_logp_difference/max": 0.3089583640947392, "eval_sampling/sampling_logp_difference/mean": 0.021393064744315432, "eval_steps_per_second": 0.041, "step": 0 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 151.625, "completions/mean_terminated_length": 151.625, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 1.0753526240587234, "epoch": 9.193711501333089e-05, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0, "num_tokens": 3285.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "sampling/importance_sampling_ratio/max": 1.169453501701355, "sampling/importance_sampling_ratio/mean": 0.9986120462417603, "sampling/importance_sampling_ratio/min": 0.7822896838188171, "sampling/sampling_logp_difference/max": 0.2455301284790039, "sampling/sampling_logp_difference/mean": 0.021735867485404015, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.1111111111111, "completions/max_terminated_length": 326.1111111111111, "completions/mean_length": 214.375, "completions/mean_terminated_length": 214.375, "completions/min_length": 144.77777777777777, "completions/min_terminated_length": 144.77777777777777, "entropy": 1.0093232616782188, "epoch": 0.0009193711501333088, "frac_reward_zero_std": 0.5555555555555556, "grad_norm": 0.08929397165775299, "kl": 0.000496348659604943, "learning_rate": 2.757352941176471e-08, "loss": -0.0002, "num_tokens": 40760.0, "reward": -0.5833333333333334, "reward_std": 0.41145816776487565, "rewards/reward_fn/mean": -0.5833333333333334, "rewards/reward_fn/std": 0.41145819425582886, "sampling/importance_sampling_ratio/max": 1.2434800465901692, "sampling/importance_sampling_ratio/mean": 1.0003360112508137, "sampling/importance_sampling_ratio/min": 0.7517881658342149, "sampling/sampling_logp_difference/max": 0.29970783326360917, "sampling/sampling_logp_difference/mean": 0.02082863812231355, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.1, "completions/max_terminated_length": 265.1, "completions/mean_length": 199.65, "completions/mean_terminated_length": 199.65, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.0601678103208543, "epoch": 0.0018387423002666175, "frac_reward_zero_std": 0.6, "grad_norm": 7.0930585861206055, "kl": 0.0007098324731487082, "learning_rate": 5.8210784313725494e-08, "loss": -0.0001, "num_tokens": 82284.0, "reward": -0.08125, "reward_std": 0.34209954738616943, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.34209956526756286, "sampling/importance_sampling_ratio/max": 1.2668246150016784, "sampling/importance_sampling_ratio/mean": 0.9997132956981659, "sampling/importance_sampling_ratio/min": 0.7440942466259003, "sampling/sampling_logp_difference/max": 0.3096800684928894, "sampling/sampling_logp_difference/mean": 0.021416753903031348, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.3, "completions/max_terminated_length": 271.3, "completions/mean_length": 188.7125, "completions/mean_terminated_length": 188.7125, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 0.9628061659634113, "epoch": 0.0027581134503999265, "frac_reward_zero_std": 0.5, "grad_norm": 5.984711170196533, "kl": 0.0006879183139972156, "learning_rate": 8.884803921568628e-08, "loss": 0.0002, "num_tokens": 122317.0, "reward": 0.15, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.2448345065116881, "sampling/importance_sampling_ratio/mean": 0.999651163816452, "sampling/importance_sampling_ratio/min": 0.7509864211082459, "sampling/sampling_logp_difference/max": 0.30414724349975586, "sampling/sampling_logp_difference/mean": 0.020484870113432408, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.9, "completions/max_terminated_length": 298.9, "completions/mean_length": 208.9875, "completions/mean_terminated_length": 208.9875, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.1158181957900524, "epoch": 0.003677484600533235, "frac_reward_zero_std": 0.4, "grad_norm": 6.788961410522461, "kl": 0.000779718585908995, "learning_rate": 1.1948529411764705e-07, "loss": -0.0002, "num_tokens": 167772.0, "reward": -0.08125, "reward_std": 0.6013369858264923, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.6013370215892792, "sampling/importance_sampling_ratio/max": 1.2946096897125243, "sampling/importance_sampling_ratio/mean": 0.9998613774776459, "sampling/importance_sampling_ratio/min": 0.7669993460178375, "sampling/sampling_logp_difference/max": 0.3092350482940674, "sampling/sampling_logp_difference/mean": 0.02185873445123434, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.6, "completions/max_terminated_length": 295.6, "completions/mean_length": 203.6125, "completions/mean_terminated_length": 203.6125, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.018258871883154, "epoch": 0.004596855750666544, "frac_reward_zero_std": 0.5, "grad_norm": 6.249603748321533, "kl": 0.0007066836638841778, "learning_rate": 1.5012254901960787e-07, "loss": 0.0001, "num_tokens": 211021.0, "reward": -0.00625, "reward_std": 0.5027885615825654, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.5027885735034943, "sampling/importance_sampling_ratio/max": 1.2536240696907044, "sampling/importance_sampling_ratio/mean": 0.9995552122592926, "sampling/importance_sampling_ratio/min": 0.8034792423248291, "sampling/sampling_logp_difference/max": 0.251944100856781, "sampling/sampling_logp_difference/mean": 0.020547057315707207, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.4, "completions/max_terminated_length": 337.4, "completions/mean_length": 230.7, "completions/mean_terminated_length": 230.7, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.0806147269904613, "epoch": 0.005516226900799853, "frac_reward_zero_std": 0.2, "grad_norm": 5.656042575836182, "kl": 0.0007581631758512231, "learning_rate": 1.8075980392156866e-07, "loss": 0.0, "num_tokens": 259141.0, "reward": -0.34375, "reward_std": 0.6123397350311279, "rewards/reward_fn/mean": -0.34375, "rewards/reward_fn/std": 0.6123397529125214, "sampling/importance_sampling_ratio/max": 1.2284629106521607, "sampling/importance_sampling_ratio/mean": 0.9997201323509216, "sampling/importance_sampling_ratio/min": 0.7522126615047455, "sampling/sampling_logp_difference/max": 0.29052999019622805, "sampling/sampling_logp_difference/mean": 0.021338467299938203, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 343.1, "completions/max_terminated_length": 339.7, "completions/mean_length": 231.425, "completions/mean_terminated_length": 228.74285888671875, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.0583767995238305, "epoch": 0.006435598050933162, "frac_reward_zero_std": 0.3, "grad_norm": 0.02864058129489422, "kl": 0.0007039500003884313, "learning_rate": 2.1139705882352945e-07, "loss": -0.0003, "num_tokens": 305279.0, "reward": -0.41875, "reward_std": 0.5974647328257561, "rewards/reward_fn/mean": -0.41875, "rewards/reward_fn/std": 0.597464744746685, "sampling/importance_sampling_ratio/max": 1.2691285848617553, "sampling/importance_sampling_ratio/mean": 0.9996894657611847, "sampling/importance_sampling_ratio/min": 0.7317392766475678, "sampling/sampling_logp_difference/max": 0.36146324276924136, "sampling/sampling_logp_difference/mean": 0.02090544067323208, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.3, "completions/max_terminated_length": 278.3, "completions/mean_length": 203.15, "completions/mean_terminated_length": 203.15, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.139736396819353, "epoch": 0.00735496920106647, "frac_reward_zero_std": 0.5, "grad_norm": 7.530137062072754, "kl": 0.0007443344678904395, "learning_rate": 2.420343137254902e-07, "loss": -0.0002, "num_tokens": 346459.0, "reward": -0.025, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.25440456867218, "sampling/importance_sampling_ratio/mean": 0.999367618560791, "sampling/importance_sampling_ratio/min": 0.7998935341835022, "sampling/sampling_logp_difference/max": 0.2461686134338379, "sampling/sampling_logp_difference/mean": 0.021959845535457136, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 206.95, "completions/mean_terminated_length": 206.95, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0894103944301605, "epoch": 0.00827434035119978, "frac_reward_zero_std": 0.5, "grad_norm": 6.244106292724609, "kl": 0.000721826519293245, "learning_rate": 2.7267156862745103e-07, "loss": 0.0001, "num_tokens": 390575.0, "reward": 0.1, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.268983578681946, "sampling/importance_sampling_ratio/mean": 0.9998877227306366, "sampling/importance_sampling_ratio/min": 0.7754014134407043, "sampling/sampling_logp_difference/max": 0.27524458765983584, "sampling/sampling_logp_difference/mean": 0.021048159897327424, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.1, "completions/max_terminated_length": 279.1, "completions/mean_length": 194.6375, "completions/mean_terminated_length": 194.6375, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.0421577889472247, "epoch": 0.009193711501333088, "frac_reward_zero_std": 0.2, "grad_norm": 5.973984241485596, "kl": 0.0007473745507013518, "learning_rate": 3.033088235294118e-07, "loss": 0.0004, "num_tokens": 434250.0, "reward": 0.025, "reward_std": 0.6968977272510528, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6968977510929107, "sampling/importance_sampling_ratio/max": 1.3083885073661805, "sampling/importance_sampling_ratio/mean": 0.9996438145637512, "sampling/importance_sampling_ratio/min": 0.7324391424655914, "sampling/sampling_logp_difference/max": 0.37632611989974973, "sampling/sampling_logp_difference/mean": 0.021190888434648513, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.3, "completions/max_terminated_length": 259.3, "completions/mean_length": 182.575, "completions/mean_terminated_length": 182.575, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.053167861700058, "epoch": 0.010113082651466397, "frac_reward_zero_std": 0.1, "grad_norm": 4.590856552124023, "kl": 0.0007628597573784646, "learning_rate": 3.3394607843137256e-07, "loss": 0.0002, "num_tokens": 472624.0, "reward": 0.24375, "reward_std": 0.8576737105846405, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.8576737403869629, "sampling/importance_sampling_ratio/max": 1.279165554046631, "sampling/importance_sampling_ratio/mean": 0.9999459743499756, "sampling/importance_sampling_ratio/min": 0.7761567234992981, "sampling/sampling_logp_difference/max": 0.2784660145640373, "sampling/sampling_logp_difference/mean": 0.0212702551856637, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 215.825, "completions/mean_terminated_length": 215.825, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0643788024783134, "epoch": 0.011032453801599706, "frac_reward_zero_std": 0.2, "grad_norm": 4.020854473114014, "kl": 0.0006747990802978166, "learning_rate": 3.6458333333333337e-07, "loss": 0.0004, "num_tokens": 515226.0, "reward": 0.1125, "reward_std": 0.6759690880775452, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.6759690940380096, "sampling/importance_sampling_ratio/max": 1.2432999014854431, "sampling/importance_sampling_ratio/mean": 1.0001067757606505, "sampling/importance_sampling_ratio/min": 0.7786146759986877, "sampling/sampling_logp_difference/max": 0.2672003597021103, "sampling/sampling_logp_difference/mean": 0.021326322481036185, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.0, "completions/max_terminated_length": 270.0, "completions/mean_length": 187.4125, "completions/mean_terminated_length": 187.4125, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.0856070943176745, "epoch": 0.011951824951733015, "frac_reward_zero_std": 0.4, "grad_norm": 7.806772708892822, "kl": 0.0007349475290538976, "learning_rate": 3.9522058823529414e-07, "loss": -0.0002, "num_tokens": 557419.0, "reward": -0.175, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2516584753990174, "sampling/importance_sampling_ratio/mean": 0.9997820079326629, "sampling/importance_sampling_ratio/min": 0.7843279838562012, "sampling/sampling_logp_difference/max": 0.2533506631851196, "sampling/sampling_logp_difference/mean": 0.02245151959359646, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.6, "completions/max_terminated_length": 326.6, "completions/mean_length": 219.8625, "completions/mean_terminated_length": 219.8625, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.055223923176527, "epoch": 0.012871196101866324, "frac_reward_zero_std": 0.7, "grad_norm": 0.02835087850689888, "kl": 0.0007502082578866975, "learning_rate": 4.2585784313725495e-07, "loss": -0.0002, "num_tokens": 604944.0, "reward": -0.55, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.3086785674095154, "sampling/importance_sampling_ratio/mean": 0.9999423027038574, "sampling/importance_sampling_ratio/min": 0.8055093109607696, "sampling/sampling_logp_difference/max": 0.27630641460418703, "sampling/sampling_logp_difference/mean": 0.021406589634716512, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.3, "completions/max_terminated_length": 288.3, "completions/mean_length": 208.05, "completions/mean_terminated_length": 208.05, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.090312299877405, "epoch": 0.013790567251999633, "frac_reward_zero_std": 0.3, "grad_norm": 5.133774757385254, "kl": 0.0007231423169287154, "learning_rate": 4.564950980392157e-07, "loss": 0.0004, "num_tokens": 648100.0, "reward": -0.225, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.291504144668579, "sampling/importance_sampling_ratio/mean": 0.9998273611068725, "sampling/importance_sampling_ratio/min": 0.7955003917217255, "sampling/sampling_logp_difference/max": 0.27066789865493773, "sampling/sampling_logp_difference/mean": 0.021314313262701036, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.0, "completions/max_terminated_length": 275.0, "completions/mean_length": 195.4, "completions/mean_terminated_length": 195.4, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0231071449816227, "epoch": 0.01470993840213294, "frac_reward_zero_std": 0.4, "grad_norm": 5.0644989013671875, "kl": 0.0007422683938784758, "learning_rate": 4.871323529411765e-07, "loss": 0.0004, "num_tokens": 685964.0, "reward": -0.53125, "reward_std": 0.473038874566555, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.473038874566555, "sampling/importance_sampling_ratio/max": 1.2740548372268676, "sampling/importance_sampling_ratio/mean": 0.9997909605503082, "sampling/importance_sampling_ratio/min": 0.6973262429237366, "sampling/sampling_logp_difference/max": 0.3868489444255829, "sampling/sampling_logp_difference/mean": 0.021984911896288394, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.8, "completions/max_terminated_length": 303.8, "completions/mean_length": 210.5625, "completions/mean_terminated_length": 210.5625, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0866083726286888, "epoch": 0.01562930955226625, "frac_reward_zero_std": 0.7, "grad_norm": 0.04306478798389435, "kl": 0.0006987493165070191, "learning_rate": 5.177696078431374e-07, "loss": 0.0, "num_tokens": 732089.0, "reward": 0.125, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2782857418060303, "sampling/importance_sampling_ratio/mean": 0.9997583210468293, "sampling/importance_sampling_ratio/min": 0.798870074748993, "sampling/sampling_logp_difference/max": 0.2631758987903595, "sampling/sampling_logp_difference/mean": 0.02120523378252983, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.5, "completions/max_terminated_length": 286.9, "completions/mean_length": 212.4875, "completions/mean_terminated_length": 209.2964294433594, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.0787800539284944, "epoch": 0.01654868070239956, "frac_reward_zero_std": 0.2, "grad_norm": 5.795551776885986, "kl": 0.0007529555674409494, "learning_rate": 5.484068627450981e-07, "loss": -0.0003, "num_tokens": 780928.0, "reward": -0.00625, "reward_std": 0.6472436890006066, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.6472437247633934, "sampling/importance_sampling_ratio/max": 1.3030185937881469, "sampling/importance_sampling_ratio/mean": 0.9994732558727264, "sampling/importance_sampling_ratio/min": 0.7617756247520446, "sampling/sampling_logp_difference/max": 0.32344851493835447, "sampling/sampling_logp_difference/mean": 0.02140289545059204, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.8, "completions/max_terminated_length": 276.8, "completions/mean_length": 197.475, "completions/mean_terminated_length": 197.475, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.0613692447543144, "epoch": 0.01746805185253287, "frac_reward_zero_std": 0.4, "grad_norm": 0.035225849598646164, "kl": 0.0007066807454975787, "learning_rate": 5.790441176470589e-07, "loss": -0.0002, "num_tokens": 823830.0, "reward": -0.125, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2403188467025756, "sampling/importance_sampling_ratio/mean": 1.0000012159347533, "sampling/importance_sampling_ratio/min": 0.7886368274688721, "sampling/sampling_logp_difference/max": 0.24980447292327881, "sampling/sampling_logp_difference/mean": 0.021550391800701618, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 221.4625, "completions/mean_terminated_length": 221.4625, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0411587201058865, "epoch": 0.018387423002666176, "frac_reward_zero_std": 0.1, "grad_norm": 5.348042011260986, "kl": 0.0007153527465561637, "learning_rate": 6.096813725490197e-07, "loss": 0.0002, "num_tokens": 868283.0, "reward": -0.15, "reward_std": 0.8912718653678894, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.8912719130516052, "sampling/importance_sampling_ratio/max": 1.2698710799217223, "sampling/importance_sampling_ratio/mean": 1.0002257406711579, "sampling/importance_sampling_ratio/min": 0.7310633957386017, "sampling/sampling_logp_difference/max": 0.3518221974372864, "sampling/sampling_logp_difference/mean": 0.02077376749366522, "step": 200 }, { "epoch": 0.018387423002666176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0032722513089005235, "eval_completions/max_length": 305.7713787085515, "eval_completions/max_terminated_length": 303.7137870855148, "eval_completions/mean_length": 212.61692844677137, "eval_completions/mean_terminated_length": 211.95678995874746, "eval_completions/min_length": 138.79232111692843, "eval_completions/min_terminated_length": 138.79232111692843, "eval_entropy": 1.084589563323149, "eval_frac_reward_zero_std": 0.418848167539267, "eval_kl": 0.0007784262420524263, "eval_loss": -8.465915016131476e-06, "eval_num_tokens": 868283.0, "eval_reward": -0.15128708551483422, "eval_reward_std": 0.5009349121421212, "eval_rewards/reward_fn/mean": -0.15128708551483422, "eval_rewards/reward_fn/std": 0.5009349251448796, "eval_runtime": 1773.8613, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2760008725404324, "eval_sampling/importance_sampling_ratio/mean": 1.0000922740233922, "eval_sampling/importance_sampling_ratio/min": 0.7714662193653887, "eval_sampling/sampling_logp_difference/max": 0.29893863554816386, "eval_sampling/sampling_logp_difference/mean": 0.020766674739229014, "eval_steps_per_second": 0.041, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 227.55, "completions/mean_terminated_length": 227.55, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0637620031833648, "epoch": 0.019306794152799484, "frac_reward_zero_std": 0.3, "grad_norm": 4.5855913162231445, "kl": 0.0007131522725103423, "learning_rate": 6.403186274509804e-07, "loss": 0.0003, "num_tokens": 915687.0, "reward": -0.1, "reward_std": 0.5639677107334137, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5639677226543427, "sampling/importance_sampling_ratio/max": 1.3379325151443482, "sampling/importance_sampling_ratio/mean": 1.0004191756248475, "sampling/importance_sampling_ratio/min": 0.7696947932243348, "sampling/sampling_logp_difference/max": 0.3192180395126343, "sampling/sampling_logp_difference/mean": 0.02091310527175665, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 203.8, "completions/mean_terminated_length": 203.8, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "entropy": 1.054504746198654, "epoch": 0.020226165302932794, "frac_reward_zero_std": 0.6, "grad_norm": 0.02270311303436756, "kl": 0.0007383580148598412, "learning_rate": 6.709558823529412e-07, "loss": -0.0001, "num_tokens": 961359.0, "reward": -0.075, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2908965349197388, "sampling/importance_sampling_ratio/mean": 1.0006054520606995, "sampling/importance_sampling_ratio/min": 0.770974624156952, "sampling/sampling_logp_difference/max": 0.27950137853622437, "sampling/sampling_logp_difference/mean": 0.020749808475375175, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.2, "completions/max_terminated_length": 339.2, "completions/mean_length": 226.3125, "completions/mean_terminated_length": 226.3125, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.1149094052612782, "epoch": 0.0211455364530661, "frac_reward_zero_std": 0.5, "grad_norm": 7.237305641174316, "kl": 0.0007813888045347994, "learning_rate": 7.01593137254902e-07, "loss": -0.0002, "num_tokens": 1006104.0, "reward": -0.475, "reward_std": 0.4738378763198853, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.4738378882408142, "sampling/importance_sampling_ratio/max": 1.2896608710289001, "sampling/importance_sampling_ratio/mean": 1.0003012120723724, "sampling/importance_sampling_ratio/min": 0.7656932353973389, "sampling/sampling_logp_difference/max": 0.3013443976640701, "sampling/sampling_logp_difference/mean": 0.022089707478880883, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.6, "completions/max_terminated_length": 335.6, "completions/mean_length": 228.9875, "completions/mean_terminated_length": 228.9875, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1022773057222366, "epoch": 0.022064907603199412, "frac_reward_zero_std": 0.3, "grad_norm": 0.03594555705785751, "kl": 0.000785395238926867, "learning_rate": 7.322303921568629e-07, "loss": 0.0002, "num_tokens": 1052479.0, "reward": -0.375, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2804145812988281, "sampling/importance_sampling_ratio/mean": 0.9998940110206604, "sampling/importance_sampling_ratio/min": 0.774997478723526, "sampling/sampling_logp_difference/max": 0.2795681834220886, "sampling/sampling_logp_difference/mean": 0.021694799512624742, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.1, "completions/max_terminated_length": 330.1, "completions/mean_length": 227.225, "completions/mean_terminated_length": 227.225, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.0706482537090778, "epoch": 0.02298427875333272, "frac_reward_zero_std": 0.3, "grad_norm": 0.02968919277191162, "kl": 0.0007527552159444894, "learning_rate": 7.628676470588237e-07, "loss": 0.0001, "num_tokens": 1096433.0, "reward": -0.71875, "reward_std": 0.44904704540967944, "rewards/reward_fn/mean": -0.71875, "rewards/reward_fn/std": 0.44904705733060835, "sampling/importance_sampling_ratio/max": 1.2450652360916137, "sampling/importance_sampling_ratio/mean": 1.000039094686508, "sampling/importance_sampling_ratio/min": 0.7914860188961029, "sampling/sampling_logp_difference/max": 0.24763996601104737, "sampling/sampling_logp_difference/mean": 0.021212419122457506, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.1, "completions/max_terminated_length": 317.1, "completions/mean_length": 225.0625, "completions/mean_terminated_length": 225.0625, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.1172458238899707, "epoch": 0.02390364990346603, "frac_reward_zero_std": 0.5, "grad_norm": 0.03349340334534645, "kl": 0.0008499694260535762, "learning_rate": 7.935049019607844e-07, "loss": -0.0003, "num_tokens": 1145214.0, "reward": -0.20625, "reward_std": 0.5212652325630188, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.5212652564048768, "sampling/importance_sampling_ratio/max": 1.3248901128768922, "sampling/importance_sampling_ratio/mean": 0.9996641159057618, "sampling/importance_sampling_ratio/min": 0.7490870416164398, "sampling/sampling_logp_difference/max": 0.3814911603927612, "sampling/sampling_logp_difference/mean": 0.021587482653558254, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.5, "completions/max_terminated_length": 310.0, "completions/mean_length": 222.4, "completions/mean_terminated_length": 220.57678833007813, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0391244489699603, "epoch": 0.024823021053599337, "frac_reward_zero_std": 0.4, "grad_norm": 5.944749355316162, "kl": 0.0007574812494567596, "learning_rate": 8.241421568627452e-07, "loss": 0.0, "num_tokens": 1191558.0, "reward": 0.0625, "reward_std": 0.5781451225280761, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.5781451463699341, "sampling/importance_sampling_ratio/max": 1.2843209266662599, "sampling/importance_sampling_ratio/mean": 1.000291246175766, "sampling/importance_sampling_ratio/min": 0.7720584690570831, "sampling/sampling_logp_difference/max": 0.2874863684177399, "sampling/sampling_logp_difference/mean": 0.020311230048537254, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.3, "completions/max_terminated_length": 262.3, "completions/mean_length": 196.7625, "completions/mean_terminated_length": 196.7625, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.131000766903162, "epoch": 0.025742392203732648, "frac_reward_zero_std": 0.5, "grad_norm": 4.971517086029053, "kl": 0.0008247875044617104, "learning_rate": 8.54779411764706e-07, "loss": -0.0003, "num_tokens": 1230531.0, "reward": -0.3375, "reward_std": 0.4203621864318848, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.42036219835281374, "sampling/importance_sampling_ratio/max": 1.2860382437705993, "sampling/importance_sampling_ratio/mean": 1.000360018014908, "sampling/importance_sampling_ratio/min": 0.7585604786872864, "sampling/sampling_logp_difference/max": 0.31217917799949646, "sampling/sampling_logp_difference/mean": 0.022694413550198078, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.3, "completions/max_terminated_length": 292.3, "completions/mean_length": 201.5875, "completions/mean_terminated_length": 201.5875, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.0057888574898244, "epoch": 0.026661763353865955, "frac_reward_zero_std": 0.7, "grad_norm": 0.025887854397296906, "kl": 0.0007503936682041968, "learning_rate": 8.854166666666668e-07, "loss": 0.0002, "num_tokens": 1273098.0, "reward": -0.35, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.294146752357483, "sampling/importance_sampling_ratio/mean": 1.00011026263237, "sampling/importance_sampling_ratio/min": 0.7418006807565689, "sampling/sampling_logp_difference/max": 0.34649689197540284, "sampling/sampling_logp_difference/mean": 0.02074468359351158, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 210.6125, "completions/mean_terminated_length": 210.6125, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.059696828573942, "epoch": 0.027581134503999266, "frac_reward_zero_std": 0.4, "grad_norm": 0.031883783638477325, "kl": 0.0007856034324504435, "learning_rate": 9.160539215686274e-07, "loss": -0.0, "num_tokens": 1316915.0, "reward": -0.00625, "reward_std": 0.5328060388565063, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.5328060567378998, "sampling/importance_sampling_ratio/max": 1.2524927854537964, "sampling/importance_sampling_ratio/mean": 0.999273020029068, "sampling/importance_sampling_ratio/min": 0.7890121281147003, "sampling/sampling_logp_difference/max": 0.24703924655914306, "sampling/sampling_logp_difference/mean": 0.021754999458789826, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 211.625, "completions/mean_terminated_length": 211.625, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.0443860024213791, "epoch": 0.028500505654132573, "frac_reward_zero_std": 0.2, "grad_norm": 0.027296965941786766, "kl": 0.0007273910714502563, "learning_rate": 9.466911764705883e-07, "loss": 0.0003, "num_tokens": 1362093.0, "reward": -0.125, "reward_std": 0.722163724899292, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.722163736820221, "sampling/importance_sampling_ratio/max": 1.2711791038513183, "sampling/importance_sampling_ratio/mean": 1.000031977891922, "sampling/importance_sampling_ratio/min": 0.7483709573745727, "sampling/sampling_logp_difference/max": 0.2989282846450806, "sampling/sampling_logp_difference/mean": 0.021247790940105916, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 205.6, "completions/mean_terminated_length": 205.6, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.02982894256711, "epoch": 0.02941987680426588, "frac_reward_zero_std": 0.3, "grad_norm": 5.300772666931152, "kl": 0.0007502335756726097, "learning_rate": 9.77328431372549e-07, "loss": 0.0001, "num_tokens": 1405957.0, "reward": 0.025, "reward_std": 0.6186381995677948, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6186382234096527, "sampling/importance_sampling_ratio/max": 1.2616475343704223, "sampling/importance_sampling_ratio/mean": 1.0004525184631348, "sampling/importance_sampling_ratio/min": 0.7676351130008697, "sampling/sampling_logp_difference/max": 0.2922071099281311, "sampling/sampling_logp_difference/mean": 0.02084492761641741, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.8, "completions/max_terminated_length": 306.5, "completions/mean_length": 219.1125, "completions/mean_terminated_length": 216.30714416503906, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.0276881348341704, "epoch": 0.03033924795439919, "frac_reward_zero_std": 0.3, "grad_norm": 8.416157722473145, "kl": 0.0007842296130547765, "learning_rate": 1.00796568627451e-06, "loss": 0.0004, "num_tokens": 1448542.0, "reward": 0.01875, "reward_std": 0.6547925412654877, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.654792582988739, "sampling/importance_sampling_ratio/max": 1.293851637840271, "sampling/importance_sampling_ratio/mean": 1.0001359045505525, "sampling/importance_sampling_ratio/min": 0.7841155111789704, "sampling/sampling_logp_difference/max": 0.2890821874141693, "sampling/sampling_logp_difference/mean": 0.02073853090405464, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.8, "completions/max_terminated_length": 302.8, "completions/mean_length": 209.7375, "completions/mean_terminated_length": 209.7375, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.1014345772564411, "epoch": 0.0312586191045325, "frac_reward_zero_std": 0.5, "grad_norm": 7.5498366355896, "kl": 0.0007843478295399108, "learning_rate": 1.0386029411764707e-06, "loss": 0.0002, "num_tokens": 1490233.0, "reward": -0.4875, "reward_std": 0.4890873610973358, "rewards/reward_fn/mean": -0.4875, "rewards/reward_fn/std": 0.4890873789787292, "sampling/importance_sampling_ratio/max": 1.2884948015213014, "sampling/importance_sampling_ratio/mean": 1.0002031207084656, "sampling/importance_sampling_ratio/min": 0.7825521230697632, "sampling/sampling_logp_difference/max": 0.2752283215522766, "sampling/sampling_logp_difference/mean": 0.02156333774328232, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.7, "completions/max_terminated_length": 233.7, "completions/mean_length": 173.875, "completions/mean_terminated_length": 173.875, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.040360262989998, "epoch": 0.032177990254665806, "frac_reward_zero_std": 0.7, "grad_norm": 8.066020011901855, "kl": 0.0008249013881140854, "learning_rate": 1.0692401960784314e-06, "loss": 0.0001, "num_tokens": 1526743.0, "reward": -0.3, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.2525104761123658, "sampling/importance_sampling_ratio/mean": 0.9996462762355804, "sampling/importance_sampling_ratio/min": 0.811801689863205, "sampling/sampling_logp_difference/max": 0.2508769273757935, "sampling/sampling_logp_difference/mean": 0.021308065578341483, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 330.6, "completions/max_terminated_length": 308.8, "completions/mean_length": 224.3375, "completions/mean_terminated_length": 220.8750015258789, "completions/min_length": 155.9, "completions/min_terminated_length": 155.9, "entropy": 1.089615571498871, "epoch": 0.03309736140479912, "frac_reward_zero_std": 0.4, "grad_norm": 5.41842794418335, "kl": 0.0007546560824266635, "learning_rate": 1.0998774509803921e-06, "loss": -0.0, "num_tokens": 1575122.0, "reward": -0.43125, "reward_std": 0.5743852078914642, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.5743852376937866, "sampling/importance_sampling_ratio/max": 1.2937683701515197, "sampling/importance_sampling_ratio/mean": 1.000017374753952, "sampling/importance_sampling_ratio/min": 0.7476147770881653, "sampling/sampling_logp_difference/max": 0.319135320186615, "sampling/sampling_logp_difference/mean": 0.02114840243011713, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.5, "completions/max_terminated_length": 330.5, "completions/mean_length": 223.175, "completions/mean_terminated_length": 223.175, "completions/min_length": 154.6, "completions/min_terminated_length": 154.6, "entropy": 1.1024190433323384, "epoch": 0.03401673255493243, "frac_reward_zero_std": 0.4, "grad_norm": 4.903257369995117, "kl": 0.0007648308470379561, "learning_rate": 1.130514705882353e-06, "loss": 0.0002, "num_tokens": 1622680.0, "reward": -0.075, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2647990822792052, "sampling/importance_sampling_ratio/mean": 1.000483900308609, "sampling/importance_sampling_ratio/min": 0.766859593987465, "sampling/sampling_logp_difference/max": 0.31113241910934447, "sampling/sampling_logp_difference/mean": 0.021067772060632706, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.0, "completions/max_terminated_length": 308.4, "completions/mean_length": 221.575, "completions/mean_terminated_length": 218.6482147216797, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.016327341645956, "epoch": 0.03493610370506574, "frac_reward_zero_std": 0.3, "grad_norm": 0.026009738445281982, "kl": 0.0007438287928380305, "learning_rate": 1.1611519607843138e-06, "loss": 0.0001, "num_tokens": 1666270.0, "reward": -0.04375, "reward_std": 0.5397725462913513, "rewards/reward_fn/mean": -0.04375, "rewards/reward_fn/std": 0.5397725462913513, "sampling/importance_sampling_ratio/max": 1.2853425025939942, "sampling/importance_sampling_ratio/mean": 0.999810516834259, "sampling/importance_sampling_ratio/min": 0.7749008536338806, "sampling/sampling_logp_difference/max": 0.2789076089859009, "sampling/sampling_logp_difference/mean": 0.020699821412563324, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 299.8, "completions/max_terminated_length": 277.2, "completions/mean_length": 196.8125, "completions/mean_terminated_length": 192.75892944335936, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 0.9837350681424141, "epoch": 0.03585547485519904, "frac_reward_zero_std": 0.6, "grad_norm": 0.02996785007417202, "kl": 0.0008069306895777117, "learning_rate": 1.1917892156862747e-06, "loss": 0.0005, "num_tokens": 1708367.0, "reward": 0.4, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2348279714584351, "sampling/importance_sampling_ratio/mean": 0.9994670450687408, "sampling/importance_sampling_ratio/min": 0.7581584215164184, "sampling/sampling_logp_difference/max": 0.2913383483886719, "sampling/sampling_logp_difference/mean": 0.02014028634876013, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.5, "completions/max_terminated_length": 317.5, "completions/mean_length": 216.3875, "completions/mean_terminated_length": 213.07857208251954, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.0829862702637911, "epoch": 0.03677484600533235, "frac_reward_zero_std": 0.5, "grad_norm": 0.034554291516542435, "kl": 0.0011342521600454347, "learning_rate": 1.2224264705882354e-06, "loss": -0.0002, "num_tokens": 1754102.0, "reward": -0.35625, "reward_std": 0.36611868590116503, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.3661187097430229, "sampling/importance_sampling_ratio/max": 1.2312871813774109, "sampling/importance_sampling_ratio/mean": 0.9993753373622895, "sampling/importance_sampling_ratio/min": 0.7490815758705139, "sampling/sampling_logp_difference/max": 0.29131460189819336, "sampling/sampling_logp_difference/mean": 0.021433533728122713, "step": 400 }, { "epoch": 0.03677484600533235, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 306.6719022687609, "eval_completions/max_terminated_length": 304.5776614310646, "eval_completions/mean_length": 211.9437172774869, "eval_completions/mean_terminated_length": 211.3185097260001, "eval_completions/min_length": 137.6003490401396, "eval_completions/min_terminated_length": 137.6003490401396, "eval_entropy": 1.0982663704254656, "eval_frac_reward_zero_std": 0.4432809773123909, "eval_kl": 0.0008996251262387141, "eval_loss": 3.357859895913862e-05, "eval_num_tokens": 1754102.0, "eval_reward": -0.18084642233856893, "eval_reward_std": 0.47613257141117443, "eval_rewards/reward_fn/mean": -0.18084642233856893, "eval_rewards/reward_fn/std": 0.4761325862863301, "eval_runtime": 1776.3376, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2862586687996749, "eval_sampling/importance_sampling_ratio/mean": 1.0000032947949713, "eval_sampling/importance_sampling_ratio/min": 0.768495137149868, "eval_sampling/sampling_logp_difference/max": 0.30993228677471685, "eval_sampling/sampling_logp_difference/mean": 0.021232050768770994, "eval_steps_per_second": 0.041, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 290.5, "completions/max_terminated_length": 266.1, "completions/mean_length": 197.4625, "completions/mean_terminated_length": 194.05357208251954, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.0071113228797912, "epoch": 0.03769421715546566, "frac_reward_zero_std": 0.4, "grad_norm": 0.030656542629003525, "kl": 0.0008346667400473962, "learning_rate": 1.2530637254901961e-06, "loss": -0.0001, "num_tokens": 1794827.0, "reward": 0.14375, "reward_std": 0.42590154260396956, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.42590155452489853, "sampling/importance_sampling_ratio/max": 1.3123347878456115, "sampling/importance_sampling_ratio/mean": 0.9997165560722351, "sampling/importance_sampling_ratio/min": 0.7640606999397278, "sampling/sampling_logp_difference/max": 0.3139296531677246, "sampling/sampling_logp_difference/mean": 0.020903043821454047, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.7, "completions/max_terminated_length": 270.7, "completions/mean_length": 186.1, "completions/mean_terminated_length": 186.1, "completions/min_length": 113.2, "completions/min_terminated_length": 113.2, "entropy": 1.0965630508959294, "epoch": 0.03861358830559897, "frac_reward_zero_std": 0.5, "grad_norm": 6.577767848968506, "kl": 0.0008652625390823232, "learning_rate": 1.2837009803921568e-06, "loss": 0.0, "num_tokens": 1835603.0, "reward": -0.4, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.3368516445159913, "sampling/importance_sampling_ratio/mean": 1.0003997206687927, "sampling/importance_sampling_ratio/min": 0.8031453013420105, "sampling/sampling_logp_difference/max": 0.29422425031661986, "sampling/sampling_logp_difference/mean": 0.02269572112709284, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 196.95, "completions/mean_terminated_length": 196.95, "completions/min_length": 122.2, "completions/min_terminated_length": 122.2, "entropy": 1.0419997997581958, "epoch": 0.03953295945573228, "frac_reward_zero_std": 0.3, "grad_norm": 0.08761044591665268, "kl": 0.000925536342401756, "learning_rate": 1.3143382352941177e-06, "loss": 0.0001, "num_tokens": 1875351.0, "reward": 0.01875, "reward_std": 0.6930353283882141, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.6930353462696075, "sampling/importance_sampling_ratio/max": 1.3110000729560851, "sampling/importance_sampling_ratio/mean": 1.0001597225666046, "sampling/importance_sampling_ratio/min": 0.7386307954788208, "sampling/sampling_logp_difference/max": 0.33132703602313995, "sampling/sampling_logp_difference/mean": 0.021656549721956252, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.2, "completions/max_terminated_length": 270.2, "completions/mean_length": 202.1625, "completions/mean_terminated_length": 202.1625, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 0.9847402110695839, "epoch": 0.04045233060586559, "frac_reward_zero_std": 0.5, "grad_norm": 7.5697174072265625, "kl": 0.000975045477389358, "learning_rate": 1.3449754901960785e-06, "loss": -0.0003, "num_tokens": 1913004.0, "reward": 0.31875, "reward_std": 0.3909388899803162, "rewards/reward_fn/mean": 0.31875, "rewards/reward_fn/std": 0.3909389078617096, "sampling/importance_sampling_ratio/max": 1.2842218041419984, "sampling/importance_sampling_ratio/mean": 1.000546109676361, "sampling/importance_sampling_ratio/min": 0.7445207893848419, "sampling/sampling_logp_difference/max": 0.32466148138046264, "sampling/sampling_logp_difference/mean": 0.02084848191589117, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.4, "completions/max_terminated_length": 316.5, "completions/mean_length": 224.85, "completions/mean_terminated_length": 223.32500305175782, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 0.998527443408966, "epoch": 0.0413717017559989, "frac_reward_zero_std": 0.0, "grad_norm": 5.569826126098633, "kl": 0.0008867308497428894, "learning_rate": 1.3756127450980394e-06, "loss": 0.0005, "num_tokens": 1958704.0, "reward": -0.23125, "reward_std": 0.8994559288024903, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.899455976486206, "sampling/importance_sampling_ratio/max": 1.2163272619247436, "sampling/importance_sampling_ratio/mean": 0.9995249330997467, "sampling/importance_sampling_ratio/min": 0.7736773133277893, "sampling/sampling_logp_difference/max": 0.25802781581878664, "sampling/sampling_logp_difference/mean": 0.02064079847186804, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 194.275, "completions/mean_terminated_length": 194.275, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.0428369402885438, "epoch": 0.0422910729061322, "frac_reward_zero_std": 0.5, "grad_norm": 0.03491758927702904, "kl": 0.0009242941596312448, "learning_rate": 1.40625e-06, "loss": 0.0002, "num_tokens": 1999558.0, "reward": -0.6, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2656971335411071, "sampling/importance_sampling_ratio/mean": 0.9994840741157531, "sampling/importance_sampling_ratio/min": 0.7741622805595398, "sampling/sampling_logp_difference/max": 0.2775112628936768, "sampling/sampling_logp_difference/mean": 0.02159971296787262, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 328.6, "completions/max_terminated_length": 315.3, "completions/mean_length": 237.3125, "completions/mean_terminated_length": 235.00357360839843, "completions/min_length": 163.2, "completions/min_terminated_length": 163.2, "entropy": 1.1162430293858052, "epoch": 0.043210444056265514, "frac_reward_zero_std": 0.1, "grad_norm": 4.483035087585449, "kl": 0.0009436175183509477, "learning_rate": 1.4368872549019608e-06, "loss": -0.0003, "num_tokens": 2047151.0, "reward": -0.3375, "reward_std": 0.6544449210166932, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.6544449210166932, "sampling/importance_sampling_ratio/max": 1.2501107692718505, "sampling/importance_sampling_ratio/mean": 0.9997945606708527, "sampling/importance_sampling_ratio/min": 0.7746005713939667, "sampling/sampling_logp_difference/max": 0.2689320743083954, "sampling/sampling_logp_difference/mean": 0.022006461583077907, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 216.8375, "completions/mean_terminated_length": 216.8375, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.052528790384531, "epoch": 0.044129815206398824, "frac_reward_zero_std": 0.4, "grad_norm": 4.8124260902404785, "kl": 0.0009975941065931693, "learning_rate": 1.4675245098039217e-06, "loss": 0.0002, "num_tokens": 2093458.0, "reward": -0.125, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.296168327331543, "sampling/importance_sampling_ratio/mean": 1.0000930726528168, "sampling/importance_sampling_ratio/min": 0.7342368364334106, "sampling/sampling_logp_difference/max": 0.3199332654476166, "sampling/sampling_logp_difference/mean": 0.021021196991205214, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.6, "completions/max_terminated_length": 323.6, "completions/mean_length": 231.95, "completions/mean_terminated_length": 231.95, "completions/min_length": 159.7, "completions/min_terminated_length": 159.7, "entropy": 1.1893933027982713, "epoch": 0.045049186356532135, "frac_reward_zero_std": 0.2, "grad_norm": 5.7040557861328125, "kl": 0.001176522639434552, "learning_rate": 1.4981617647058824e-06, "loss": 0.0, "num_tokens": 2138998.0, "reward": -0.275, "reward_std": 0.7659063935279846, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.7659064054489135, "sampling/importance_sampling_ratio/max": 1.3383163571357728, "sampling/importance_sampling_ratio/mean": 1.0001608550548553, "sampling/importance_sampling_ratio/min": 0.7536799430847168, "sampling/sampling_logp_difference/max": 0.3456514239311218, "sampling/sampling_logp_difference/mean": 0.02211232502013445, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.0, "completions/max_terminated_length": 323.0, "completions/mean_length": 223.875, "completions/mean_terminated_length": 223.875, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.0605631455779077, "epoch": 0.04596855750666544, "frac_reward_zero_std": 0.3, "grad_norm": 6.125982761383057, "kl": 0.0011687688624078874, "learning_rate": 1.5287990196078434e-06, "loss": 0.0002, "num_tokens": 2184916.0, "reward": 0.35, "reward_std": 0.6480426907539367, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.6480427384376526, "sampling/importance_sampling_ratio/max": 1.3130293846130372, "sampling/importance_sampling_ratio/mean": 0.999935120344162, "sampling/importance_sampling_ratio/min": 0.802650249004364, "sampling/sampling_logp_difference/max": 0.2747745752334595, "sampling/sampling_logp_difference/mean": 0.02131553664803505, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.6, "completions/max_terminated_length": 283.6, "completions/mean_length": 208.525, "completions/mean_terminated_length": 208.525, "completions/min_length": 155.2, "completions/min_terminated_length": 155.2, "entropy": 1.0905836947262286, "epoch": 0.04688792865679875, "frac_reward_zero_std": 0.5, "grad_norm": 0.04456010460853577, "kl": 0.0013766567695711273, "learning_rate": 1.559436274509804e-06, "loss": -0.0002, "num_tokens": 2227638.0, "reward": -0.3, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.3539927124977111, "sampling/importance_sampling_ratio/mean": 0.9998978614807129, "sampling/importance_sampling_ratio/min": 0.7679110407829285, "sampling/sampling_logp_difference/max": 0.3173940181732178, "sampling/sampling_logp_difference/mean": 0.0213023629039526, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 209.6375, "completions/mean_terminated_length": 209.6375, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0228771261870861, "epoch": 0.04780729980693206, "frac_reward_zero_std": 0.6, "grad_norm": 0.035280320793390274, "kl": 0.0011215075188374612, "learning_rate": 1.5900735294117648e-06, "loss": 0.0001, "num_tokens": 2269729.0, "reward": -0.475, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.3167005777359009, "sampling/importance_sampling_ratio/mean": 1.0001216948032379, "sampling/importance_sampling_ratio/min": 0.74132699072361, "sampling/sampling_logp_difference/max": 0.3585148513317108, "sampling/sampling_logp_difference/mean": 0.02094185519963503, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.2, "completions/max_terminated_length": 334.2, "completions/mean_length": 224.35, "completions/mean_terminated_length": 224.35, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.098367767035961, "epoch": 0.048726670957065364, "frac_reward_zero_std": 0.4, "grad_norm": 4.98056697845459, "kl": 0.001078227729885839, "learning_rate": 1.6207107843137257e-06, "loss": -0.0001, "num_tokens": 2312077.0, "reward": 0.025, "reward_std": 0.5773320138454437, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5773320615291595, "sampling/importance_sampling_ratio/max": 1.2767532229423524, "sampling/importance_sampling_ratio/mean": 1.00035662651062, "sampling/importance_sampling_ratio/min": 0.7426575392484664, "sampling/sampling_logp_difference/max": 0.3436178207397461, "sampling/sampling_logp_difference/mean": 0.02215595431625843, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 361.9, "completions/max_terminated_length": 344.1, "completions/mean_length": 225.5125, "completions/mean_terminated_length": 222.13214416503905, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.042804618924856, "epoch": 0.049646042107198675, "frac_reward_zero_std": 0.4, "grad_norm": 5.938933372497559, "kl": 0.0009656700611230917, "learning_rate": 1.6513480392156864e-06, "loss": -0.0004, "num_tokens": 2355894.0, "reward": -0.18125, "reward_std": 0.4854166030883789, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.4854166030883789, "sampling/importance_sampling_ratio/max": 1.3111233353614806, "sampling/importance_sampling_ratio/mean": 0.9998191714286804, "sampling/importance_sampling_ratio/min": 0.7706706762313843, "sampling/sampling_logp_difference/max": 0.298222029209137, "sampling/sampling_logp_difference/mean": 0.021084594167768957, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 203.7875, "completions/mean_terminated_length": 203.7875, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.9164508890360594, "epoch": 0.050565413257331986, "frac_reward_zero_std": 0.5, "grad_norm": 5.232804775238037, "kl": 0.0009991643542889505, "learning_rate": 1.6819852941176473e-06, "loss": 0.0003, "num_tokens": 2395925.0, "reward": -0.20625, "reward_std": 0.4147059261798859, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.4147059261798859, "sampling/importance_sampling_ratio/max": 1.261693501472473, "sampling/importance_sampling_ratio/mean": 0.9999246239662171, "sampling/importance_sampling_ratio/min": 0.781638216972351, "sampling/sampling_logp_difference/max": 0.26950339078903196, "sampling/sampling_logp_difference/mean": 0.019529063254594803, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.4, "completions/max_terminated_length": 290.4, "completions/mean_length": 205.7625, "completions/mean_terminated_length": 205.7625, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.0242043904960156, "epoch": 0.051484784407465296, "frac_reward_zero_std": 0.3, "grad_norm": 5.56468391418457, "kl": 0.00109058719863242, "learning_rate": 1.712622549019608e-06, "loss": 0.0007, "num_tokens": 2439154.0, "reward": -0.33125, "reward_std": 0.5799433693289757, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.5799433693289757, "sampling/importance_sampling_ratio/max": 1.2894307732582093, "sampling/importance_sampling_ratio/mean": 0.9998777091503144, "sampling/importance_sampling_ratio/min": 0.7637379586696624, "sampling/sampling_logp_difference/max": 0.3082369178533554, "sampling/sampling_logp_difference/mean": 0.020879466272890567, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.7, "completions/max_terminated_length": 339.7, "completions/mean_length": 231.4625, "completions/mean_terminated_length": 231.4625, "completions/min_length": 158.1, "completions/min_terminated_length": 158.1, "entropy": 1.0548430915921927, "epoch": 0.0524041555575986, "frac_reward_zero_std": 0.5, "grad_norm": 0.04127700626850128, "kl": 0.00104598246034584, "learning_rate": 1.7432598039215688e-06, "loss": -0.0001, "num_tokens": 2486343.0, "reward": -0.575, "reward_std": 0.4410387217998505, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.4410387217998505, "sampling/importance_sampling_ratio/max": 1.2494512677192688, "sampling/importance_sampling_ratio/mean": 0.999860680103302, "sampling/importance_sampling_ratio/min": 0.7401307821273804, "sampling/sampling_logp_difference/max": 0.3266911029815674, "sampling/sampling_logp_difference/mean": 0.02129487544298172, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 309.6, "completions/max_terminated_length": 305.2, "completions/mean_length": 223.8875, "completions/mean_terminated_length": 221.23333435058595, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 0.9710639856755734, "epoch": 0.05332352670773191, "frac_reward_zero_std": 0.3, "grad_norm": 3.8523507118225098, "kl": 0.0012585494136146736, "learning_rate": 1.7738970588235297e-06, "loss": 0.0004, "num_tokens": 2531614.0, "reward": -0.4, "reward_std": 0.7129938900470734, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.7129939138889313, "sampling/importance_sampling_ratio/max": 1.3235880255699157, "sampling/importance_sampling_ratio/mean": 1.0000851511955262, "sampling/importance_sampling_ratio/min": 0.7355387687683106, "sampling/sampling_logp_difference/max": 0.3719643592834473, "sampling/sampling_logp_difference/mean": 0.020554713159799575, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 291.3, "completions/max_terminated_length": 285.0, "completions/mean_length": 195.625, "completions/mean_terminated_length": 192.39642944335938, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.0893535792827607, "epoch": 0.05424289785786522, "frac_reward_zero_std": 0.7, "grad_norm": 0.06870053708553314, "kl": 0.0013320552461664192, "learning_rate": 1.8045343137254904e-06, "loss": 0.0004, "num_tokens": 2572280.0, "reward": -0.23125, "reward_std": 0.28421239256858827, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.28421240448951723, "sampling/importance_sampling_ratio/max": 1.2365685105323792, "sampling/importance_sampling_ratio/mean": 0.99977046251297, "sampling/importance_sampling_ratio/min": 0.772972309589386, "sampling/sampling_logp_difference/max": 0.2626013606786728, "sampling/sampling_logp_difference/mean": 0.022679639421403407, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 313.9, "completions/max_terminated_length": 308.6, "completions/mean_length": 221.5125, "completions/mean_terminated_length": 217.13333435058593, "completions/min_length": 148.8, "completions/min_terminated_length": 148.8, "entropy": 1.0613995477557183, "epoch": 0.05516226900799853, "frac_reward_zero_std": 0.4, "grad_norm": 5.938645839691162, "kl": 0.0013355496288568248, "learning_rate": 1.8351715686274509e-06, "loss": 0.0002, "num_tokens": 2619889.0, "reward": -0.14375, "reward_std": 0.493762868642807, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.493762868642807, "sampling/importance_sampling_ratio/max": 1.2431057453155518, "sampling/importance_sampling_ratio/mean": 1.0003792643547058, "sampling/importance_sampling_ratio/min": 0.76592977643013, "sampling/sampling_logp_difference/max": 0.27083415985107423, "sampling/sampling_logp_difference/mean": 0.021108966507017612, "step": 600 }, { "epoch": 0.05516226900799853, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 297.7783595113438, "eval_completions/max_terminated_length": 295.5445026178011, "eval_completions/mean_length": 206.83398778359512, "eval_completions/mean_terminated_length": 206.15125100234, "eval_completions/min_length": 132.45375218150087, "eval_completions/min_terminated_length": 132.45375218150087, "eval_entropy": 1.0627036587730128, "eval_frac_reward_zero_std": 0.4223385689354276, "eval_kl": 0.0012314453785857071, "eval_loss": 3.286407809355296e-05, "eval_num_tokens": 2619889.0, "eval_reward": -0.16219458987783594, "eval_reward_std": 0.49389825220403455, "eval_rewards/reward_fn/mean": -0.16219458987783594, "eval_rewards/reward_fn/std": 0.4938982657789144, "eval_runtime": 1732.2926, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2862002460536324, "eval_sampling/importance_sampling_ratio/mean": 1.000029017475887, "eval_sampling/importance_sampling_ratio/min": 0.764572526329891, "eval_sampling/sampling_logp_difference/max": 0.3053815302424406, "eval_sampling/sampling_logp_difference/mean": 0.02147742730758785, "eval_steps_per_second": 0.042, "step": 600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.7, "completions/max_terminated_length": 316.7, "completions/mean_length": 208.175, "completions/mean_terminated_length": 208.175, "completions/min_length": 120.9, "completions/min_terminated_length": 120.9, "entropy": 1.03872514590621, "epoch": 0.056081640158131836, "frac_reward_zero_std": 0.4, "grad_norm": 0.056788720190525055, "kl": 0.00121288244263269, "learning_rate": 1.865808823529412e-06, "loss": 0.0002, "num_tokens": 2661247.0, "reward": -0.05, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.257973074913025, "sampling/importance_sampling_ratio/mean": 1.0000994205474854, "sampling/importance_sampling_ratio/min": 0.7748952269554138, "sampling/sampling_logp_difference/max": 0.264211368560791, "sampling/sampling_logp_difference/mean": 0.02121465392410755, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 218.9, "completions/mean_terminated_length": 218.9, "completions/min_length": 152.2, "completions/min_terminated_length": 152.2, "entropy": 1.0815337374806404, "epoch": 0.05700101130826515, "frac_reward_zero_std": 0.3, "grad_norm": 6.619087219238281, "kl": 0.0011759240536775907, "learning_rate": 1.8964460784313725e-06, "loss": 0.0, "num_tokens": 2709143.0, "reward": 0.175, "reward_std": 0.7170356631278991, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.7170357227325439, "sampling/importance_sampling_ratio/max": 1.266796875, "sampling/importance_sampling_ratio/mean": 1.0003431558609008, "sampling/importance_sampling_ratio/min": 0.7859063923358918, "sampling/sampling_logp_difference/max": 0.28662999868392947, "sampling/sampling_logp_difference/mean": 0.02180895209312439, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.0, "completions/max_terminated_length": 325.0, "completions/mean_length": 217.175, "completions/mean_terminated_length": 217.175, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0763624630868436, "epoch": 0.05792038245839846, "frac_reward_zero_std": 0.3, "grad_norm": 0.03173713758587837, "kl": 0.001208967759157531, "learning_rate": 1.9270833333333334e-06, "loss": 0.0007, "num_tokens": 2753037.0, "reward": 0.025, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2695078730583191, "sampling/importance_sampling_ratio/mean": 1.0000222146511077, "sampling/importance_sampling_ratio/min": 0.7777882277965545, "sampling/sampling_logp_difference/max": 0.27039283514022827, "sampling/sampling_logp_difference/mean": 0.02245225626975298, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.4, "completions/max_terminated_length": 312.4, "completions/mean_length": 218.775, "completions/mean_terminated_length": 218.775, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0208023361861707, "epoch": 0.05883975360853176, "frac_reward_zero_std": 0.5, "grad_norm": 6.868015766143799, "kl": 0.0015023421467049048, "learning_rate": 1.9577205882352944e-06, "loss": -0.0, "num_tokens": 2800971.0, "reward": -0.15, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2782096028327943, "sampling/importance_sampling_ratio/mean": 1.000561672449112, "sampling/importance_sampling_ratio/min": 0.7729856491088867, "sampling/sampling_logp_difference/max": 0.28899781703948973, "sampling/sampling_logp_difference/mean": 0.020929868705570696, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.1, "completions/max_terminated_length": 257.1, "completions/mean_length": 182.325, "completions/mean_terminated_length": 182.325, "completions/min_length": 116.5, "completions/min_terminated_length": 116.5, "entropy": 0.9606361258774996, "epoch": 0.05975912475866507, "frac_reward_zero_std": 0.3, "grad_norm": 5.849686145782471, "kl": 0.0015432325832080095, "learning_rate": 1.988357843137255e-06, "loss": -0.0004, "num_tokens": 2843893.0, "reward": -0.4625, "reward_std": 0.4902707502245903, "rewards/reward_fn/mean": -0.4625, "rewards/reward_fn/std": 0.4902707561850548, "sampling/importance_sampling_ratio/max": 1.3648534655570983, "sampling/importance_sampling_ratio/mean": 1.0000509321689606, "sampling/importance_sampling_ratio/min": 0.7811578750610352, "sampling/sampling_logp_difference/max": 0.3446467578411102, "sampling/sampling_logp_difference/mean": 0.02051286268979311, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 205.975, "completions/mean_terminated_length": 205.975, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.0442710891366005, "epoch": 0.06067849590879838, "frac_reward_zero_std": 0.7, "grad_norm": 7.370030879974365, "kl": 0.0017782938222808298, "learning_rate": 2.018995098039216e-06, "loss": 0.0001, "num_tokens": 2889523.0, "reward": -0.2, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.3204310417175293, "sampling/importance_sampling_ratio/mean": 0.9999224841594696, "sampling/importance_sampling_ratio/min": 0.7776239693164826, "sampling/sampling_logp_difference/max": 0.3050168752670288, "sampling/sampling_logp_difference/mean": 0.021412669494748116, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.8, "completions/max_terminated_length": 312.8, "completions/mean_length": 209.3, "completions/mean_terminated_length": 209.3, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.0749442718923092, "epoch": 0.06159786705893169, "frac_reward_zero_std": 0.3, "grad_norm": 6.7996368408203125, "kl": 0.0016358415108697955, "learning_rate": 2.0496323529411767e-06, "loss": -0.0001, "num_tokens": 2931715.0, "reward": -0.38125, "reward_std": 0.645913690328598, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.6459136962890625, "sampling/importance_sampling_ratio/max": 1.2533612847328186, "sampling/importance_sampling_ratio/mean": 0.9997433185577392, "sampling/importance_sampling_ratio/min": 0.7343491375446319, "sampling/sampling_logp_difference/max": 0.34496967792510985, "sampling/sampling_logp_difference/mean": 0.021782933548092843, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.9, "completions/max_terminated_length": 316.9, "completions/mean_length": 216.9875, "completions/mean_terminated_length": 216.9875, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0137108966708184, "epoch": 0.062517238209065, "frac_reward_zero_std": 0.3, "grad_norm": 5.074869155883789, "kl": 0.0017037904923199675, "learning_rate": 2.0802696078431376e-06, "loss": 0.0, "num_tokens": 2978442.0, "reward": -0.075, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2926046013832093, "sampling/importance_sampling_ratio/mean": 1.0000953376293182, "sampling/importance_sampling_ratio/min": 0.7409349262714386, "sampling/sampling_logp_difference/max": 0.3371389746665955, "sampling/sampling_logp_difference/mean": 0.0214530723169446, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 333.4, "completions/max_terminated_length": 313.7, "completions/mean_length": 215.75, "completions/mean_terminated_length": 212.24285888671875, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.0651947773993016, "epoch": 0.06343660935919831, "frac_reward_zero_std": 0.3, "grad_norm": 0.05954078584909439, "kl": 0.0017330386501271278, "learning_rate": 2.110906862745098e-06, "loss": 0.0003, "num_tokens": 3024694.0, "reward": 0.11875, "reward_std": 0.6484905481338501, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.6484905660152436, "sampling/importance_sampling_ratio/max": 1.2370661735534667, "sampling/importance_sampling_ratio/mean": 0.999890410900116, "sampling/importance_sampling_ratio/min": 0.7444906890392303, "sampling/sampling_logp_difference/max": 0.29905885457992554, "sampling/sampling_logp_difference/mean": 0.021552206389605998, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.5, "completions/max_terminated_length": 312.5, "completions/mean_length": 216.175, "completions/mean_terminated_length": 216.175, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 0.9432189542800188, "epoch": 0.06435598050933161, "frac_reward_zero_std": 0.1, "grad_norm": 6.23246955871582, "kl": 0.001657073431852041, "learning_rate": 2.141544117647059e-06, "loss": 0.0, "num_tokens": 3068908.0, "reward": -0.2, "reward_std": 0.8441502273082733, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.8441502630710602, "sampling/importance_sampling_ratio/max": 1.296573567390442, "sampling/importance_sampling_ratio/mean": 0.9996736824512482, "sampling/importance_sampling_ratio/min": 0.7530927360057831, "sampling/sampling_logp_difference/max": 0.30832052826881406, "sampling/sampling_logp_difference/mean": 0.020345782116055488, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.7, "completions/max_terminated_length": 286.7, "completions/mean_length": 197.0625, "completions/mean_terminated_length": 197.0625, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.0143106929957866, "epoch": 0.06527535165946492, "frac_reward_zero_std": 0.4, "grad_norm": 6.906300067901611, "kl": 0.0020244062929123175, "learning_rate": 2.17218137254902e-06, "loss": -0.0003, "num_tokens": 3110705.0, "reward": -0.075, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2391364574432373, "sampling/importance_sampling_ratio/mean": 1.000609815120697, "sampling/importance_sampling_ratio/min": 0.7993077874183655, "sampling/sampling_logp_difference/max": 0.24117908477783204, "sampling/sampling_logp_difference/mean": 0.02108825948089361, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.3, "completions/max_terminated_length": 291.3, "completions/mean_length": 206.4375, "completions/mean_terminated_length": 206.4375, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 0.9966378450393677, "epoch": 0.06619472280959823, "frac_reward_zero_std": 0.6, "grad_norm": 0.11044661700725555, "kl": 0.0020103168273635673, "learning_rate": 2.2028186274509805e-06, "loss": -0.0001, "num_tokens": 3154052.0, "reward": 0.175, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.2732292771339417, "sampling/importance_sampling_ratio/mean": 0.9998150587081909, "sampling/importance_sampling_ratio/min": 0.760818350315094, "sampling/sampling_logp_difference/max": 0.29322814345359804, "sampling/sampling_logp_difference/mean": 0.02123959716409445, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 217.6625, "completions/mean_terminated_length": 217.6625, "completions/min_length": 154.8, "completions/min_terminated_length": 154.8, "entropy": 0.9929412227123976, "epoch": 0.06711409395973154, "frac_reward_zero_std": 0.4, "grad_norm": 4.585963249206543, "kl": 0.0019504479445458856, "learning_rate": 2.2334558823529414e-06, "loss": 0.0002, "num_tokens": 3199641.0, "reward": -0.525, "reward_std": 0.5882755279541015, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.5882755637168884, "sampling/importance_sampling_ratio/max": 1.2704686284065247, "sampling/importance_sampling_ratio/mean": 1.000099629163742, "sampling/importance_sampling_ratio/min": 0.6950209498405456, "sampling/sampling_logp_difference/max": 0.4034956395626068, "sampling/sampling_logp_difference/mean": 0.02126265149563551, "step": 730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.8, "completions/max_terminated_length": 274.8, "completions/mean_length": 196.7625, "completions/mean_terminated_length": 196.7625, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0132611095905304, "epoch": 0.06803346510986485, "frac_reward_zero_std": 0.7, "grad_norm": 0.053630780428647995, "kl": 0.00195974279413349, "learning_rate": 2.2640931372549023e-06, "loss": 0.0001, "num_tokens": 3241366.0, "reward": -0.05, "reward_std": 0.30639100074768066, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.30639100074768066, "sampling/importance_sampling_ratio/max": 1.3064595103263854, "sampling/importance_sampling_ratio/mean": 1.000084012746811, "sampling/importance_sampling_ratio/min": 0.7783202111721039, "sampling/sampling_logp_difference/max": 0.34617817401885986, "sampling/sampling_logp_difference/mean": 0.02142065316438675, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.2, "completions/max_terminated_length": 271.8, "completions/mean_length": 191.175, "completions/mean_terminated_length": 187.25892944335936, "completions/min_length": 104.4, "completions/min_terminated_length": 104.4, "entropy": 0.9739638395607472, "epoch": 0.06895283625999817, "frac_reward_zero_std": 0.4, "grad_norm": 0.04500054568052292, "kl": 0.0017718404968036338, "learning_rate": 2.294730392156863e-06, "loss": 0.0002, "num_tokens": 3279804.0, "reward": -0.15625, "reward_std": 0.49634442329406736, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.4963444352149963, "sampling/importance_sampling_ratio/max": 1.2622020959854126, "sampling/importance_sampling_ratio/mean": 1.0002052187919617, "sampling/importance_sampling_ratio/min": 0.7531908631324769, "sampling/sampling_logp_difference/max": 0.2994641989469528, "sampling/sampling_logp_difference/mean": 0.02157723568379879, "step": 750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 202.4125, "completions/mean_terminated_length": 202.4125, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0432365268468857, "epoch": 0.06987220741013148, "frac_reward_zero_std": 0.1, "grad_norm": 5.255068778991699, "kl": 0.0017438780625525397, "learning_rate": 2.3253676470588237e-06, "loss": -0.0001, "num_tokens": 3323109.0, "reward": -0.14375, "reward_std": 0.7921732127666473, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.7921732366085052, "sampling/importance_sampling_ratio/max": 1.2539901614189148, "sampling/importance_sampling_ratio/mean": 1.000424599647522, "sampling/importance_sampling_ratio/min": 0.7727521300315857, "sampling/sampling_logp_difference/max": 0.28244086503982546, "sampling/sampling_logp_difference/mean": 0.021925252676010133, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 193.5375, "completions/mean_terminated_length": 193.5375, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.012076286226511, "epoch": 0.07079157856026477, "frac_reward_zero_std": 0.3, "grad_norm": 6.82067346572876, "kl": 0.0017061013240891043, "learning_rate": 2.3560049019607847e-06, "loss": 0.0002, "num_tokens": 3365616.0, "reward": -0.5, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2723500370979308, "sampling/importance_sampling_ratio/mean": 0.9995468199253082, "sampling/importance_sampling_ratio/min": 0.7581611931324005, "sampling/sampling_logp_difference/max": 0.31287830471992495, "sampling/sampling_logp_difference/mean": 0.021884988248348235, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 309.3, "completions/max_terminated_length": 283.5, "completions/mean_length": 199.7125, "completions/mean_terminated_length": 195.2875, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 1.0154704481363297, "epoch": 0.07171094971039808, "frac_reward_zero_std": 0.6, "grad_norm": 0.05952152982354164, "kl": 0.0017597968937479891, "learning_rate": 2.3866421568627456e-06, "loss": 0.0001, "num_tokens": 3405673.0, "reward": -0.15625, "reward_std": 0.33117172718048093, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.33117173314094545, "sampling/importance_sampling_ratio/max": 1.3024473547935487, "sampling/importance_sampling_ratio/mean": 0.9999376177787781, "sampling/importance_sampling_ratio/min": 0.7216887325048447, "sampling/sampling_logp_difference/max": 0.39665725231170657, "sampling/sampling_logp_difference/mean": 0.021124787256121637, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.8, "completions/max_terminated_length": 274.8, "completions/mean_length": 189.875, "completions/mean_terminated_length": 189.875, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.0324124250561, "epoch": 0.0726303208605314, "frac_reward_zero_std": 0.3, "grad_norm": 0.1988135278224945, "kl": 0.002061978252459085, "learning_rate": 2.417279411764706e-06, "loss": 0.0004, "num_tokens": 3446823.0, "reward": -0.175, "reward_std": 0.643904197216034, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6439042091369629, "sampling/importance_sampling_ratio/max": 1.3203303813934326, "sampling/importance_sampling_ratio/mean": 0.9995044827461242, "sampling/importance_sampling_ratio/min": 0.7359187811613083, "sampling/sampling_logp_difference/max": 0.3507401287555695, "sampling/sampling_logp_difference/mean": 0.021843368001282216, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.4, "completions/max_terminated_length": 284.4, "completions/mean_length": 195.7375, "completions/mean_terminated_length": 195.7375, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0591415595263243, "epoch": 0.0735496920106647, "frac_reward_zero_std": 0.4, "grad_norm": 0.09118170291185379, "kl": 0.001806329005921725, "learning_rate": 2.4479166666666666e-06, "loss": 0.0003, "num_tokens": 3488850.0, "reward": -0.75, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.75, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.3196043848991394, "sampling/importance_sampling_ratio/mean": 1.000228226184845, "sampling/importance_sampling_ratio/min": 0.7723842620849609, "sampling/sampling_logp_difference/max": 0.2969871997833252, "sampling/sampling_logp_difference/mean": 0.02219330407679081, "step": 800 }, { "epoch": 0.0735496920106647, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 292.326352530541, "eval_completions/max_terminated_length": 290.13438045375216, "eval_completions/mean_length": 204.66252181500872, "eval_completions/mean_terminated_length": 204.24049505263724, "eval_completions/min_length": 134.68411867364748, "eval_completions/min_terminated_length": 134.68411867364748, "eval_entropy": 1.067717766418507, "eval_frac_reward_zero_std": 0.43455497382198954, "eval_kl": 0.0017551934680971359, "eval_loss": 9.424526069778949e-05, "eval_num_tokens": 3488850.0, "eval_reward": -0.1330715532286213, "eval_reward_std": 0.48902597141827586, "eval_rewards/reward_fn/mean": -0.1330715532286213, "eval_rewards/reward_fn/std": 0.4890259848371226, "eval_runtime": 1705.7028, "eval_samples_per_second": 0.336, "eval_sampling/importance_sampling_ratio/max": 1.2813909738267695, "eval_sampling/importance_sampling_ratio/mean": 0.9999343695648886, "eval_sampling/importance_sampling_ratio/min": 0.7634916923953184, "eval_sampling/sampling_logp_difference/max": 0.30235680365645656, "eval_sampling/sampling_logp_difference/mean": 0.02141143238077665, "eval_steps_per_second": 0.042, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 212.0375, "completions/mean_terminated_length": 212.0375, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.1123028375208377, "epoch": 0.07446906316079802, "frac_reward_zero_std": 0.4, "grad_norm": 4.881479263305664, "kl": 0.0016898360569030046, "learning_rate": 2.478553921568628e-06, "loss": 0.0003, "num_tokens": 3536141.0, "reward": -0.075, "reward_std": 0.5185230314731598, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5185230314731598, "sampling/importance_sampling_ratio/max": 1.2481329679489135, "sampling/importance_sampling_ratio/mean": 1.0003389298915863, "sampling/importance_sampling_ratio/min": 0.7531068503856659, "sampling/sampling_logp_difference/max": 0.3072474956512451, "sampling/sampling_logp_difference/mean": 0.022413133643567562, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.8, "completions/max_terminated_length": 265.8, "completions/mean_length": 196.675, "completions/mean_terminated_length": 196.675, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.0159375354647637, "epoch": 0.07538843431093133, "frac_reward_zero_std": 0.4, "grad_norm": 0.06665199249982834, "kl": 0.0019429382278758566, "learning_rate": 2.5091911764705884e-06, "loss": 0.0003, "num_tokens": 3579115.0, "reward": -0.175, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2589051485061646, "sampling/importance_sampling_ratio/mean": 1.000573980808258, "sampling/importance_sampling_ratio/min": 0.7763292849063873, "sampling/sampling_logp_difference/max": 0.2887113094329834, "sampling/sampling_logp_difference/mean": 0.021148764342069627, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.0, "completions/max_terminated_length": 284.0, "completions/mean_length": 199.1375, "completions/mean_terminated_length": 199.1375, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0616732493042946, "epoch": 0.07630780546106464, "frac_reward_zero_std": 0.4, "grad_norm": 0.05373047664761543, "kl": 0.001541483613254968, "learning_rate": 2.539828431372549e-06, "loss": 0.0001, "num_tokens": 3620286.0, "reward": -0.10625, "reward_std": 0.48568440079689024, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.48568440675735475, "sampling/importance_sampling_ratio/max": 1.2394949316978454, "sampling/importance_sampling_ratio/mean": 1.000156396627426, "sampling/importance_sampling_ratio/min": 0.7752691149711609, "sampling/sampling_logp_difference/max": 0.27491912841796873, "sampling/sampling_logp_difference/mean": 0.02172053139656782, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 201.5, "completions/mean_terminated_length": 201.5, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0489766851067543, "epoch": 0.07722717661119793, "frac_reward_zero_std": 0.5, "grad_norm": 0.14965102076530457, "kl": 0.002268621870462084, "learning_rate": 2.57046568627451e-06, "loss": -0.0, "num_tokens": 3662054.0, "reward": -0.48125, "reward_std": 0.3909388899803162, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.3909389078617096, "sampling/importance_sampling_ratio/max": 1.2531113505363465, "sampling/importance_sampling_ratio/mean": 0.9995721042156219, "sampling/importance_sampling_ratio/min": 0.7524945557117462, "sampling/sampling_logp_difference/max": 0.2876538634300232, "sampling/sampling_logp_difference/mean": 0.021034392155706884, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 211.275, "completions/mean_terminated_length": 211.275, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.113775184750557, "epoch": 0.07814654776133124, "frac_reward_zero_std": 0.6, "grad_norm": 6.193023681640625, "kl": 0.001864306250354275, "learning_rate": 2.601102941176471e-06, "loss": -0.0001, "num_tokens": 3708148.0, "reward": -0.125, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2874543786048889, "sampling/importance_sampling_ratio/mean": 0.9998222053050995, "sampling/importance_sampling_ratio/min": 0.758028781414032, "sampling/sampling_logp_difference/max": 0.2961814165115356, "sampling/sampling_logp_difference/mean": 0.02222363092005253, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.3, "completions/max_terminated_length": 269.3, "completions/mean_length": 187.825, "completions/mean_terminated_length": 187.825, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.006027702987194, "epoch": 0.07906591891146456, "frac_reward_zero_std": 0.5, "grad_norm": 5.26637077331543, "kl": 0.0018376265928964131, "learning_rate": 2.6317401960784317e-06, "loss": 0.0003, "num_tokens": 3747134.0, "reward": 0.0, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3035466432571412, "sampling/importance_sampling_ratio/mean": 0.9999394595623017, "sampling/importance_sampling_ratio/min": 0.7669378161430359, "sampling/sampling_logp_difference/max": 0.2993964433670044, "sampling/sampling_logp_difference/mean": 0.021250744909048082, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.4, "completions/max_terminated_length": 308.4, "completions/mean_length": 224.1375, "completions/mean_terminated_length": 224.1375, "completions/min_length": 148.7, "completions/min_terminated_length": 148.7, "entropy": 1.1438939705491067, "epoch": 0.07998529006159787, "frac_reward_zero_std": 0.4, "grad_norm": 6.803271770477295, "kl": 0.0017782498740416485, "learning_rate": 2.662377450980392e-06, "loss": -0.0002, "num_tokens": 3793081.0, "reward": 0.0, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2393657922744752, "sampling/importance_sampling_ratio/mean": 1.0002630174160003, "sampling/importance_sampling_ratio/min": 0.7587973475456238, "sampling/sampling_logp_difference/max": 0.28630841970443727, "sampling/sampling_logp_difference/mean": 0.02212189566344023, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 208.3875, "completions/mean_terminated_length": 208.3875, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 0.9970151208341121, "epoch": 0.08090466121173118, "frac_reward_zero_std": 0.5, "grad_norm": 5.3805084228515625, "kl": 0.0018299176590517163, "learning_rate": 2.693014705882353e-06, "loss": 0.0004, "num_tokens": 3837880.0, "reward": 0.125, "reward_std": 0.4696836888790131, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4696836888790131, "sampling/importance_sampling_ratio/max": 1.2837275743484498, "sampling/importance_sampling_ratio/mean": 0.9996275126934051, "sampling/importance_sampling_ratio/min": 0.7866223514080047, "sampling/sampling_logp_difference/max": 0.25756648778915403, "sampling/sampling_logp_difference/mean": 0.020612158440053464, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.4, "completions/max_terminated_length": 233.4, "completions/mean_length": 183.425, "completions/mean_terminated_length": 183.425, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 0.9781659416854381, "epoch": 0.08182403236186449, "frac_reward_zero_std": 0.5, "grad_norm": 6.861085891723633, "kl": 0.001668582950514974, "learning_rate": 2.7236519607843136e-06, "loss": 0.0, "num_tokens": 3877146.0, "reward": -0.125, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.310480523109436, "sampling/importance_sampling_ratio/mean": 0.9997694909572601, "sampling/importance_sampling_ratio/min": 0.7876611113548279, "sampling/sampling_logp_difference/max": 0.2968543887138367, "sampling/sampling_logp_difference/mean": 0.020834846794605254, "step": 890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.9, "completions/max_terminated_length": 302.9, "completions/mean_length": 202.675, "completions/mean_terminated_length": 202.675, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.1158585757017137, "epoch": 0.0827434035119978, "frac_reward_zero_std": 0.3, "grad_norm": 6.763261318206787, "kl": 0.002203183453093516, "learning_rate": 2.754289215686275e-06, "loss": 0.0004, "num_tokens": 3918848.0, "reward": -0.05625, "reward_std": 0.5889264345169067, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.5889264464378356, "sampling/importance_sampling_ratio/max": 1.3060641288757324, "sampling/importance_sampling_ratio/mean": 0.999544495344162, "sampling/importance_sampling_ratio/min": 0.758137845993042, "sampling/sampling_logp_difference/max": 0.31300843954086305, "sampling/sampling_logp_difference/mean": 0.023172317445278166, "step": 900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 204.875, "completions/mean_terminated_length": 204.875, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 1.046335819363594, "epoch": 0.08366277466213111, "frac_reward_zero_std": 0.4, "grad_norm": 9.291740417480469, "kl": 0.0020701600369648077, "learning_rate": 2.7849264705882355e-06, "loss": 0.0001, "num_tokens": 3960438.0, "reward": -0.35, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.267976462841034, "sampling/importance_sampling_ratio/mean": 1.0002643465995789, "sampling/importance_sampling_ratio/min": 0.7907709240913391, "sampling/sampling_logp_difference/max": 0.255202317237854, "sampling/sampling_logp_difference/mean": 0.0213860634714365, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.8, "completions/max_terminated_length": 293.8, "completions/mean_length": 203.125, "completions/mean_terminated_length": 203.125, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.0469430848956107, "epoch": 0.0845821458122644, "frac_reward_zero_std": 0.1, "grad_norm": 5.371211528778076, "kl": 0.0022050377549021503, "learning_rate": 2.8155637254901964e-06, "loss": 0.0005, "num_tokens": 4002240.0, "reward": 0.275, "reward_std": 0.8366013765335083, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.8366014122962951, "sampling/importance_sampling_ratio/max": 1.2684680700302124, "sampling/importance_sampling_ratio/mean": 1.0004808187484742, "sampling/importance_sampling_ratio/min": 0.7718102157115936, "sampling/sampling_logp_difference/max": 0.27216763496398927, "sampling/sampling_logp_difference/mean": 0.021093057468533516, "step": 920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.1, "completions/max_terminated_length": 290.1, "completions/mean_length": 206.5125, "completions/mean_terminated_length": 206.5125, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0443747162818908, "epoch": 0.08550151696239772, "frac_reward_zero_std": 0.5, "grad_norm": 5.608982086181641, "kl": 0.0021327944414224476, "learning_rate": 2.846200980392157e-06, "loss": 0.0001, "num_tokens": 4047537.0, "reward": -0.10625, "reward_std": 0.4074248731136322, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.4074248790740967, "sampling/importance_sampling_ratio/max": 1.2869680166244506, "sampling/importance_sampling_ratio/mean": 1.0001376152038575, "sampling/importance_sampling_ratio/min": 0.7572178602218628, "sampling/sampling_logp_difference/max": 0.30820580422878263, "sampling/sampling_logp_difference/mean": 0.020781058445572854, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 205.6625, "completions/mean_terminated_length": 205.6625, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.040501171350479, "epoch": 0.08642088811253103, "frac_reward_zero_std": 0.3, "grad_norm": 0.1562378704547882, "kl": 0.0025675140423118137, "learning_rate": 2.876838235294118e-06, "loss": 0.0001, "num_tokens": 4087934.0, "reward": -0.3, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.3012214183807373, "sampling/importance_sampling_ratio/mean": 0.9998708426952362, "sampling/importance_sampling_ratio/min": 0.7598832547664642, "sampling/sampling_logp_difference/max": 0.29014265537261963, "sampling/sampling_logp_difference/mean": 0.02159572783857584, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 198.0375, "completions/mean_terminated_length": 198.0375, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.0710629001259804, "epoch": 0.08734025926266434, "frac_reward_zero_std": 0.3, "grad_norm": 6.260932922363281, "kl": 0.0028521675529191272, "learning_rate": 2.9074754901960787e-06, "loss": 0.0, "num_tokens": 4129697.0, "reward": -0.2, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2824070692062377, "sampling/importance_sampling_ratio/mean": 0.9993419766426086, "sampling/importance_sampling_ratio/min": 0.7723364472389221, "sampling/sampling_logp_difference/max": 0.2920758485794067, "sampling/sampling_logp_difference/mean": 0.021670861914753915, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 327.2, "completions/max_terminated_length": 308.1, "completions/mean_length": 220.6625, "completions/mean_terminated_length": 217.55892944335938, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.078759252279997, "epoch": 0.08825963041279765, "frac_reward_zero_std": 0.3, "grad_norm": 7.763372421264648, "kl": 0.0025614762547775173, "learning_rate": 2.9381127450980397e-06, "loss": 0.0003, "num_tokens": 4172518.0, "reward": 0.11875, "reward_std": 0.5546773716807365, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.5546773836016655, "sampling/importance_sampling_ratio/max": 1.2598665833473206, "sampling/importance_sampling_ratio/mean": 0.9996676504611969, "sampling/importance_sampling_ratio/min": 0.7530906051397324, "sampling/sampling_logp_difference/max": 0.3044284462928772, "sampling/sampling_logp_difference/mean": 0.02125330679118633, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.4, "completions/max_terminated_length": 318.4, "completions/mean_length": 200.7375, "completions/mean_terminated_length": 200.7375, "completions/min_length": 121.2, "completions/min_terminated_length": 121.2, "entropy": 1.0066220205277205, "epoch": 0.08917900156293096, "frac_reward_zero_std": 0.6, "grad_norm": 0.0667622834444046, "kl": 0.002858960410230793, "learning_rate": 2.96875e-06, "loss": 0.0002, "num_tokens": 4213697.0, "reward": -0.025, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2603552103042603, "sampling/importance_sampling_ratio/mean": 0.9995657682418824, "sampling/importance_sampling_ratio/min": 0.7035784393548965, "sampling/sampling_logp_difference/max": 0.41779356598854067, "sampling/sampling_logp_difference/mean": 0.020449365489184856, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.5, "completions/max_terminated_length": 322.5, "completions/mean_length": 213.675, "completions/mean_terminated_length": 213.675, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.0962146133184434, "epoch": 0.09009837271306427, "frac_reward_zero_std": 0.4, "grad_norm": 8.118996620178223, "kl": 0.0027096636418718844, "learning_rate": 2.999387254901961e-06, "loss": -0.0001, "num_tokens": 4256567.0, "reward": -0.175, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.277831530570984, "sampling/importance_sampling_ratio/mean": 1.0000641345977783, "sampling/importance_sampling_ratio/min": 0.748142546415329, "sampling/sampling_logp_difference/max": 0.31624951362609866, "sampling/sampling_logp_difference/mean": 0.02175966780632734, "step": 980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.9, "completions/max_terminated_length": 262.9, "completions/mean_length": 200.775, "completions/mean_terminated_length": 200.775, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.086778636276722, "epoch": 0.09101774386319757, "frac_reward_zero_std": 0.6, "grad_norm": 0.1284385770559311, "kl": 0.003251417653518729, "learning_rate": 3.0300245098039216e-06, "loss": -0.0001, "num_tokens": 4296541.0, "reward": 0.175, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.231396698951721, "sampling/importance_sampling_ratio/mean": 1.0001080989837647, "sampling/importance_sampling_ratio/min": 0.7700096130371094, "sampling/sampling_logp_difference/max": 0.26566314697265625, "sampling/sampling_logp_difference/mean": 0.022068274952471257, "step": 990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.1, "completions/max_terminated_length": 281.1, "completions/mean_length": 205.6875, "completions/mean_terminated_length": 205.6875, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.0664017863571644, "epoch": 0.09193711501333088, "frac_reward_zero_std": 0.4, "grad_norm": 5.382896900177002, "kl": 0.002668237904435955, "learning_rate": 3.060661764705883e-06, "loss": 0.0003, "num_tokens": 4339316.0, "reward": -0.35, "reward_std": 0.5882598340511322, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.588259893655777, "sampling/importance_sampling_ratio/max": 1.2827893853187562, "sampling/importance_sampling_ratio/mean": 1.0001607477664947, "sampling/importance_sampling_ratio/min": 0.7696988165378571, "sampling/sampling_logp_difference/max": 0.2990830570459366, "sampling/sampling_logp_difference/mean": 0.020726638101041316, "step": 1000 }, { "epoch": 0.09193711501333088, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.006108202443280977, "eval_completions/max_length": 308.31588132635255, "eval_completions/max_terminated_length": 303.29319371727746, "eval_completions/mean_length": 213.97294938917975, "eval_completions/mean_terminated_length": 212.38828691678938, "eval_completions/min_length": 138.12041884816753, "eval_completions/min_terminated_length": 138.12041884816753, "eval_entropy": 1.1259224732612856, "eval_frac_reward_zero_std": 0.42757417102966844, "eval_kl": 0.003067009064229124, "eval_loss": 0.00011799554340541363, "eval_num_tokens": 4339316.0, "eval_reward": -0.13110820244328097, "eval_reward_std": 0.494082900216026, "eval_rewards/reward_fn/mean": -0.13110820244328097, "eval_rewards/reward_fn/std": 0.4940829148831376, "eval_runtime": 1785.049, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.2871015525405112, "eval_sampling/importance_sampling_ratio/mean": 1.0000478590970263, "eval_sampling/importance_sampling_ratio/min": 0.7710496226608441, "eval_sampling/sampling_logp_difference/max": 0.306034102541822, "eval_sampling/sampling_logp_difference/mean": 0.02131823371010093, "eval_steps_per_second": 0.04, "step": 1000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.7, "completions/max_terminated_length": 316.7, "completions/mean_length": 213.4, "completions/mean_terminated_length": 213.4, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.1697705589234828, "epoch": 0.09285648616346419, "frac_reward_zero_std": 0.2, "grad_norm": 5.934372425079346, "kl": 0.0032135222558281385, "learning_rate": 3.0912990196078434e-06, "loss": 0.0003, "num_tokens": 4386804.0, "reward": -0.20625, "reward_std": 0.7530511975288391, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.753051245212555, "sampling/importance_sampling_ratio/max": 1.3689023733139039, "sampling/importance_sampling_ratio/mean": 1.0000486433506013, "sampling/importance_sampling_ratio/min": 0.7916115403175354, "sampling/sampling_logp_difference/max": 0.35494189262390136, "sampling/sampling_logp_difference/mean": 0.02291801981627941, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 221.3, "completions/mean_terminated_length": 221.3, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.1494231924414635, "epoch": 0.0937758573135975, "frac_reward_zero_std": 0.5, "grad_norm": 0.06582934409379959, "kl": 0.0029980961204273625, "learning_rate": 3.1219362745098044e-06, "loss": 0.0003, "num_tokens": 4432124.0, "reward": 0.1, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.291193664073944, "sampling/importance_sampling_ratio/mean": 0.9991415023803711, "sampling/importance_sampling_ratio/min": 0.7477850258350373, "sampling/sampling_logp_difference/max": 0.3137397766113281, "sampling/sampling_logp_difference/mean": 0.02211917508393526, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 195.7875, "completions/mean_terminated_length": 195.7875, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.0138084262609481, "epoch": 0.09469522846373081, "frac_reward_zero_std": 0.3, "grad_norm": 0.11766874045133591, "kl": 0.003110911238763947, "learning_rate": 3.152573529411765e-06, "loss": 0.0007, "num_tokens": 4471379.0, "reward": -0.175, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.279722845554352, "sampling/importance_sampling_ratio/mean": 1.0000698804855346, "sampling/importance_sampling_ratio/min": 0.7371732026338578, "sampling/sampling_logp_difference/max": 0.3426006555557251, "sampling/sampling_logp_difference/mean": 0.021059765294194222, "step": 1030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.7, "completions/max_terminated_length": 327.7, "completions/mean_length": 231.65, "completions/mean_terminated_length": 231.65, "completions/min_length": 161.6, "completions/min_terminated_length": 161.6, "entropy": 1.151977390795946, "epoch": 0.09561459961386412, "frac_reward_zero_std": 0.6, "grad_norm": 0.07867686450481415, "kl": 0.0043193907826207575, "learning_rate": 3.1832107843137254e-06, "loss": 0.0004, "num_tokens": 4517191.0, "reward": -0.65, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2505372166633606, "sampling/importance_sampling_ratio/mean": 0.9997167825698853, "sampling/importance_sampling_ratio/min": 0.7438284993171692, "sampling/sampling_logp_difference/max": 0.3340789318084717, "sampling/sampling_logp_difference/mean": 0.02274187430739403, "step": 1040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 354.5, "completions/max_terminated_length": 354.5, "completions/mean_length": 236.0875, "completions/mean_terminated_length": 236.0875, "completions/min_length": 157.5, "completions/min_terminated_length": 157.5, "entropy": 1.0956060737371445, "epoch": 0.09653397076399743, "frac_reward_zero_std": 0.2, "grad_norm": 5.112902641296387, "kl": 0.0032581306877546012, "learning_rate": 3.2138480392156867e-06, "loss": 0.0005, "num_tokens": 4562006.0, "reward": -0.075, "reward_std": 0.6784210562705993, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6784210681915284, "sampling/importance_sampling_ratio/max": 1.2789223074913025, "sampling/importance_sampling_ratio/mean": 0.9999773323535919, "sampling/importance_sampling_ratio/min": 0.7801309049129486, "sampling/sampling_logp_difference/max": 0.27711803317070005, "sampling/sampling_logp_difference/mean": 0.021694541722536088, "step": 1050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.5, "completions/max_terminated_length": 315.5, "completions/mean_length": 221.7125, "completions/mean_terminated_length": 221.7125, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.0999601170420648, "epoch": 0.09745334191413073, "frac_reward_zero_std": 0.3, "grad_norm": 0.10967661440372467, "kl": 0.0027793115441454574, "learning_rate": 3.2444852941176476e-06, "loss": 0.0001, "num_tokens": 4607087.0, "reward": -0.05, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2498607158660888, "sampling/importance_sampling_ratio/mean": 0.9996707201004028, "sampling/importance_sampling_ratio/min": 0.7488687098026275, "sampling/sampling_logp_difference/max": 0.291935333609581, "sampling/sampling_logp_difference/mean": 0.022061102837324143, "step": 1060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.1, "completions/max_terminated_length": 259.1, "completions/mean_length": 179.0625, "completions/mean_terminated_length": 179.0625, "completions/min_length": 110.6, "completions/min_terminated_length": 110.6, "entropy": 1.1744194880127907, "epoch": 0.09837271306426404, "frac_reward_zero_std": 0.4, "grad_norm": 0.07905009388923645, "kl": 0.0036214482170180418, "learning_rate": 3.275122549019608e-06, "loss": -0.0002, "num_tokens": 4648572.0, "reward": -0.075, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.3122403979301454, "sampling/importance_sampling_ratio/mean": 1.0003362953662873, "sampling/importance_sampling_ratio/min": 0.7701992273330689, "sampling/sampling_logp_difference/max": 0.31100776195526125, "sampling/sampling_logp_difference/mean": 0.024047176353633404, "step": 1070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 309.8, "completions/max_terminated_length": 289.5, "completions/mean_length": 206.9125, "completions/mean_terminated_length": 203.46964416503906, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.105036687105894, "epoch": 0.09929208421439735, "frac_reward_zero_std": 0.4, "grad_norm": 6.938143253326416, "kl": 0.004171417787438259, "learning_rate": 3.3057598039215686e-06, "loss": -0.0002, "num_tokens": 4695717.0, "reward": -0.2375, "reward_std": 0.6155621170997619, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.615562129020691, "sampling/importance_sampling_ratio/max": 1.2579758644104004, "sampling/importance_sampling_ratio/mean": 1.0007081508636475, "sampling/importance_sampling_ratio/min": 0.7784537315368653, "sampling/sampling_logp_difference/max": 0.2756911039352417, "sampling/sampling_logp_difference/mean": 0.022661498934030532, "step": 1080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 203.575, "completions/mean_terminated_length": 203.575, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0489739820361137, "epoch": 0.10021145536453066, "frac_reward_zero_std": 0.2, "grad_norm": 6.294253349304199, "kl": 0.003155817700462649, "learning_rate": 3.3363970588235295e-06, "loss": 0.0004, "num_tokens": 4739699.0, "reward": 0.25, "reward_std": 0.714614874124527, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.714614886045456, "sampling/importance_sampling_ratio/max": 1.2861385226249695, "sampling/importance_sampling_ratio/mean": 0.9998547673225403, "sampling/importance_sampling_ratio/min": 0.7795443892478943, "sampling/sampling_logp_difference/max": 0.29224889874458315, "sampling/sampling_logp_difference/mean": 0.02151954509317875, "step": 1090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.5, "completions/max_terminated_length": 250.5, "completions/mean_length": 183.225, "completions/mean_terminated_length": 183.225, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.0402593992650508, "epoch": 0.10113082651466397, "frac_reward_zero_std": 0.5, "grad_norm": 6.32190465927124, "kl": 0.003246776689775288, "learning_rate": 3.367034313725491e-06, "loss": -0.0001, "num_tokens": 4779717.0, "reward": -0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2957336187362671, "sampling/importance_sampling_ratio/mean": 0.9999243676662445, "sampling/importance_sampling_ratio/min": 0.7566905915737152, "sampling/sampling_logp_difference/max": 0.32305335998535156, "sampling/sampling_logp_difference/mean": 0.021789796091616152, "step": 1100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 211.9875, "completions/mean_terminated_length": 211.9875, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.0717760935425757, "epoch": 0.10205019766479728, "frac_reward_zero_std": 0.5, "grad_norm": 0.06534834951162338, "kl": 0.0033345270858262666, "learning_rate": 3.3976715686274514e-06, "loss": -0.0, "num_tokens": 4827212.0, "reward": 0.3, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2725584387779236, "sampling/importance_sampling_ratio/mean": 1.0002706706523896, "sampling/importance_sampling_ratio/min": 0.7767893195152282, "sampling/sampling_logp_difference/max": 0.2735177516937256, "sampling/sampling_logp_difference/mean": 0.022257849760353565, "step": 1110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.4, "completions/max_terminated_length": 282.4, "completions/mean_length": 196.175, "completions/mean_terminated_length": 196.175, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.059814279526472, "epoch": 0.10296956881493059, "frac_reward_zero_std": 0.3, "grad_norm": 0.07813742011785507, "kl": 0.003166015054739546, "learning_rate": 3.428308823529412e-06, "loss": -0.0002, "num_tokens": 4868762.0, "reward": -0.225, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2502918243408203, "sampling/importance_sampling_ratio/mean": 0.9997494935989379, "sampling/importance_sampling_ratio/min": 0.7651277840137481, "sampling/sampling_logp_difference/max": 0.27439218759536743, "sampling/sampling_logp_difference/mean": 0.021695746481418608, "step": 1120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 197.4625, "completions/mean_terminated_length": 197.4625, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.1343149237334729, "epoch": 0.10388893996506389, "frac_reward_zero_std": 0.6, "grad_norm": 0.08258821815252304, "kl": 0.0032833090648637154, "learning_rate": 3.458946078431373e-06, "loss": 0.0005, "num_tokens": 4911983.0, "reward": -0.45, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.3310258746147157, "sampling/importance_sampling_ratio/mean": 1.0002725899219513, "sampling/importance_sampling_ratio/min": 0.6708868443965912, "sampling/sampling_logp_difference/max": 0.5029920518398285, "sampling/sampling_logp_difference/mean": 0.022683802619576456, "step": 1130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 364.0, "completions/max_terminated_length": 364.0, "completions/mean_length": 246.75, "completions/mean_terminated_length": 246.75, "completions/min_length": 168.7, "completions/min_terminated_length": 168.7, "entropy": 1.1668302975594997, "epoch": 0.1048083111151972, "frac_reward_zero_std": 0.2, "grad_norm": 4.6910905838012695, "kl": 0.0033729393166140654, "learning_rate": 3.4895833333333333e-06, "loss": -0.0001, "num_tokens": 4961051.0, "reward": -0.275, "reward_std": 0.6784210562705993, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6784210681915284, "sampling/importance_sampling_ratio/max": 1.228622591495514, "sampling/importance_sampling_ratio/mean": 0.9997471153736115, "sampling/importance_sampling_ratio/min": 0.7630474984645843, "sampling/sampling_logp_difference/max": 0.2761464476585388, "sampling/sampling_logp_difference/mean": 0.022630465030670167, "step": 1140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.2, "completions/max_terminated_length": 314.2, "completions/mean_length": 210.4, "completions/mean_terminated_length": 210.4, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0697052132338285, "epoch": 0.10572768226533051, "frac_reward_zero_std": 0.2, "grad_norm": 5.9781293869018555, "kl": 0.003334551932493923, "learning_rate": 3.5202205882352947e-06, "loss": 0.0002, "num_tokens": 5002539.0, "reward": -0.20625, "reward_std": 0.693935227394104, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.6939352571964263, "sampling/importance_sampling_ratio/max": 1.2995353102684022, "sampling/importance_sampling_ratio/mean": 1.0001205146312713, "sampling/importance_sampling_ratio/min": 0.7657004237174988, "sampling/sampling_logp_difference/max": 0.28591806888580323, "sampling/sampling_logp_difference/mean": 0.020997253805398942, "step": 1150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.8, "completions/max_terminated_length": 292.8, "completions/mean_length": 214.875, "completions/mean_terminated_length": 214.875, "completions/min_length": 151.3, "completions/min_terminated_length": 151.3, "entropy": 1.140690841525793, "epoch": 0.10664705341546382, "frac_reward_zero_std": 0.2, "grad_norm": 6.6904191970825195, "kl": 0.003691112337401137, "learning_rate": 3.550857843137255e-06, "loss": 0.0007, "num_tokens": 5048409.0, "reward": -0.225, "reward_std": 0.7255426943302155, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.7255427181720734, "sampling/importance_sampling_ratio/max": 1.2627278566360474, "sampling/importance_sampling_ratio/mean": 1.0002590537071228, "sampling/importance_sampling_ratio/min": 0.7386715978384018, "sampling/sampling_logp_difference/max": 0.32672255039215087, "sampling/sampling_logp_difference/mean": 0.022760916128754617, "step": 1160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.9, "completions/max_terminated_length": 324.9, "completions/mean_length": 223.0375, "completions/mean_terminated_length": 223.0375, "completions/min_length": 155.5, "completions/min_terminated_length": 155.5, "entropy": 1.0626814141869545, "epoch": 0.10756642456559713, "frac_reward_zero_std": 0.3, "grad_norm": 8.584588050842285, "kl": 0.0044396442535799, "learning_rate": 3.581495098039216e-06, "loss": -0.0, "num_tokens": 5092196.0, "reward": -0.1, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2525327444076537, "sampling/importance_sampling_ratio/mean": 1.0001048028469086, "sampling/importance_sampling_ratio/min": 0.754019683599472, "sampling/sampling_logp_difference/max": 0.28982633352279663, "sampling/sampling_logp_difference/mean": 0.02212144173681736, "step": 1170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.3, "completions/max_terminated_length": 330.3, "completions/mean_length": 204.0375, "completions/mean_terminated_length": 204.0375, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0867654427886009, "epoch": 0.10848579571573044, "frac_reward_zero_std": 0.4, "grad_norm": 0.1350507140159607, "kl": 0.006295298783516046, "learning_rate": 3.6121323529411766e-06, "loss": -0.0002, "num_tokens": 5136655.0, "reward": -0.2625, "reward_std": 0.4915019154548645, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.49150192737579346, "sampling/importance_sampling_ratio/max": 1.390040636062622, "sampling/importance_sampling_ratio/mean": 1.0002334713935852, "sampling/importance_sampling_ratio/min": 0.6998829111456871, "sampling/sampling_logp_difference/max": 0.5231615900993347, "sampling/sampling_logp_difference/mean": 0.023195735178887844, "step": 1180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.9, "completions/max_terminated_length": 288.9, "completions/mean_length": 205.0375, "completions/mean_terminated_length": 205.0375, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 0.9735405094921589, "epoch": 0.10940516686586375, "frac_reward_zero_std": 0.4, "grad_norm": 5.498814582824707, "kl": 0.004569582993281074, "learning_rate": 3.6427696078431375e-06, "loss": 0.0003, "num_tokens": 5180786.0, "reward": 0.2875, "reward_std": 0.47637836784124377, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.47637839764356615, "sampling/importance_sampling_ratio/max": 1.291914200782776, "sampling/importance_sampling_ratio/mean": 1.0002384603023529, "sampling/importance_sampling_ratio/min": 0.7795219480991363, "sampling/sampling_logp_difference/max": 0.29397920370101926, "sampling/sampling_logp_difference/mean": 0.021471133176237344, "step": 1190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 350.2, "completions/max_terminated_length": 346.6, "completions/mean_length": 234.4, "completions/mean_terminated_length": 232.3000030517578, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.146508201956749, "epoch": 0.11032453801599706, "frac_reward_zero_std": 0.5, "grad_norm": 4.669467449188232, "kl": 0.004668859124649316, "learning_rate": 3.6734068627450984e-06, "loss": 0.0001, "num_tokens": 5229026.0, "reward": 0.04375, "reward_std": 0.48846607804298403, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.48846608996391294, "sampling/importance_sampling_ratio/max": 1.3217424988746642, "sampling/importance_sampling_ratio/mean": 1.0000478506088257, "sampling/importance_sampling_ratio/min": 0.6811802014708519, "sampling/sampling_logp_difference/max": 0.5259185910224915, "sampling/sampling_logp_difference/mean": 0.02270745113492012, "step": 1200 }, { "epoch": 0.11032453801599706, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.005671902268760907, "eval_completions/max_length": 318.1326352530541, "eval_completions/max_terminated_length": 312.5043630017452, "eval_completions/mean_length": 220.35929319371726, "eval_completions/mean_terminated_length": 219.0392797979385, "eval_completions/min_length": 143.90750436300175, "eval_completions/min_terminated_length": 143.90750436300175, "eval_entropy": 1.067815515643431, "eval_frac_reward_zero_std": 0.4048865619546248, "eval_kl": 0.0041569339391412975, "eval_loss": 0.0002309717092430219, "eval_num_tokens": 5229026.0, "eval_reward": -0.15783158813263526, "eval_reward_std": 0.505927014751376, "eval_rewards/reward_fn/mean": -0.15783158813263526, "eval_rewards/reward_fn/std": 0.5059270311868628, "eval_runtime": 1837.7855, "eval_samples_per_second": 0.312, "eval_sampling/importance_sampling_ratio/max": 1.2828964822995517, "eval_sampling/importance_sampling_ratio/mean": 0.999976660984765, "eval_sampling/importance_sampling_ratio/min": 0.7627015068759755, "eval_sampling/sampling_logp_difference/max": 0.3140193665006814, "eval_sampling/sampling_logp_difference/mean": 0.02126675082230859, "eval_steps_per_second": 0.039, "step": 1200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.1, "completions/max_terminated_length": 331.1, "completions/mean_length": 233.2375, "completions/mean_terminated_length": 233.2375, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 0.9978276476264, "epoch": 0.11124390916613036, "frac_reward_zero_std": 0.3, "grad_norm": 0.10827454924583435, "kl": 0.0038461405638372526, "learning_rate": 3.7040441176470593e-06, "loss": 0.0006, "num_tokens": 5276093.0, "reward": -0.2625, "reward_std": 0.5558690592646599, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.5558690890669823, "sampling/importance_sampling_ratio/max": 1.2944961071014405, "sampling/importance_sampling_ratio/mean": 1.0003490447998047, "sampling/importance_sampling_ratio/min": 0.6948616623878479, "sampling/sampling_logp_difference/max": 0.39618085622787474, "sampling/sampling_logp_difference/mean": 0.02043803222477436, "step": 1210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.3, "completions/max_terminated_length": 315.5, "completions/mean_length": 221.425, "completions/mean_terminated_length": 218.62857360839843, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.9814214553683996, "epoch": 0.11216328031626367, "frac_reward_zero_std": 0.4, "grad_norm": 4.405625820159912, "kl": 0.0034056433411024044, "learning_rate": 3.73468137254902e-06, "loss": 0.0004, "num_tokens": 5318695.0, "reward": -0.0125, "reward_std": 0.5011823400855064, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.5011823639273644, "sampling/importance_sampling_ratio/max": 1.2945587754249572, "sampling/importance_sampling_ratio/mean": 1.0001309514045715, "sampling/importance_sampling_ratio/min": 0.7831118106842041, "sampling/sampling_logp_difference/max": 0.28227869272232053, "sampling/sampling_logp_difference/mean": 0.0204302370082587, "step": 1220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.3, "completions/max_terminated_length": 270.3, "completions/mean_length": 201.8, "completions/mean_terminated_length": 201.8, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0407528646290303, "epoch": 0.11308265146639698, "frac_reward_zero_std": 0.7, "grad_norm": 0.0981360599398613, "kl": 0.004294796654721722, "learning_rate": 3.7653186274509808e-06, "loss": 0.0002, "num_tokens": 5359303.0, "reward": -0.65, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.2586891651153564, "sampling/importance_sampling_ratio/mean": 0.9997124969959259, "sampling/importance_sampling_ratio/min": 0.7649224400520325, "sampling/sampling_logp_difference/max": 0.2903438061475754, "sampling/sampling_logp_difference/mean": 0.021337260864675044, "step": 1230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.8, "completions/max_terminated_length": 312.8, "completions/mean_length": 210.2125, "completions/mean_terminated_length": 210.2125, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0735914170742036, "epoch": 0.1140020226165303, "frac_reward_zero_std": 0.6, "grad_norm": 0.07430858910083771, "kl": 0.004256828321376816, "learning_rate": 3.7959558823529413e-06, "loss": 0.0002, "num_tokens": 5404576.0, "reward": 0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2625141739845276, "sampling/importance_sampling_ratio/mean": 1.000032889842987, "sampling/importance_sampling_ratio/min": 0.7131502598524093, "sampling/sampling_logp_difference/max": 0.3669726014137268, "sampling/sampling_logp_difference/mean": 0.02155184801667929, "step": 1240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.0, "completions/max_terminated_length": 279.0, "completions/mean_length": 201.0375, "completions/mean_terminated_length": 201.0375, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.013702467828989, "epoch": 0.1149213937666636, "frac_reward_zero_std": 0.2, "grad_norm": 6.665727138519287, "kl": 0.0038952339440584184, "learning_rate": 3.826593137254902e-06, "loss": -0.0002, "num_tokens": 5445587.0, "reward": 0.0125, "reward_std": 0.6539935395121574, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.6539935693144798, "sampling/importance_sampling_ratio/max": 1.2494479894638062, "sampling/importance_sampling_ratio/mean": 1.0003366231918336, "sampling/importance_sampling_ratio/min": 0.7206558614969254, "sampling/sampling_logp_difference/max": 0.3531486988067627, "sampling/sampling_logp_difference/mean": 0.02120848447084427, "step": 1250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.8, "completions/max_terminated_length": 284.8, "completions/mean_length": 195.775, "completions/mean_terminated_length": 195.775, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 0.96270222812891, "epoch": 0.11584076491679692, "frac_reward_zero_std": 0.6, "grad_norm": 0.0751090869307518, "kl": 0.0039115644540288486, "learning_rate": 3.857230392156863e-06, "loss": 0.0005, "num_tokens": 5482353.0, "reward": 0.2, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.2567622303962707, "sampling/importance_sampling_ratio/mean": 0.9999596297740936, "sampling/importance_sampling_ratio/min": 0.7956125319004059, "sampling/sampling_logp_difference/max": 0.24971704483032225, "sampling/sampling_logp_difference/mean": 0.020302335917949676, "step": 1260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 223.7625, "completions/mean_terminated_length": 223.7625, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 1.1314456053078175, "epoch": 0.11676013606693023, "frac_reward_zero_std": 0.5, "grad_norm": 5.154245853424072, "kl": 0.004506811234750785, "learning_rate": 3.887867647058824e-06, "loss": 0.0, "num_tokens": 5527358.0, "reward": -0.00625, "reward_std": 0.4708753764629364, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.47087539434432985, "sampling/importance_sampling_ratio/max": 1.279786503314972, "sampling/importance_sampling_ratio/mean": 0.999714457988739, "sampling/importance_sampling_ratio/min": 0.7777524292469025, "sampling/sampling_logp_difference/max": 0.27552487552165983, "sampling/sampling_logp_difference/mean": 0.022305747494101524, "step": 1270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 209.2, "completions/mean_terminated_length": 209.2, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.0697576768696309, "epoch": 0.11767950721706352, "frac_reward_zero_std": 0.5, "grad_norm": 7.813284397125244, "kl": 0.004821015807101503, "learning_rate": 3.918504901960785e-06, "loss": 0.0001, "num_tokens": 5571790.0, "reward": -0.125, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.3216192603111268, "sampling/importance_sampling_ratio/mean": 1.0002794861793518, "sampling/importance_sampling_ratio/min": 0.7769640505313873, "sampling/sampling_logp_difference/max": 0.29776922762393954, "sampling/sampling_logp_difference/mean": 0.02150882314890623, "step": 1280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.7, "completions/max_terminated_length": 247.7, "completions/mean_length": 169.075, "completions/mean_terminated_length": 169.075, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 0.8968605823814869, "epoch": 0.11859887836719683, "frac_reward_zero_std": 0.4, "grad_norm": 0.12292447686195374, "kl": 0.008229461711016484, "learning_rate": 3.9491421568627455e-06, "loss": 0.0007, "num_tokens": 5607900.0, "reward": -0.15, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.3008840680122375, "sampling/importance_sampling_ratio/mean": 1.000325345993042, "sampling/importance_sampling_ratio/min": 0.7930408120155334, "sampling/sampling_logp_difference/max": 0.28127992153167725, "sampling/sampling_logp_difference/mean": 0.019989703968167304, "step": 1290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.5, "completions/max_terminated_length": 280.5, "completions/mean_length": 194.225, "completions/mean_terminated_length": 194.225, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 0.9936360083520412, "epoch": 0.11951824951733014, "frac_reward_zero_std": 0.5, "grad_norm": 6.092287063598633, "kl": 0.004593243860290386, "learning_rate": 3.979779411764706e-06, "loss": 0.0007, "num_tokens": 5651046.0, "reward": -0.35, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2518170356750489, "sampling/importance_sampling_ratio/mean": 0.9992234945297241, "sampling/importance_sampling_ratio/min": 0.7824578225612641, "sampling/sampling_logp_difference/max": 0.25957123637199403, "sampling/sampling_logp_difference/mean": 0.02077052593231201, "step": 1300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 208.45, "completions/mean_terminated_length": 208.45, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.0042010121047498, "epoch": 0.12043762066746345, "frac_reward_zero_std": 0.3, "grad_norm": 5.101361274719238, "kl": 0.004405161170870997, "learning_rate": 4.010416666666667e-06, "loss": 0.0003, "num_tokens": 5695578.0, "reward": 0.025, "reward_std": 0.6767033517360688, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6767033755779266, "sampling/importance_sampling_ratio/max": 1.3843995809555054, "sampling/importance_sampling_ratio/mean": 0.9999094188213349, "sampling/importance_sampling_ratio/min": 0.7536225259304047, "sampling/sampling_logp_difference/max": 0.40685489773750305, "sampling/sampling_logp_difference/mean": 0.021114468201994897, "step": 1310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 225.8, "completions/mean_terminated_length": 225.8, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.1614429451525212, "epoch": 0.12135699181759677, "frac_reward_zero_std": 0.6, "grad_norm": 6.484318733215332, "kl": 0.006007309338747291, "learning_rate": 4.041053921568628e-06, "loss": 0.0002, "num_tokens": 5739114.0, "reward": 0.05, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.2683799266815186, "sampling/importance_sampling_ratio/mean": 0.9994219303131103, "sampling/importance_sampling_ratio/min": 0.7826501846313476, "sampling/sampling_logp_difference/max": 0.26778310537338257, "sampling/sampling_logp_difference/mean": 0.022283046413213013, "step": 1320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 335.3, "completions/max_terminated_length": 330.2, "completions/mean_length": 239.7125, "completions/mean_terminated_length": 237.82857360839844, "completions/min_length": 170.0, "completions/min_terminated_length": 170.0, "entropy": 1.0879293970763684, "epoch": 0.12227636296773008, "frac_reward_zero_std": 0.4, "grad_norm": 0.10041707754135132, "kl": 0.0059077501675346865, "learning_rate": 4.071691176470588e-06, "loss": 0.0006, "num_tokens": 5787259.0, "reward": -0.13125, "reward_std": 0.46962851732969285, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.4696285530924797, "sampling/importance_sampling_ratio/max": 1.3359501600265502, "sampling/importance_sampling_ratio/mean": 1.0002283096313476, "sampling/importance_sampling_ratio/min": 0.7784946084022522, "sampling/sampling_logp_difference/max": 0.3245913624763489, "sampling/sampling_logp_difference/mean": 0.02081667333841324, "step": 1330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 231.125, "completions/mean_terminated_length": 231.125, "completions/min_length": 163.3, "completions/min_terminated_length": 163.3, "entropy": 1.0511620368808507, "epoch": 0.12319573411786339, "frac_reward_zero_std": 0.4, "grad_norm": 0.08397785574197769, "kl": 0.005463651182071772, "learning_rate": 4.102328431372549e-06, "loss": 0.0004, "num_tokens": 5833709.0, "reward": -0.05625, "reward_std": 0.4982891783118248, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.49828919023275375, "sampling/importance_sampling_ratio/max": 1.2389087796211242, "sampling/importance_sampling_ratio/mean": 0.9996604442596435, "sampling/importance_sampling_ratio/min": 0.7840703129768372, "sampling/sampling_logp_difference/max": 0.25620375871658324, "sampling/sampling_logp_difference/mean": 0.02038785219192505, "step": 1340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.4, "completions/max_terminated_length": 329.4, "completions/mean_length": 222.2875, "completions/mean_terminated_length": 222.2875, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 1.1444878630340098, "epoch": 0.12411510526799668, "frac_reward_zero_std": 0.4, "grad_norm": 5.481135368347168, "kl": 0.005984389304649085, "learning_rate": 4.13296568627451e-06, "loss": 0.0003, "num_tokens": 5880620.0, "reward": 0.35, "reward_std": 0.6135258316993714, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.6135258793830871, "sampling/importance_sampling_ratio/max": 1.2724161624908448, "sampling/importance_sampling_ratio/mean": 0.9996695160865784, "sampling/importance_sampling_ratio/min": 0.7898568212985992, "sampling/sampling_logp_difference/max": 0.27834402918815615, "sampling/sampling_logp_difference/mean": 0.022280721738934518, "step": 1350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.8, "completions/max_terminated_length": 293.8, "completions/mean_length": 210.7875, "completions/mean_terminated_length": 210.7875, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.1222901090979576, "epoch": 0.12503447641813, "frac_reward_zero_std": 0.6, "grad_norm": 0.09709859639406204, "kl": 0.005905412789434195, "learning_rate": 4.1636029411764715e-06, "loss": 0.0003, "num_tokens": 5928851.0, "reward": 0.275, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.3598933458328246, "sampling/importance_sampling_ratio/mean": 1.0000855505466462, "sampling/importance_sampling_ratio/min": 0.8022277176380157, "sampling/sampling_logp_difference/max": 0.30679272413253783, "sampling/sampling_logp_difference/mean": 0.02132700476795435, "step": 1360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.2, "completions/max_terminated_length": 313.7, "completions/mean_length": 204.0375, "completions/mean_terminated_length": 201.51785888671876, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 1.0529061183333397, "epoch": 0.1259538475682633, "frac_reward_zero_std": 0.5, "grad_norm": 0.12638092041015625, "kl": 0.00508540990849724, "learning_rate": 4.194240196078432e-06, "loss": 0.0004, "num_tokens": 5970246.0, "reward": 0.26875, "reward_std": 0.41947310566902163, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.41947310566902163, "sampling/importance_sampling_ratio/max": 1.30142480134964, "sampling/importance_sampling_ratio/mean": 0.9999548316001892, "sampling/importance_sampling_ratio/min": 0.7245550215244293, "sampling/sampling_logp_difference/max": 0.3807424545288086, "sampling/sampling_logp_difference/mean": 0.021413808688521385, "step": 1370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.7, "completions/max_terminated_length": 310.7, "completions/mean_length": 208.2125, "completions/mean_terminated_length": 208.2125, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.1299782782793044, "epoch": 0.12687321871839663, "frac_reward_zero_std": 0.3, "grad_norm": 7.523990631103516, "kl": 0.005992138932924718, "learning_rate": 4.2248774509803925e-06, "loss": 0.0006, "num_tokens": 6013951.0, "reward": -0.18125, "reward_std": 0.6408152520656586, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.6408152759075165, "sampling/importance_sampling_ratio/max": 1.2439507007598878, "sampling/importance_sampling_ratio/mean": 0.9997854888439178, "sampling/importance_sampling_ratio/min": 0.7221481144428253, "sampling/sampling_logp_difference/max": 0.3603989124298096, "sampling/sampling_logp_difference/mean": 0.021925201639533043, "step": 1380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.3, "completions/max_terminated_length": 306.3, "completions/mean_length": 222.4375, "completions/mean_terminated_length": 222.4375, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.1457575887441636, "epoch": 0.12779258986852993, "frac_reward_zero_std": 0.4, "grad_norm": 6.5946760177612305, "kl": 0.005931628748658113, "learning_rate": 4.255514705882353e-06, "loss": 0.0001, "num_tokens": 6060810.0, "reward": -0.4, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.3694018006324769, "sampling/importance_sampling_ratio/mean": 1.000184553861618, "sampling/importance_sampling_ratio/min": 0.8071199774742126, "sampling/sampling_logp_difference/max": 0.3114263415336609, "sampling/sampling_logp_difference/mean": 0.021994932554662228, "step": 1390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.9, "completions/max_terminated_length": 274.9, "completions/mean_length": 198.2, "completions/mean_terminated_length": 198.2, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.0926256619393826, "epoch": 0.12871196101866322, "frac_reward_zero_std": 0.4, "grad_norm": 6.28213357925415, "kl": 0.005877035576850176, "learning_rate": 4.2861519607843135e-06, "loss": 0.0003, "num_tokens": 6105298.0, "reward": -0.325, "reward_std": 0.5773320138454437, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5773320615291595, "sampling/importance_sampling_ratio/max": 1.2843565940856934, "sampling/importance_sampling_ratio/mean": 1.0006065607070922, "sampling/importance_sampling_ratio/min": 0.7772300660610199, "sampling/sampling_logp_difference/max": 0.2836942672729492, "sampling/sampling_logp_difference/mean": 0.020828850008547305, "step": 1400 }, { "epoch": 0.12871196101866322, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 287.01221640488654, "eval_completions/max_terminated_length": 285.0418848167539, "eval_completions/mean_length": 198.3854712041885, "eval_completions/mean_terminated_length": 197.8735354758058, "eval_completions/min_length": 127.29842931937172, "eval_completions/min_terminated_length": 127.29842931937172, "eval_entropy": 1.1042271299095887, "eval_frac_reward_zero_std": 0.4799301919720768, "eval_kl": 0.005685513568505723, "eval_loss": 0.00026846470427699387, "eval_num_tokens": 6105298.0, "eval_reward": -0.15008726003490402, "eval_reward_std": 0.4544920666471618, "eval_rewards/reward_fn/mean": -0.15008726003490402, "eval_rewards/reward_fn/std": 0.45449208162633953, "eval_runtime": 1680.952, "eval_samples_per_second": 0.341, "eval_sampling/importance_sampling_ratio/max": 1.267474968187888, "eval_sampling/importance_sampling_ratio/mean": 1.0000139350042294, "eval_sampling/importance_sampling_ratio/min": 0.7717791251565149, "eval_sampling/sampling_logp_difference/max": 0.2921701604679201, "eval_sampling/sampling_logp_difference/mean": 0.021077305148396296, "eval_steps_per_second": 0.043, "step": 1400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.6, "completions/max_terminated_length": 270.6, "completions/mean_length": 193.825, "completions/mean_terminated_length": 193.825, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.0765355192124844, "epoch": 0.12963133216879655, "frac_reward_zero_std": 0.8, "grad_norm": 0.10633435100317001, "kl": 0.00502969676599605, "learning_rate": 4.316789215686275e-06, "loss": 0.0003, "num_tokens": 6147652.0, "reward": -0.25, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.228092908859253, "sampling/importance_sampling_ratio/mean": 0.9994353294372559, "sampling/importance_sampling_ratio/min": 0.8049301207065582, "sampling/sampling_logp_difference/max": 0.23094289302825927, "sampling/sampling_logp_difference/mean": 0.021201788168400525, "step": 1410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 198.4625, "completions/mean_terminated_length": 198.4625, "completions/min_length": 109.0, "completions/min_terminated_length": 109.0, "entropy": 1.0187456153333188, "epoch": 0.13055070331892984, "frac_reward_zero_std": 0.7, "grad_norm": 0.16345858573913574, "kl": 0.005319896445143968, "learning_rate": 4.347426470588235e-06, "loss": 0.0003, "num_tokens": 6189849.0, "reward": -0.25, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2896673083305359, "sampling/importance_sampling_ratio/mean": 1.0003308832645417, "sampling/importance_sampling_ratio/min": 0.7075693607330322, "sampling/sampling_logp_difference/max": 0.37187924385070803, "sampling/sampling_logp_difference/mean": 0.02077707536518574, "step": 1420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 210.9, "completions/mean_terminated_length": 210.9, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.001291160658002, "epoch": 0.13147007446906317, "frac_reward_zero_std": 0.2, "grad_norm": 6.712734222412109, "kl": 0.004692256468115374, "learning_rate": 4.378063725490197e-06, "loss": -0.0002, "num_tokens": 6230769.0, "reward": -0.13125, "reward_std": 0.7067587494850158, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.7067587733268738, "sampling/importance_sampling_ratio/max": 1.3609673857688904, "sampling/importance_sampling_ratio/mean": 0.999944144487381, "sampling/importance_sampling_ratio/min": 0.7840885937213897, "sampling/sampling_logp_difference/max": 0.328003454208374, "sampling/sampling_logp_difference/mean": 0.02043430833145976, "step": 1430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.7, "completions/max_terminated_length": 320.7, "completions/mean_length": 223.9875, "completions/mean_terminated_length": 223.9875, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.0741894267499448, "epoch": 0.13238944561919647, "frac_reward_zero_std": 0.3, "grad_norm": 0.07823669910430908, "kl": 0.005145831701520364, "learning_rate": 4.408700980392157e-06, "loss": -0.0001, "num_tokens": 6273464.0, "reward": 0.25, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.3091425776481629, "sampling/importance_sampling_ratio/mean": 1.0000085532665253, "sampling/importance_sampling_ratio/min": 0.7394974529743195, "sampling/sampling_logp_difference/max": 0.3405857503414154, "sampling/sampling_logp_difference/mean": 0.022102873586118222, "step": 1440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 197.1, "completions/mean_terminated_length": 197.1, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 0.9857777960598468, "epoch": 0.1333088167693298, "frac_reward_zero_std": 0.2, "grad_norm": 5.30831241607666, "kl": 0.006255742415669374, "learning_rate": 4.439338235294118e-06, "loss": 0.0008, "num_tokens": 6314904.0, "reward": -0.2, "reward_std": 0.7187690615653992, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.718769085407257, "sampling/importance_sampling_ratio/max": 1.2603877305984497, "sampling/importance_sampling_ratio/mean": 1.0002587735652924, "sampling/importance_sampling_ratio/min": 0.7616799056529999, "sampling/sampling_logp_difference/max": 0.2963370144367218, "sampling/sampling_logp_difference/mean": 0.021206727996468545, "step": 1450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.4, "completions/max_terminated_length": 285.4, "completions/mean_length": 211.85, "completions/mean_terminated_length": 211.85, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 0.968796381354332, "epoch": 0.1342281879194631, "frac_reward_zero_std": 0.4, "grad_norm": 0.07986993342638016, "kl": 0.006176900261198171, "learning_rate": 4.469975490196079e-06, "loss": 0.0002, "num_tokens": 6359764.0, "reward": -0.025, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.3567649483680726, "sampling/importance_sampling_ratio/mean": 0.999694311618805, "sampling/importance_sampling_ratio/min": 0.7824565291404724, "sampling/sampling_logp_difference/max": 0.36064422130584717, "sampling/sampling_logp_difference/mean": 0.02010431569069624, "step": 1460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.5, "completions/max_terminated_length": 262.5, "completions/mean_length": 190.2125, "completions/mean_terminated_length": 190.2125, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0152163669466971, "epoch": 0.13514755906959638, "frac_reward_zero_std": 0.5, "grad_norm": 6.681584358215332, "kl": 0.004536326776724308, "learning_rate": 4.5006127450980395e-06, "loss": 0.0003, "num_tokens": 6399117.0, "reward": -0.025, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.234797477722168, "sampling/importance_sampling_ratio/mean": 0.9995251715183258, "sampling/importance_sampling_ratio/min": 0.8046580910682678, "sampling/sampling_logp_difference/max": 0.22579296827316284, "sampling/sampling_logp_difference/mean": 0.02104624453932047, "step": 1470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 192.6875, "completions/mean_terminated_length": 192.6875, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 0.990882970392704, "epoch": 0.1360669302197297, "frac_reward_zero_std": 0.7, "grad_norm": 0.08012332767248154, "kl": 0.0049002677216776645, "learning_rate": 4.53125e-06, "loss": 0.0003, "num_tokens": 6441452.0, "reward": 0.025, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.3065441489219665, "sampling/importance_sampling_ratio/mean": 1.0006222486495973, "sampling/importance_sampling_ratio/min": 0.7439396500587463, "sampling/sampling_logp_difference/max": 0.32861300706863406, "sampling/sampling_logp_difference/mean": 0.020516683161258698, "step": 1480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.8, "completions/max_terminated_length": 311.8, "completions/mean_length": 225.1375, "completions/mean_terminated_length": 225.1375, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 1.1216545231640338, "epoch": 0.136986301369863, "frac_reward_zero_std": 0.6, "grad_norm": 6.090054035186768, "kl": 0.00512039921595715, "learning_rate": 4.561887254901961e-06, "loss": 0.0002, "num_tokens": 6486023.0, "reward": -0.25, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2582533836364747, "sampling/importance_sampling_ratio/mean": 0.9994962573051452, "sampling/importance_sampling_ratio/min": 0.7660017490386963, "sampling/sampling_logp_difference/max": 0.28443912863731385, "sampling/sampling_logp_difference/mean": 0.021003433503210545, "step": 1490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 212.7875, "completions/mean_terminated_length": 212.7875, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0975494764745235, "epoch": 0.13790567251999633, "frac_reward_zero_std": 0.3, "grad_norm": 5.93871545791626, "kl": 0.0060364163655322045, "learning_rate": 4.592524509803922e-06, "loss": 0.0005, "num_tokens": 6533918.0, "reward": -0.18125, "reward_std": 0.686385840177536, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.6863858520984649, "sampling/importance_sampling_ratio/max": 1.2779659271240233, "sampling/importance_sampling_ratio/mean": 1.000190222263336, "sampling/importance_sampling_ratio/min": 0.7645667970180512, "sampling/sampling_logp_difference/max": 0.2859131693840027, "sampling/sampling_logp_difference/mean": 0.02096188087016344, "step": 1500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 194.55, "completions/mean_terminated_length": 194.55, "completions/min_length": 117.1, "completions/min_terminated_length": 117.1, "entropy": 1.077169705182314, "epoch": 0.13882504367012963, "frac_reward_zero_std": 0.4, "grad_norm": 0.09802401810884476, "kl": 0.005575049226172269, "learning_rate": 4.623161764705883e-06, "loss": 0.0006, "num_tokens": 6579738.0, "reward": -0.3, "reward_std": 0.5226615250110627, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5226615607738495, "sampling/importance_sampling_ratio/max": 1.3519586086273194, "sampling/importance_sampling_ratio/mean": 0.9998842477798462, "sampling/importance_sampling_ratio/min": 0.7850383162498474, "sampling/sampling_logp_difference/max": 0.3397265076637268, "sampling/sampling_logp_difference/mean": 0.021545779518783094, "step": 1510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.5, "completions/max_terminated_length": 262.5, "completions/mean_length": 181.15, "completions/mean_terminated_length": 181.15, "completions/min_length": 118.2, "completions/min_terminated_length": 118.2, "entropy": 1.0568670578300954, "epoch": 0.13974441482026295, "frac_reward_zero_std": 0.2, "grad_norm": 6.223104476928711, "kl": 0.00639324348885566, "learning_rate": 4.653799019607844e-06, "loss": 0.0001, "num_tokens": 6621598.0, "reward": 0.025, "reward_std": 0.7296968817710876, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.7296969175338746, "sampling/importance_sampling_ratio/max": 1.2864805817604066, "sampling/importance_sampling_ratio/mean": 1.0004252791404724, "sampling/importance_sampling_ratio/min": 0.7965053677558899, "sampling/sampling_logp_difference/max": 0.26050500869750975, "sampling/sampling_logp_difference/mean": 0.021610717102885247, "step": 1520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 287.6, "completions/max_terminated_length": 274.1, "completions/mean_length": 192.0, "completions/mean_terminated_length": 188.47857208251952, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.0190536431968211, "epoch": 0.14066378597039625, "frac_reward_zero_std": 0.7, "grad_norm": 0.160631000995636, "kl": 0.010466382774757222, "learning_rate": 4.684436274509804e-06, "loss": 0.0005, "num_tokens": 6664070.0, "reward": -0.63125, "reward_std": 0.18097035735845565, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.18097035735845565, "sampling/importance_sampling_ratio/max": 1.2771115899085999, "sampling/importance_sampling_ratio/mean": 1.0001590073108673, "sampling/importance_sampling_ratio/min": 0.7835388720035553, "sampling/sampling_logp_difference/max": 0.28040971159934996, "sampling/sampling_logp_difference/mean": 0.02140217460691929, "step": 1530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.6, "completions/max_terminated_length": 291.6, "completions/mean_length": 202.2, "completions/mean_terminated_length": 202.2, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.086371947824955, "epoch": 0.14158315712052955, "frac_reward_zero_std": 0.2, "grad_norm": 6.947596073150635, "kl": 0.007361587104969658, "learning_rate": 4.715073529411765e-06, "loss": 0.0005, "num_tokens": 6710374.0, "reward": 0.125, "reward_std": 0.7112202107906341, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.7112202346324921, "sampling/importance_sampling_ratio/max": 1.2322092056274414, "sampling/importance_sampling_ratio/mean": 0.9994845986366272, "sampling/importance_sampling_ratio/min": 0.788561999797821, "sampling/sampling_logp_difference/max": 0.24470129013061523, "sampling/sampling_logp_difference/mean": 0.02149435617029667, "step": 1540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.2, "completions/max_terminated_length": 244.2, "completions/mean_length": 169.175, "completions/mean_terminated_length": 169.175, "completions/min_length": 106.2, "completions/min_terminated_length": 106.2, "entropy": 1.0657698854804039, "epoch": 0.14250252827066287, "frac_reward_zero_std": 0.4, "grad_norm": 0.1104065552353859, "kl": 0.006785096530802548, "learning_rate": 4.745710784313726e-06, "loss": 0.0001, "num_tokens": 6750012.0, "reward": 0.125, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2504637718200684, "sampling/importance_sampling_ratio/mean": 0.9994118690490723, "sampling/importance_sampling_ratio/min": 0.7676426231861114, "sampling/sampling_logp_difference/max": 0.27286410331726074, "sampling/sampling_logp_difference/mean": 0.021456832811236382, "step": 1550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.2, "completions/max_terminated_length": 254.2, "completions/mean_length": 171.5375, "completions/mean_terminated_length": 171.5375, "completions/min_length": 114.8, "completions/min_terminated_length": 114.8, "entropy": 1.0693279758095742, "epoch": 0.14342189942079617, "frac_reward_zero_std": 0.3, "grad_norm": 8.026269912719727, "kl": 0.008455702004721389, "learning_rate": 4.7763480392156866e-06, "loss": 0.0006, "num_tokens": 6791423.0, "reward": -0.525, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2635118007659911, "sampling/importance_sampling_ratio/mean": 0.9995228230953217, "sampling/importance_sampling_ratio/min": 0.7623455166816712, "sampling/sampling_logp_difference/max": 0.295576286315918, "sampling/sampling_logp_difference/mean": 0.02288513518869877, "step": 1560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 272.0, "completions/max_terminated_length": 250.1, "completions/mean_length": 182.6, "completions/mean_terminated_length": 179.08214416503907, "completions/min_length": 108.1, "completions/min_terminated_length": 108.1, "entropy": 1.13099010810256, "epoch": 0.1443412705709295, "frac_reward_zero_std": 0.3, "grad_norm": 6.107755661010742, "kl": 0.007700269622728228, "learning_rate": 4.806985294117648e-06, "loss": 0.0008, "num_tokens": 6832863.0, "reward": 0.26875, "reward_std": 0.6660812497138977, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.6660812616348266, "sampling/importance_sampling_ratio/max": 1.2474673509597778, "sampling/importance_sampling_ratio/mean": 1.0000731706619264, "sampling/importance_sampling_ratio/min": 0.7657226860523224, "sampling/sampling_logp_difference/max": 0.27794852256774905, "sampling/sampling_logp_difference/mean": 0.022988536581397056, "step": 1570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.7, "completions/max_terminated_length": 241.7, "completions/mean_length": 171.6125, "completions/mean_terminated_length": 171.6125, "completions/min_length": 108.8, "completions/min_terminated_length": 108.8, "entropy": 1.0121632996946572, "epoch": 0.1452606417210628, "frac_reward_zero_std": 0.3, "grad_norm": 0.12659914791584015, "kl": 0.0066006919339997696, "learning_rate": 4.837622549019608e-06, "loss": 0.0004, "num_tokens": 6870568.0, "reward": 0.16875, "reward_std": 0.5656208857893944, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.5656208857893944, "sampling/importance_sampling_ratio/max": 1.2542443513870238, "sampling/importance_sampling_ratio/mean": 0.9996154844760895, "sampling/importance_sampling_ratio/min": 0.7739153504371643, "sampling/sampling_logp_difference/max": 0.2776555925607681, "sampling/sampling_logp_difference/mean": 0.021532597579061986, "step": 1580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.1, "completions/max_terminated_length": 262.1, "completions/mean_length": 174.4125, "completions/mean_terminated_length": 174.4125, "completions/min_length": 102.7, "completions/min_terminated_length": 102.7, "entropy": 1.056274377554655, "epoch": 0.1461800128711961, "frac_reward_zero_std": 0.8, "grad_norm": 0.08866989612579346, "kl": 0.007677699081250466, "learning_rate": 4.868259803921569e-06, "loss": 0.0003, "num_tokens": 6909689.0, "reward": 0.05, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2406139850616456, "sampling/importance_sampling_ratio/mean": 1.0003020644187928, "sampling/importance_sampling_ratio/min": 0.7592837333679199, "sampling/sampling_logp_difference/max": 0.28732244968414306, "sampling/sampling_logp_difference/mean": 0.021396166272461416, "step": 1590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.1, "completions/max_terminated_length": 300.1, "completions/mean_length": 200.9125, "completions/mean_terminated_length": 200.9125, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.0975555822253227, "epoch": 0.1470993840213294, "frac_reward_zero_std": 0.3, "grad_norm": 7.27940034866333, "kl": 0.0071688009280478585, "learning_rate": 4.898897058823529e-06, "loss": 0.0001, "num_tokens": 6950578.0, "reward": -0.3, "reward_std": 0.5605887413024903, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5605887413024903, "sampling/importance_sampling_ratio/max": 1.3361032843589782, "sampling/importance_sampling_ratio/mean": 0.9997352719306946, "sampling/importance_sampling_ratio/min": 0.7706340551376343, "sampling/sampling_logp_difference/max": 0.3277870535850525, "sampling/sampling_logp_difference/mean": 0.022191434726119043, "step": 1600 }, { "epoch": 0.1470993840213294, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0006544502617801048, "eval_completions/max_length": 282.6876090750436, "eval_completions/max_terminated_length": 281.7556719022688, "eval_completions/mean_length": 198.37194589877836, "eval_completions/mean_terminated_length": 198.19262660693957, "eval_completions/min_length": 127.30541012216405, "eval_completions/min_terminated_length": 127.30541012216405, "eval_entropy": 1.0355058373789097, "eval_frac_reward_zero_std": 0.4607329842931937, "eval_kl": 0.006847355883311721, "eval_loss": 0.0003164914669468999, "eval_num_tokens": 6950578.0, "eval_reward": -0.12030977312390925, "eval_reward_std": 0.47213354173651123, "eval_rewards/reward_fn/mean": -0.12030977312390925, "eval_rewards/reward_fn/std": 0.47213355744384344, "eval_runtime": 1657.5259, "eval_samples_per_second": 0.346, "eval_sampling/importance_sampling_ratio/max": 1.2794355996616225, "eval_sampling/importance_sampling_ratio/mean": 1.0000021116479738, "eval_sampling/importance_sampling_ratio/min": 0.768444153858937, "eval_sampling/sampling_logp_difference/max": 0.3027959303185994, "eval_sampling/sampling_logp_difference/mean": 0.020665150681871393, "eval_steps_per_second": 0.043, "step": 1600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 206.6, "completions/mean_terminated_length": 206.6, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.0377972185611726, "epoch": 0.1480187551714627, "frac_reward_zero_std": 0.5, "grad_norm": 0.16842025518417358, "kl": 0.007496340433135629, "learning_rate": 4.929534313725491e-06, "loss": 0.0008, "num_tokens": 6992866.0, "reward": -0.35625, "reward_std": 0.4475050806999207, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.4475050926208496, "sampling/importance_sampling_ratio/max": 1.3030020594596863, "sampling/importance_sampling_ratio/mean": 1.0000835418701173, "sampling/importance_sampling_ratio/min": 0.7157977752387523, "sampling/sampling_logp_difference/max": 0.475048828125, "sampling/sampling_logp_difference/mean": 0.021372266486287116, "step": 1610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.5, "completions/max_terminated_length": 272.5, "completions/mean_length": 189.8125, "completions/mean_terminated_length": 189.8125, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.0085635162889957, "epoch": 0.14893812632159603, "frac_reward_zero_std": 0.6, "grad_norm": 0.17543968558311462, "kl": 0.007132637084578164, "learning_rate": 4.960171568627451e-06, "loss": 0.0004, "num_tokens": 7035147.0, "reward": 0.075, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.232132863998413, "sampling/importance_sampling_ratio/mean": 0.9997389256954193, "sampling/importance_sampling_ratio/min": 0.7771494150161743, "sampling/sampling_logp_difference/max": 0.26644191741943357, "sampling/sampling_logp_difference/mean": 0.021017968468368054, "step": 1620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.2, "completions/max_terminated_length": 315.2, "completions/mean_length": 221.8, "completions/mean_terminated_length": 221.8, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.073548749834299, "epoch": 0.14985749747172933, "frac_reward_zero_std": 0.3, "grad_norm": 6.494510650634766, "kl": 0.0066894360788865015, "learning_rate": 4.990808823529413e-06, "loss": 0.0009, "num_tokens": 7079003.0, "reward": -0.2, "reward_std": 0.5168460726737976, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5168460726737976, "sampling/importance_sampling_ratio/max": 1.221497929096222, "sampling/importance_sampling_ratio/mean": 0.9995462596416473, "sampling/importance_sampling_ratio/min": 0.7898821890354156, "sampling/sampling_logp_difference/max": 0.23973069190979004, "sampling/sampling_logp_difference/mean": 0.020719478838145733, "step": 1630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 216.5, "completions/mean_terminated_length": 216.5, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.0623955361545085, "epoch": 0.15077686862186265, "frac_reward_zero_std": 0.5, "grad_norm": 0.09507337957620621, "kl": 0.008266611120779999, "learning_rate": 4.9999993709133756e-06, "loss": 0.0002, "num_tokens": 7121155.0, "reward": -0.125, "reward_std": 0.48816035985946654, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4881603717803955, "sampling/importance_sampling_ratio/max": 1.285227882862091, "sampling/importance_sampling_ratio/mean": 1.0001374363899231, "sampling/importance_sampling_ratio/min": 0.7572620451450348, "sampling/sampling_logp_difference/max": 0.31640519499778746, "sampling/sampling_logp_difference/mean": 0.02176865767687559, "step": 1640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 204.075, "completions/mean_terminated_length": 204.075, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.085703296214342, "epoch": 0.15169623977199595, "frac_reward_zero_std": 0.7, "grad_norm": 0.08493228256702423, "kl": 0.006988291465677321, "learning_rate": 4.999996289673523e-06, "loss": -0.0001, "num_tokens": 7161169.0, "reward": 0.425, "reward_std": 0.30299633741378784, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.3029963493347168, "sampling/importance_sampling_ratio/max": 1.2630980134010314, "sampling/importance_sampling_ratio/mean": 1.0005512475967406, "sampling/importance_sampling_ratio/min": 0.7768666446208954, "sampling/sampling_logp_difference/max": 0.2644318461418152, "sampling/sampling_logp_difference/mean": 0.021617133542895316, "step": 1650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.5, "completions/max_terminated_length": 271.5, "completions/mean_length": 194.1625, "completions/mean_terminated_length": 194.1625, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.1018084548413754, "epoch": 0.15261561092212927, "frac_reward_zero_std": 0.4, "grad_norm": 0.1122647300362587, "kl": 0.006595259191817604, "learning_rate": 4.9999906407370825e-06, "loss": 0.0003, "num_tokens": 7202454.0, "reward": -0.175, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.2791176676750182, "sampling/importance_sampling_ratio/mean": 1.0003723084926606, "sampling/importance_sampling_ratio/min": 0.7942777812480927, "sampling/sampling_logp_difference/max": 0.2569770336151123, "sampling/sampling_logp_difference/mean": 0.021152010560035704, "step": 1660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.8, "completions/max_terminated_length": 320.8, "completions/mean_length": 210.4125, "completions/mean_terminated_length": 210.4125, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.1089817076921462, "epoch": 0.15353498207226257, "frac_reward_zero_std": 0.4, "grad_norm": 0.10599858313798904, "kl": 0.008286274355486966, "learning_rate": 4.999982424109855e-06, "loss": 0.0004, "num_tokens": 7243439.0, "reward": -0.275, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2682781100273133, "sampling/importance_sampling_ratio/mean": 0.9998232841491699, "sampling/importance_sampling_ratio/min": 0.7653062760829925, "sampling/sampling_logp_difference/max": 0.28518197536468504, "sampling/sampling_logp_difference/mean": 0.021753094904124737, "step": 1670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.2, "completions/max_terminated_length": 348.2, "completions/mean_length": 227.7125, "completions/mean_terminated_length": 227.7125, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.1046518936753273, "epoch": 0.15445435322239587, "frac_reward_zero_std": 0.3, "grad_norm": 3.559222459793091, "kl": 0.00803981300268788, "learning_rate": 4.999971639800281e-06, "loss": 0.0005, "num_tokens": 7289120.0, "reward": -0.075, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.3012434720993042, "sampling/importance_sampling_ratio/mean": 1.0003988683223723, "sampling/importance_sampling_ratio/min": 0.8038273274898529, "sampling/sampling_logp_difference/max": 0.2731226861476898, "sampling/sampling_logp_difference/mean": 0.02086831545457244, "step": 1680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/max_terminated_length": 275.3, "completions/mean_length": 210.95, "completions/mean_terminated_length": 210.95, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.102028501033783, "epoch": 0.1553737243725292, "frac_reward_zero_std": 0.5, "grad_norm": 5.665643692016602, "kl": 0.007538376882439479, "learning_rate": 4.9999582878194345e-06, "loss": 0.0005, "num_tokens": 7333532.0, "reward": 0.3, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2648537397384643, "sampling/importance_sampling_ratio/mean": 1.0001300275325775, "sampling/importance_sampling_ratio/min": 0.8053418874740601, "sampling/sampling_logp_difference/max": 0.24303230047225952, "sampling/sampling_logp_difference/mean": 0.02094775643199682, "step": 1690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 199.4125, "completions/mean_terminated_length": 199.4125, "completions/min_length": 117.5, "completions/min_terminated_length": 117.5, "entropy": 1.1668434783816337, "epoch": 0.1562930955226625, "frac_reward_zero_std": 0.6, "grad_norm": 0.15041440725326538, "kl": 0.00868740932783112, "learning_rate": 4.99994236818103e-06, "loss": 0.0007, "num_tokens": 7373805.0, "reward": -0.43125, "reward_std": 0.4252885580062866, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.42528859376907346, "sampling/importance_sampling_ratio/max": 1.3247631311416626, "sampling/importance_sampling_ratio/mean": 1.0001298248767854, "sampling/importance_sampling_ratio/min": 0.7590107977390289, "sampling/sampling_logp_difference/max": 0.3218884408473969, "sampling/sampling_logp_difference/mean": 0.02264628801494837, "step": 1700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 200.7625, "completions/mean_terminated_length": 200.7625, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.083328975737095, "epoch": 0.15721246667279581, "frac_reward_zero_std": 0.5, "grad_norm": 5.828578948974609, "kl": 0.00776079579081852, "learning_rate": 4.999923880901419e-06, "loss": 0.0004, "num_tokens": 7419266.0, "reward": -0.35, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.231118392944336, "sampling/importance_sampling_ratio/mean": 1.000261551141739, "sampling/importance_sampling_ratio/min": 0.7601912915706635, "sampling/sampling_logp_difference/max": 0.2906831383705139, "sampling/sampling_logp_difference/mean": 0.02097844947129488, "step": 1710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.8, "completions/max_terminated_length": 312.8, "completions/mean_length": 208.0375, "completions/mean_terminated_length": 208.0375, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0358453068882227, "epoch": 0.1581318378229291, "frac_reward_zero_std": 0.3, "grad_norm": 0.11827094852924347, "kl": 0.008112310012802482, "learning_rate": 4.999902825999588e-06, "loss": 0.0003, "num_tokens": 7465021.0, "reward": 0.175, "reward_std": 0.6548477113246918, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6548477113246918, "sampling/importance_sampling_ratio/max": 1.3290834426879883, "sampling/importance_sampling_ratio/mean": 1.0000976383686067, "sampling/importance_sampling_ratio/min": 0.7675565600395202, "sampling/sampling_logp_difference/max": 0.3088190913200378, "sampling/sampling_logp_difference/mean": 0.021620805189013482, "step": 1720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 219.525, "completions/mean_terminated_length": 219.525, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0798671215772628, "epoch": 0.15905120897306244, "frac_reward_zero_std": 0.4, "grad_norm": 6.6527605056762695, "kl": 0.017248101232689807, "learning_rate": 4.9998792034971645e-06, "loss": 0.0008, "num_tokens": 7509407.0, "reward": 0.075, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.3607568264007568, "sampling/importance_sampling_ratio/mean": 0.9997735321521759, "sampling/importance_sampling_ratio/min": 0.7006454512476921, "sampling/sampling_logp_difference/max": 0.5129691064357758, "sampling/sampling_logp_difference/mean": 0.020811816211789846, "step": 1730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.4, "completions/max_terminated_length": 320.4, "completions/mean_length": 223.5875, "completions/mean_terminated_length": 223.5875, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.2038073986768723, "epoch": 0.15997058012319573, "frac_reward_zero_std": 0.3, "grad_norm": 7.0669941902160645, "kl": 0.008927359161316418, "learning_rate": 4.999853013418409e-06, "loss": 0.0006, "num_tokens": 7557718.0, "reward": -0.325, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.2579605221748351, "sampling/importance_sampling_ratio/mean": 0.9998698651790618, "sampling/importance_sampling_ratio/min": 0.7821671485900878, "sampling/sampling_logp_difference/max": 0.263225793838501, "sampling/sampling_logp_difference/mean": 0.02273548822849989, "step": 1740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 205.725, "completions/mean_terminated_length": 205.725, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0792374148964883, "epoch": 0.16088995127332903, "frac_reward_zero_std": 0.4, "grad_norm": 0.09510912001132965, "kl": 0.008922025474021211, "learning_rate": 4.99982425579022e-06, "loss": 0.0003, "num_tokens": 7599312.0, "reward": -0.125, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2492369413375854, "sampling/importance_sampling_ratio/mean": 0.9999273002147675, "sampling/importance_sampling_ratio/min": 0.7776693642139435, "sampling/sampling_logp_difference/max": 0.2652453124523163, "sampling/sampling_logp_difference/mean": 0.021386993676424028, "step": 1750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.3, "completions/max_terminated_length": 289.3, "completions/mean_length": 214.05, "completions/mean_terminated_length": 214.05, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.129462357610464, "epoch": 0.16180932242346235, "frac_reward_zero_std": 0.5, "grad_norm": 0.09432012587785721, "kl": 0.007701684607309289, "learning_rate": 4.999792930642135e-06, "loss": 0.0007, "num_tokens": 7642156.0, "reward": -0.075, "reward_std": 0.48400617241859434, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.48400617241859434, "sampling/importance_sampling_ratio/max": 1.274362826347351, "sampling/importance_sampling_ratio/mean": 1.0002899646759034, "sampling/importance_sampling_ratio/min": 0.7959905683994293, "sampling/sampling_logp_difference/max": 0.26303625106811523, "sampling/sampling_logp_difference/mean": 0.021532628685235977, "step": 1760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 328.3, "completions/max_terminated_length": 318.6, "completions/mean_length": 217.4875, "completions/mean_terminated_length": 214.29285888671876, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.0613250091671944, "epoch": 0.16272869357359565, "frac_reward_zero_std": 0.4, "grad_norm": 7.952963829040527, "kl": 0.007132105820346624, "learning_rate": 4.999759038006328e-06, "loss": -0.0001, "num_tokens": 7688443.0, "reward": -0.0125, "reward_std": 0.5238720178604126, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.5238720417022705, "sampling/importance_sampling_ratio/max": 1.2506524801254273, "sampling/importance_sampling_ratio/mean": 1.0002643644809723, "sampling/importance_sampling_ratio/min": 0.7923784255981445, "sampling/sampling_logp_difference/max": 0.25070300698280334, "sampling/sampling_logp_difference/mean": 0.020951641164720057, "step": 1770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.4, "completions/max_terminated_length": 260.4, "completions/mean_length": 195.6625, "completions/mean_terminated_length": 195.6625, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.1096381649374962, "epoch": 0.16364806472372898, "frac_reward_zero_std": 0.5, "grad_norm": 0.1390967220067978, "kl": 0.009223904350074008, "learning_rate": 4.9997225779176075e-06, "loss": 0.0004, "num_tokens": 7732368.0, "reward": 0.025, "reward_std": 0.3535533845424652, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2650764107704162, "sampling/importance_sampling_ratio/mean": 1.000060772895813, "sampling/importance_sampling_ratio/min": 0.7871014595031738, "sampling/sampling_logp_difference/max": 0.2560166895389557, "sampling/sampling_logp_difference/mean": 0.022475911863148214, "step": 1780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.1, "completions/max_terminated_length": 252.1, "completions/mean_length": 185.8, "completions/mean_terminated_length": 185.8, "completions/min_length": 113.6, "completions/min_terminated_length": 113.6, "entropy": 1.0376261431723832, "epoch": 0.16456743587386227, "frac_reward_zero_std": 0.2, "grad_norm": 9.646512031555176, "kl": 0.009650581009918825, "learning_rate": 4.999683550413424e-06, "loss": 0.0001, "num_tokens": 7770736.0, "reward": 0.225, "reward_std": 0.769285362958908, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.769285386800766, "sampling/importance_sampling_ratio/max": 1.2981437206268311, "sampling/importance_sampling_ratio/mean": 1.0002058029174805, "sampling/importance_sampling_ratio/min": 0.7568092167377471, "sampling/sampling_logp_difference/max": 0.31393855810165405, "sampling/sampling_logp_difference/mean": 0.021482637338340282, "step": 1790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 289.0, "completions/max_terminated_length": 270.5, "completions/mean_length": 192.3875, "completions/mean_terminated_length": 186.65833435058593, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0369072146713734, "epoch": 0.1654868070239956, "frac_reward_zero_std": 0.7, "grad_norm": 0.2796126902103424, "kl": 0.010954614775255323, "learning_rate": 4.999641955533858e-06, "loss": 0.0008, "num_tokens": 7815463.0, "reward": -0.375, "reward_std": 0.13887301683425904, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.13887301683425904, "sampling/importance_sampling_ratio/max": 1.2759472727775574, "sampling/importance_sampling_ratio/mean": 1.0002878904342651, "sampling/importance_sampling_ratio/min": 0.7607795417308807, "sampling/sampling_logp_difference/max": 0.29021975994110105, "sampling/sampling_logp_difference/mean": 0.021058506146073342, "step": 1800 }, { "epoch": 0.1654868070239956, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 272.9738219895288, "eval_completions/max_terminated_length": 271.02966841186736, "eval_completions/mean_length": 183.695462478185, "eval_completions/mean_terminated_length": 183.30176401720828, "eval_completions/min_length": 111.00349040139616, "eval_completions/min_terminated_length": 111.00349040139616, "eval_entropy": 1.0662119608586995, "eval_frac_reward_zero_std": 0.47469458987783597, "eval_kl": 0.010483158101777938, "eval_loss": 0.0005819857469759881, "eval_num_tokens": 7815463.0, "eval_reward": -0.1857547993019197, "eval_reward_std": 0.45793868853575687, "eval_rewards/reward_fn/mean": -0.1857547993019197, "eval_rewards/reward_fn/std": 0.45793870403504494, "eval_runtime": 1608.0854, "eval_samples_per_second": 0.356, "eval_sampling/importance_sampling_ratio/max": 1.2706023107232431, "eval_sampling/importance_sampling_ratio/mean": 1.0000473557342409, "eval_sampling/importance_sampling_ratio/min": 0.7638521223064076, "eval_sampling/sampling_logp_difference/max": 0.3035139055048192, "eval_sampling/sampling_logp_difference/mean": 0.021172364029454726, "eval_steps_per_second": 0.045, "step": 1800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.4, "completions/max_terminated_length": 298.2, "completions/mean_length": 211.875, "completions/mean_terminated_length": 207.9982147216797, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.1855968341231347, "epoch": 0.1664061781741289, "frac_reward_zero_std": 0.3, "grad_norm": 7.360311031341553, "kl": 0.010860234769643285, "learning_rate": 4.999597793321635e-06, "loss": 0.0005, "num_tokens": 7861325.0, "reward": -0.43125, "reward_std": 0.595009683072567, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.5950097307562828, "sampling/importance_sampling_ratio/max": 1.337994647026062, "sampling/importance_sampling_ratio/mean": 0.9995708465576172, "sampling/importance_sampling_ratio/min": 0.7491858452558517, "sampling/sampling_logp_difference/max": 0.3574322462081909, "sampling/sampling_logp_difference/mean": 0.021946886368095875, "step": 1810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 212.175, "completions/mean_terminated_length": 212.175, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.068941879272461, "epoch": 0.16732554932426222, "frac_reward_zero_std": 0.5, "grad_norm": 6.8922438621521, "kl": 0.007878050176077522, "learning_rate": 4.9995510638221086e-06, "loss": 0.0006, "num_tokens": 7907099.0, "reward": -0.575, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.254954719543457, "sampling/importance_sampling_ratio/mean": 1.0000268459320067, "sampling/importance_sampling_ratio/min": 0.7655709505081176, "sampling/sampling_logp_difference/max": 0.2868260383605957, "sampling/sampling_logp_difference/mean": 0.021543628163635732, "step": 1820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.4, "completions/max_terminated_length": 311.4, "completions/mean_length": 231.8625, "completions/mean_terminated_length": 231.8625, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.1080614011734724, "epoch": 0.16824492047439552, "frac_reward_zero_std": 0.3, "grad_norm": 6.639761924743652, "kl": 0.008735984200029634, "learning_rate": 4.999501767083278e-06, "loss": -0.0001, "num_tokens": 7953664.0, "reward": -0.25, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.275916051864624, "sampling/importance_sampling_ratio/mean": 0.9999524414539337, "sampling/importance_sampling_ratio/min": 0.7769168615341187, "sampling/sampling_logp_difference/max": 0.2853513896465302, "sampling/sampling_logp_difference/mean": 0.021585115510970355, "step": 1830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.2, "completions/max_terminated_length": 321.2, "completions/mean_length": 229.75, "completions/mean_terminated_length": 229.75, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.1224500104784965, "epoch": 0.1691642916245288, "frac_reward_zero_std": 0.3, "grad_norm": 4.249851226806641, "kl": 0.008717592194443569, "learning_rate": 4.999449903155772e-06, "loss": 0.0003, "num_tokens": 7998100.0, "reward": -0.65, "reward_std": 0.6186538934707642, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.6186538934707642, "sampling/importance_sampling_ratio/max": 1.3173491835594178, "sampling/importance_sampling_ratio/mean": 0.9997834503650666, "sampling/importance_sampling_ratio/min": 0.7325367331504822, "sampling/sampling_logp_difference/max": 0.3386121392250061, "sampling/sampling_logp_difference/mean": 0.022351453453302382, "step": 1840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.2, "completions/max_terminated_length": 271.2, "completions/mean_length": 194.4375, "completions/mean_terminated_length": 194.4375, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.0774458341300488, "epoch": 0.17008366277466214, "frac_reward_zero_std": 0.6, "grad_norm": 0.13067704439163208, "kl": 0.010113066429039464, "learning_rate": 4.999395472092861e-06, "loss": 0.0007, "num_tokens": 8039639.0, "reward": -0.725, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.725, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.362171471118927, "sampling/importance_sampling_ratio/mean": 1.000049102306366, "sampling/importance_sampling_ratio/min": 0.7438823401927948, "sampling/sampling_logp_difference/max": 0.3882533311843872, "sampling/sampling_logp_difference/mean": 0.02169228680431843, "step": 1850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 194.475, "completions/mean_terminated_length": 194.475, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.015919890999794, "epoch": 0.17100303392479543, "frac_reward_zero_std": 0.6, "grad_norm": 6.464287757873535, "kl": 0.008342243506922387, "learning_rate": 4.999338473950448e-06, "loss": 0.0003, "num_tokens": 8077621.0, "reward": -0.325, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2455684304237367, "sampling/importance_sampling_ratio/mean": 0.9997163593769074, "sampling/importance_sampling_ratio/min": 0.7812203526496887, "sampling/sampling_logp_difference/max": 0.2642441838979721, "sampling/sampling_logp_difference/mean": 0.020471646450459957, "step": 1860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.0, "completions/max_terminated_length": 323.0, "completions/mean_length": 232.275, "completions/mean_terminated_length": 228.98928680419922, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1037378549575805, "epoch": 0.17192240507492876, "frac_reward_zero_std": 0.3, "grad_norm": 4.6465654373168945, "kl": 0.008008757716743276, "learning_rate": 4.9992789087870764e-06, "loss": 0.0008, "num_tokens": 8122339.0, "reward": 0.06875, "reward_std": 0.6873395919799805, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.6873396158218383, "sampling/importance_sampling_ratio/max": 1.2757189273834229, "sampling/importance_sampling_ratio/mean": 0.999925845861435, "sampling/importance_sampling_ratio/min": 0.7916803956031799, "sampling/sampling_logp_difference/max": 0.26590805053710936, "sampling/sampling_logp_difference/mean": 0.021784370206296443, "step": 1870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.5, "completions/max_terminated_length": 275.5, "completions/mean_length": 194.8875, "completions/mean_terminated_length": 194.8875, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0662693813443185, "epoch": 0.17284177622506205, "frac_reward_zero_std": 0.2, "grad_norm": 0.11798768490552902, "kl": 0.008588754804804921, "learning_rate": 4.999216776663924e-06, "loss": 0.0002, "num_tokens": 8160066.0, "reward": 0.475, "reward_std": 0.6784210562705993, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.6784210681915284, "sampling/importance_sampling_ratio/max": 1.2470642566680907, "sampling/importance_sampling_ratio/mean": 1.0001885712146759, "sampling/importance_sampling_ratio/min": 0.7667438149452209, "sampling/sampling_logp_difference/max": 0.28001530170440675, "sampling/sampling_logp_difference/mean": 0.021667362935841084, "step": 1880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.9, "completions/max_terminated_length": 317.3, "completions/mean_length": 228.375, "completions/mean_terminated_length": 225.73035888671876, "completions/min_length": 155.1, "completions/min_terminated_length": 155.1, "entropy": 1.1105477765202523, "epoch": 0.17376114737519538, "frac_reward_zero_std": 0.1, "grad_norm": 5.358621597290039, "kl": 0.009375753049971536, "learning_rate": 4.999152077644805e-06, "loss": 0.0004, "num_tokens": 8204912.0, "reward": -0.26875, "reward_std": 0.7296625792980194, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.7296625971794128, "sampling/importance_sampling_ratio/max": 1.2792662382125854, "sampling/importance_sampling_ratio/mean": 0.9996852576732635, "sampling/importance_sampling_ratio/min": 0.7683384239673614, "sampling/sampling_logp_difference/max": 0.2858198881149292, "sampling/sampling_logp_difference/mean": 0.021452724188566207, "step": 1890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.7, "completions/max_terminated_length": 285.7, "completions/mean_length": 216.925, "completions/mean_terminated_length": 216.925, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.1943703643977641, "epoch": 0.17468051852532868, "frac_reward_zero_std": 0.6, "grad_norm": 0.20943057537078857, "kl": 0.009251559985568748, "learning_rate": 4.9990848117961705e-06, "loss": 0.0002, "num_tokens": 8248234.0, "reward": -0.125, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.3323838233947753, "sampling/importance_sampling_ratio/mean": 1.0000596165657043, "sampling/importance_sampling_ratio/min": 0.7852665305137634, "sampling/sampling_logp_difference/max": 0.3081549942493439, "sampling/sampling_logp_difference/mean": 0.022943649627268314, "step": 1900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 193.0375, "completions/mean_terminated_length": 193.0375, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 1.0933258004486561, "epoch": 0.17559988967546197, "frac_reward_zero_std": 0.5, "grad_norm": 6.00364875793457, "kl": 0.00794901025656145, "learning_rate": 4.999014979187108e-06, "loss": 0.0002, "num_tokens": 8288477.0, "reward": 0.4, "reward_std": 0.49493399262428284, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.4949340045452118, "sampling/importance_sampling_ratio/max": 1.2580102205276489, "sampling/importance_sampling_ratio/mean": 0.9996150851249694, "sampling/importance_sampling_ratio/min": 0.7558135360479354, "sampling/sampling_logp_difference/max": 0.3105516850948334, "sampling/sampling_logp_difference/mean": 0.021253474429249762, "step": 1910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.3, "completions/max_terminated_length": 298.8, "completions/mean_length": 215.125, "completions/mean_terminated_length": 211.6750015258789, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0249982446432113, "epoch": 0.1765192608255953, "frac_reward_zero_std": 0.4, "grad_norm": 0.4242212176322937, "kl": 0.00941032420960255, "learning_rate": 4.9989425798893415e-06, "loss": 0.0006, "num_tokens": 8329015.0, "reward": -0.63125, "reward_std": 0.5751873195171356, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.575187349319458, "sampling/importance_sampling_ratio/max": 1.2843928456306457, "sampling/importance_sampling_ratio/mean": 0.9992348253726959, "sampling/importance_sampling_ratio/min": 0.7338935405015945, "sampling/sampling_logp_difference/max": 0.3411496192216873, "sampling/sampling_logp_difference/mean": 0.02091215066611767, "step": 1920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.8, "completions/max_terminated_length": 302.8, "completions/mean_length": 209.35, "completions/mean_terminated_length": 209.35, "completions/min_length": 146.8, "completions/min_terminated_length": 146.8, "entropy": 1.1095786310732365, "epoch": 0.1774386319757286, "frac_reward_zero_std": 0.4, "grad_norm": 6.8876214027404785, "kl": 0.009385722974548117, "learning_rate": 4.998867613977231e-06, "loss": 0.0005, "num_tokens": 8375739.0, "reward": -0.1, "reward_std": 0.5664041936397552, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5664042294025421, "sampling/importance_sampling_ratio/max": 1.2749662280082703, "sampling/importance_sampling_ratio/mean": 0.9996235370635986, "sampling/importance_sampling_ratio/min": 0.8019274890422821, "sampling/sampling_logp_difference/max": 0.2559288263320923, "sampling/sampling_logp_difference/mean": 0.021461096778512, "step": 1930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.7, "completions/max_terminated_length": 320.7, "completions/mean_length": 209.4625, "completions/mean_terminated_length": 209.4625, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.0853445515036584, "epoch": 0.17835800312586192, "frac_reward_zero_std": 0.4, "grad_norm": 0.1024390310049057, "kl": 0.008839805651223286, "learning_rate": 4.998790081527771e-06, "loss": 0.0003, "num_tokens": 8417744.0, "reward": -0.15, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.286285436153412, "sampling/importance_sampling_ratio/mean": 1.0002357482910156, "sampling/importance_sampling_ratio/min": 0.7510183811187744, "sampling/sampling_logp_difference/max": 0.33968801498413087, "sampling/sampling_logp_difference/mean": 0.021789978072047234, "step": 1940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 341.6, "completions/max_terminated_length": 341.6, "completions/mean_length": 230.525, "completions/mean_terminated_length": 230.525, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.0966296821832657, "epoch": 0.17927737427599522, "frac_reward_zero_std": 0.5, "grad_norm": 0.10174008458852768, "kl": 0.008760085818357766, "learning_rate": 4.9987099826205966e-06, "loss": 0.0005, "num_tokens": 8463978.0, "reward": -0.375, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.235067069530487, "sampling/importance_sampling_ratio/mean": 1.000037181377411, "sampling/importance_sampling_ratio/min": 0.7855679810047149, "sampling/sampling_logp_difference/max": 0.25130143761634827, "sampling/sampling_logp_difference/mean": 0.021080931462347508, "step": 1950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.8, "completions/max_terminated_length": 254.8, "completions/mean_length": 190.3375, "completions/mean_terminated_length": 190.3375, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.1106417536735536, "epoch": 0.18019674542612854, "frac_reward_zero_std": 0.4, "grad_norm": 0.1281871348619461, "kl": 0.008916488726390526, "learning_rate": 4.998627317337973e-06, "loss": 0.0004, "num_tokens": 8502781.0, "reward": 0.05, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.3126219749450683, "sampling/importance_sampling_ratio/mean": 0.9997373521327972, "sampling/importance_sampling_ratio/min": 0.7666497230529785, "sampling/sampling_logp_difference/max": 0.28213725090026853, "sampling/sampling_logp_difference/mean": 0.0221559202298522, "step": 1960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 216.0875, "completions/mean_terminated_length": 216.0875, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.1194576874375344, "epoch": 0.18111611657626184, "frac_reward_zero_std": 0.4, "grad_norm": 0.12405950576066971, "kl": 0.009292167512467132, "learning_rate": 4.9985420857648044e-06, "loss": 0.0004, "num_tokens": 8552004.0, "reward": -0.03125, "reward_std": 0.5762809097766877, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5762809216976166, "sampling/importance_sampling_ratio/max": 1.3339093804359436, "sampling/importance_sampling_ratio/mean": 0.9996419489383698, "sampling/importance_sampling_ratio/min": 0.7308150798082351, "sampling/sampling_logp_difference/max": 0.3759077787399292, "sampling/sampling_logp_difference/mean": 0.021584430150687693, "step": 1970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 178.8, "completions/mean_terminated_length": 178.8, "completions/min_length": 112.6, "completions/min_terminated_length": 112.6, "entropy": 1.0099565394222736, "epoch": 0.18203548772639513, "frac_reward_zero_std": 0.5, "grad_norm": 0.15651923418045044, "kl": 0.011201527283992619, "learning_rate": 4.998454287988632e-06, "loss": 0.0005, "num_tokens": 8590852.0, "reward": -0.225, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.261948335170746, "sampling/importance_sampling_ratio/mean": 0.9996395230293273, "sampling/importance_sampling_ratio/min": 0.7338244318962097, "sampling/sampling_logp_difference/max": 0.3314939260482788, "sampling/sampling_logp_difference/mean": 0.021469095163047312, "step": 1980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.3, "completions/max_terminated_length": 268.3, "completions/mean_length": 178.4, "completions/mean_terminated_length": 178.4, "completions/min_length": 97.7, "completions/min_terminated_length": 97.7, "entropy": 0.9872049666941166, "epoch": 0.18295485887652846, "frac_reward_zero_std": 0.6, "grad_norm": 0.15274612605571747, "kl": 0.010602542938431724, "learning_rate": 4.998363924099629e-06, "loss": 0.0005, "num_tokens": 8632748.0, "reward": 0.025, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.30131676197052, "sampling/importance_sampling_ratio/mean": 0.9999166250228881, "sampling/importance_sampling_ratio/min": 0.7901581645011901, "sampling/sampling_logp_difference/max": 0.2878017783164978, "sampling/sampling_logp_difference/mean": 0.02156129963696003, "step": 1990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.9, "completions/max_terminated_length": 258.9, "completions/mean_length": 171.8375, "completions/mean_terminated_length": 171.8375, "completions/min_length": 113.2, "completions/min_terminated_length": 113.2, "entropy": 0.9613868180662394, "epoch": 0.18387423002666176, "frac_reward_zero_std": 0.2, "grad_norm": 5.039412021636963, "kl": 0.009318033905583435, "learning_rate": 4.998270994190609e-06, "loss": -0.0, "num_tokens": 8673887.0, "reward": 0.1, "reward_std": 0.6750263929367065, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6750264167785645, "sampling/importance_sampling_ratio/max": 1.2867394924163817, "sampling/importance_sampling_ratio/mean": 1.0002319514751434, "sampling/importance_sampling_ratio/min": 0.7547361731529236, "sampling/sampling_logp_difference/max": 0.31070716083049776, "sampling/sampling_logp_difference/mean": 0.020357457920908928, "step": 2000 }, { "epoch": 0.18387423002666176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 289.3891797556719, "eval_completions/max_terminated_length": 288.3123909249564, "eval_completions/mean_length": 201.33093368237348, "eval_completions/mean_terminated_length": 201.0068458170999, "eval_completions/min_length": 131.40837696335078, "eval_completions/min_terminated_length": 131.40837696335078, "eval_entropy": 1.0621050550987583, "eval_frac_reward_zero_std": 0.4607329842931937, "eval_kl": 0.00949068724447585, "eval_loss": 0.00043526157969608903, "eval_num_tokens": 8673887.0, "eval_reward": -0.11671029668411867, "eval_reward_std": 0.47905677765035715, "eval_rewards/reward_fn/mean": -0.11671029668411867, "eval_rewards/reward_fn/std": 0.4790567925255128, "eval_runtime": 1690.8134, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2726942540047264, "eval_sampling/importance_sampling_ratio/mean": 0.9999344130461129, "eval_sampling/importance_sampling_ratio/min": 0.7725813164656907, "eval_sampling/sampling_logp_difference/max": 0.2968630015225935, "eval_sampling/sampling_logp_difference/mean": 0.021140853837002427, "eval_steps_per_second": 0.043, "step": 2000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.2, "completions/max_terminated_length": 317.2, "completions/mean_length": 204.9875, "completions/mean_terminated_length": 204.9875, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 1.052908793836832, "epoch": 0.18479360117679508, "frac_reward_zero_std": 0.2, "grad_norm": 0.07956591993570328, "kl": 0.009983776323497295, "learning_rate": 4.9981754983570165e-06, "loss": 0.0002, "num_tokens": 8715774.0, "reward": -0.44375, "reward_std": 0.6589846178889275, "rewards/reward_fn/mean": -0.44375, "rewards/reward_fn/std": 0.6589846417307854, "sampling/importance_sampling_ratio/max": 1.2626225471496582, "sampling/importance_sampling_ratio/mean": 0.9996070027351379, "sampling/importance_sampling_ratio/min": 0.7615952372550965, "sampling/sampling_logp_difference/max": 0.28874300718307494, "sampling/sampling_logp_difference/mean": 0.021093208529055118, "step": 2010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.4, "completions/max_terminated_length": 310.4, "completions/mean_length": 222.5625, "completions/mean_terminated_length": 222.5625, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.1314774371683598, "epoch": 0.18571297232692838, "frac_reward_zero_std": 0.3, "grad_norm": 0.10946657508611679, "kl": 0.009395581035641953, "learning_rate": 4.998077436696934e-06, "loss": 0.0004, "num_tokens": 8760707.0, "reward": 0.0, "reward_std": 0.6876311719417572, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6876312077045441, "sampling/importance_sampling_ratio/max": 1.3036259174346925, "sampling/importance_sampling_ratio/mean": 0.999922925233841, "sampling/importance_sampling_ratio/min": 0.7925902545452118, "sampling/sampling_logp_difference/max": 0.27834399938583376, "sampling/sampling_logp_difference/mean": 0.02141685876995325, "step": 2020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.1, "completions/max_terminated_length": 275.1, "completions/mean_length": 204.1125, "completions/mean_terminated_length": 204.1125, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.1219103820621967, "epoch": 0.1866323434770617, "frac_reward_zero_std": 0.3, "grad_norm": 4.161088466644287, "kl": 0.01039814909454435, "learning_rate": 4.997976809311078e-06, "loss": 0.0004, "num_tokens": 8801876.0, "reward": 0.19375, "reward_std": 0.6450958847999573, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.6450959146022797, "sampling/importance_sampling_ratio/max": 1.2597630262374877, "sampling/importance_sampling_ratio/mean": 0.9999996244907379, "sampling/importance_sampling_ratio/min": 0.7537624537944794, "sampling/sampling_logp_difference/max": 0.3382760763168335, "sampling/sampling_logp_difference/mean": 0.02115944568067789, "step": 2030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.0, "completions/max_terminated_length": 313.0, "completions/mean_length": 205.325, "completions/mean_terminated_length": 205.325, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.0763554230332375, "epoch": 0.187551714627195, "frac_reward_zero_std": 0.6, "grad_norm": 0.10967079550027847, "kl": 0.00953908590017818, "learning_rate": 4.997873616302802e-06, "loss": 0.0007, "num_tokens": 8845294.0, "reward": -0.075, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.25970801115036, "sampling/importance_sampling_ratio/mean": 1.0001145958900453, "sampling/importance_sampling_ratio/min": 0.772824478149414, "sampling/sampling_logp_difference/max": 0.27637338638305664, "sampling/sampling_logp_difference/mean": 0.021243065781891345, "step": 2040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 366.4, "completions/max_terminated_length": 334.2, "completions/mean_length": 250.85, "completions/mean_terminated_length": 236.99642944335938, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.0832148134708404, "epoch": 0.1884710857773283, "frac_reward_zero_std": 0.3, "grad_norm": 0.12180796265602112, "kl": 0.010077079801703803, "learning_rate": 4.997767857778093e-06, "loss": 0.0003, "num_tokens": 8894138.0, "reward": -0.08125, "reward_std": 0.5979054629802704, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5979054689407348, "sampling/importance_sampling_ratio/max": 1.2449756741523743, "sampling/importance_sampling_ratio/mean": 0.9995810627937317, "sampling/importance_sampling_ratio/min": 0.7724187672138214, "sampling/sampling_logp_difference/max": 0.26953349113464353, "sampling/sampling_logp_difference/mean": 0.020882124453783034, "step": 2050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.0, "completions/max_terminated_length": 305.9, "completions/mean_length": 231.7125, "completions/mean_terminated_length": 229.47321472167968, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.1501486636698246, "epoch": 0.18939045692746162, "frac_reward_zero_std": 0.4, "grad_norm": 0.1129380315542221, "kl": 0.013266102253692225, "learning_rate": 4.997659533845574e-06, "loss": 0.0007, "num_tokens": 8940907.0, "reward": -0.2125, "reward_std": 0.5091521501541137, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.5091521620750428, "sampling/importance_sampling_ratio/max": 1.3718178153038025, "sampling/importance_sampling_ratio/mean": 1.0005676805973054, "sampling/importance_sampling_ratio/min": 0.7703902542591095, "sampling/sampling_logp_difference/max": 0.3407881557941437, "sampling/sampling_logp_difference/mean": 0.02117590755224228, "step": 2060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 212.4375, "completions/mean_terminated_length": 212.4375, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0545032516121864, "epoch": 0.19030982807759492, "frac_reward_zero_std": 0.6, "grad_norm": 0.1044619008898735, "kl": 0.014077803451800718, "learning_rate": 4.997548644616501e-06, "loss": 0.0007, "num_tokens": 8983774.0, "reward": -0.43125, "reward_std": 0.3063515231013298, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.30635153502225876, "sampling/importance_sampling_ratio/max": 1.280693483352661, "sampling/importance_sampling_ratio/mean": 1.0002897560596467, "sampling/importance_sampling_ratio/min": 0.7362377345561981, "sampling/sampling_logp_difference/max": 0.3151408314704895, "sampling/sampling_logp_difference/mean": 0.021131117083132266, "step": 2070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 362.9, "completions/max_terminated_length": 343.6, "completions/mean_length": 239.425, "completions/mean_terminated_length": 235.88392944335936, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1346502289175988, "epoch": 0.19122919922772824, "frac_reward_zero_std": 0.2, "grad_norm": 5.414846897125244, "kl": 0.012112543318653479, "learning_rate": 4.997435190204767e-06, "loss": 0.0004, "num_tokens": 9031056.0, "reward": -0.25625, "reward_std": 0.7671249747276306, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.767125004529953, "sampling/importance_sampling_ratio/max": 1.3362827539443969, "sampling/importance_sampling_ratio/mean": 1.0002984821796417, "sampling/importance_sampling_ratio/min": 0.7704026579856873, "sampling/sampling_logp_difference/max": 0.3244939923286438, "sampling/sampling_logp_difference/mean": 0.022117613069713115, "step": 2080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.4, "completions/max_terminated_length": 286.4, "completions/mean_length": 208.0875, "completions/mean_terminated_length": 208.0875, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.068577367812395, "epoch": 0.19214857037786154, "frac_reward_zero_std": 0.2, "grad_norm": 5.641380310058594, "kl": 0.01127834916114807, "learning_rate": 4.9973191707268986e-06, "loss": 0.0004, "num_tokens": 9077823.0, "reward": 0.4, "reward_std": 0.7406247019767761, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.740624749660492, "sampling/importance_sampling_ratio/max": 1.2676469087600708, "sampling/importance_sampling_ratio/mean": 0.999646270275116, "sampling/importance_sampling_ratio/min": 0.7905237674713135, "sampling/sampling_logp_difference/max": 0.2555447816848755, "sampling/sampling_logp_difference/mean": 0.021050736121833325, "step": 2090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 212.375, "completions/mean_terminated_length": 212.375, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.1040468320250512, "epoch": 0.19306794152799486, "frac_reward_zero_std": 0.6, "grad_norm": 0.0959465503692627, "kl": 0.012862538726767524, "learning_rate": 4.9972005863020574e-06, "loss": 0.0006, "num_tokens": 9124149.0, "reward": -0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2464789867401123, "sampling/importance_sampling_ratio/mean": 1.0003184258937836, "sampling/importance_sampling_ratio/min": 0.7806608080863953, "sampling/sampling_logp_difference/max": 0.25816773176193236, "sampling/sampling_logp_difference/mean": 0.021766837500035763, "step": 2100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.7, "completions/max_terminated_length": 320.7, "completions/mean_length": 226.5375, "completions/mean_terminated_length": 226.5375, "completions/min_length": 157.1, "completions/min_terminated_length": 157.1, "entropy": 1.105923917889595, "epoch": 0.19398731267812816, "frac_reward_zero_std": 0.5, "grad_norm": 4.980881690979004, "kl": 0.01485194059787318, "learning_rate": 4.997079437052039e-06, "loss": 0.0004, "num_tokens": 9169416.0, "reward": -0.35, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.344807004928589, "sampling/importance_sampling_ratio/mean": 0.999737024307251, "sampling/importance_sampling_ratio/min": 0.7202730610966682, "sampling/sampling_logp_difference/max": 0.5083509892225265, "sampling/sampling_logp_difference/mean": 0.02176111061125994, "step": 2110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.6, "completions/max_terminated_length": 279.6, "completions/mean_length": 202.025, "completions/mean_terminated_length": 202.025, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.1049255192279817, "epoch": 0.19490668382826146, "frac_reward_zero_std": 0.3, "grad_norm": 5.9103875160217285, "kl": 0.013782999315299093, "learning_rate": 4.996955723101273e-06, "loss": 0.0006, "num_tokens": 9214442.0, "reward": -0.55625, "reward_std": 0.5799276754260063, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.5799276992678642, "sampling/importance_sampling_ratio/max": 1.3616218090057373, "sampling/importance_sampling_ratio/mean": 0.9997422397136688, "sampling/importance_sampling_ratio/min": 0.7852019667625427, "sampling/sampling_logp_difference/max": 0.3278943657875061, "sampling/sampling_logp_difference/mean": 0.021817888505756854, "step": 2120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.3, "completions/max_terminated_length": 293.0, "completions/mean_length": 210.0875, "completions/mean_terminated_length": 206.5375015258789, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0501337140798568, "epoch": 0.19582605497839478, "frac_reward_zero_std": 0.6, "grad_norm": 6.406388282775879, "kl": 0.009661100339144468, "learning_rate": 4.996829444576825e-06, "loss": 0.0001, "num_tokens": 9261217.0, "reward": -0.50625, "reward_std": 0.2844801887869835, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.28448020070791247, "sampling/importance_sampling_ratio/max": 1.246323835849762, "sampling/importance_sampling_ratio/mean": 0.9996880650520324, "sampling/importance_sampling_ratio/min": 0.7717740118503571, "sampling/sampling_logp_difference/max": 0.2800556182861328, "sampling/sampling_logp_difference/mean": 0.021689028665423393, "step": 2130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 206.45, "completions/mean_terminated_length": 206.45, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.0111594945192337, "epoch": 0.19674542612852808, "frac_reward_zero_std": 0.4, "grad_norm": 0.0989614874124527, "kl": 0.010089837774285116, "learning_rate": 4.996700601608391e-06, "loss": 0.0, "num_tokens": 9305581.0, "reward": -0.15, "reward_std": 0.49665169715881347, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.49665169715881347, "sampling/importance_sampling_ratio/max": 1.2875762224197387, "sampling/importance_sampling_ratio/mean": 0.9994701027870179, "sampling/importance_sampling_ratio/min": 0.6685572385787963, "sampling/sampling_logp_difference/max": 0.48670253753662107, "sampling/sampling_logp_difference/mean": 0.02052514683455229, "step": 2140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 204.5125, "completions/mean_terminated_length": 204.5125, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.1336037639528513, "epoch": 0.1976647972786614, "frac_reward_zero_std": 0.2, "grad_norm": 6.411035537719727, "kl": 0.010150204499950632, "learning_rate": 4.9965691943283045e-06, "loss": 0.0002, "num_tokens": 9347270.0, "reward": -0.40625, "reward_std": 0.7081469595432281, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.7081469714641571, "sampling/importance_sampling_ratio/max": 1.393151330947876, "sampling/importance_sampling_ratio/mean": 1.0001453757286072, "sampling/importance_sampling_ratio/min": 0.7859051167964936, "sampling/sampling_logp_difference/max": 0.3693438768386841, "sampling/sampling_logp_difference/mean": 0.021855442970991134, "step": 2150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.9, "completions/max_terminated_length": 308.3, "completions/mean_length": 233.1875, "completions/mean_terminated_length": 230.3607147216797, "completions/min_length": 152.4, "completions/min_terminated_length": 152.4, "entropy": 1.1562392208725214, "epoch": 0.1985841684287947, "frac_reward_zero_std": 0.5, "grad_norm": 5.743379592895508, "kl": 0.011502511578146368, "learning_rate": 4.996435222871531e-06, "loss": 0.0005, "num_tokens": 9395357.0, "reward": 0.21875, "reward_std": 0.39893353432416917, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.39893354624509814, "sampling/importance_sampling_ratio/max": 1.2847472071647643, "sampling/importance_sampling_ratio/mean": 1.000288712978363, "sampling/importance_sampling_ratio/min": 0.7956617772579193, "sampling/sampling_logp_difference/max": 0.26701268553733826, "sampling/sampling_logp_difference/mean": 0.02263814155012369, "step": 2160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 208.9625, "completions/mean_terminated_length": 208.9625, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 1.1117930106818676, "epoch": 0.19950353957892802, "frac_reward_zero_std": 0.6, "grad_norm": 0.0819622352719307, "kl": 0.009076377624296583, "learning_rate": 4.996298687375669e-06, "loss": 0.0007, "num_tokens": 9436266.0, "reward": -0.10625, "reward_std": 0.2844801887869835, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.28448020070791247, "sampling/importance_sampling_ratio/max": 1.27475163936615, "sampling/importance_sampling_ratio/mean": 0.999764496088028, "sampling/importance_sampling_ratio/min": 0.7945587038993835, "sampling/sampling_logp_difference/max": 0.259120774269104, "sampling/sampling_logp_difference/mean": 0.02176323588937521, "step": 2170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.4, "completions/max_terminated_length": 261.4, "completions/mean_length": 189.375, "completions/mean_terminated_length": 189.375, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.1052240416407586, "epoch": 0.20042291072906132, "frac_reward_zero_std": 0.5, "grad_norm": 6.45853853225708, "kl": 0.010670766263501719, "learning_rate": 4.996159587980953e-06, "loss": 0.0006, "num_tokens": 9474344.0, "reward": -0.15, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2813013195991516, "sampling/importance_sampling_ratio/mean": 0.9998811841011047, "sampling/importance_sampling_ratio/min": 0.8031397759914398, "sampling/sampling_logp_difference/max": 0.2494514226913452, "sampling/sampling_logp_difference/mean": 0.021995647437870504, "step": 2180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.0, "completions/max_terminated_length": 316.0, "completions/mean_length": 223.4375, "completions/mean_terminated_length": 223.4375, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.1404649838805199, "epoch": 0.20134228187919462, "frac_reward_zero_std": 0.4, "grad_norm": 0.3386058509349823, "kl": 0.010448697314132005, "learning_rate": 4.9960179248302485e-06, "loss": 0.0006, "num_tokens": 9519227.0, "reward": -0.375, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.344618809223175, "sampling/importance_sampling_ratio/mean": 0.9999821424484253, "sampling/importance_sampling_ratio/min": 0.7918500304222107, "sampling/sampling_logp_difference/max": 0.3408365368843079, "sampling/sampling_logp_difference/mean": 0.021739713475108148, "step": 2190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.7, "completions/max_terminated_length": 273.7, "completions/mean_length": 190.1, "completions/mean_terminated_length": 190.1, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 0.8923455711454154, "epoch": 0.20226165302932794, "frac_reward_zero_std": 0.2, "grad_norm": 5.345462799072266, "kl": 0.00922615857416531, "learning_rate": 4.995873698069054e-06, "loss": 0.0003, "num_tokens": 9558043.0, "reward": 0.425, "reward_std": 0.6750420868396759, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.6750420868396759, "sampling/importance_sampling_ratio/max": 1.2807377338409425, "sampling/importance_sampling_ratio/mean": 0.999938017129898, "sampling/importance_sampling_ratio/min": 0.7530856490135193, "sampling/sampling_logp_difference/max": 0.33180355429649355, "sampling/sampling_logp_difference/mean": 0.018686248734593392, "step": 2200 }, { "epoch": 0.20226165302932794, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 297.13438045375216, "eval_completions/max_terminated_length": 294.66841186736474, "eval_completions/mean_length": 206.26548865619546, "eval_completions/mean_terminated_length": 205.75037422646196, "eval_completions/min_length": 129.82547993019196, "eval_completions/min_terminated_length": 129.82547993019196, "eval_entropy": 1.0895941741179422, "eval_frac_reward_zero_std": 0.42757417102966844, "eval_kl": 0.0098922226454471, "eval_loss": 0.0004769227234646678, "eval_num_tokens": 9558043.0, "eval_reward": -0.1243455497382199, "eval_reward_std": 0.4937476048190764, "eval_rewards/reward_fn/mean": -0.1243455497382199, "eval_rewards/reward_fn/std": 0.49374761761379077, "eval_runtime": 1729.5306, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2704259897817909, "eval_sampling/importance_sampling_ratio/mean": 1.0001026732135194, "eval_sampling/importance_sampling_ratio/min": 0.7717691784233739, "eval_sampling/sampling_logp_difference/max": 0.28961696295005906, "eval_sampling/sampling_logp_difference/mean": 0.020751071099133392, "eval_steps_per_second": 0.042, "step": 2200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.7, "completions/max_terminated_length": 301.3, "completions/mean_length": 212.2875, "completions/mean_terminated_length": 208.19285736083984, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 1.082700741291046, "epoch": 0.20318102417946124, "frac_reward_zero_std": 0.4, "grad_norm": 6.397327423095703, "kl": 0.009991930046817287, "learning_rate": 4.995726907845505e-06, "loss": 0.0009, "num_tokens": 9601442.0, "reward": -0.38125, "reward_std": 0.4835209012031555, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.483520919084549, "sampling/importance_sampling_ratio/max": 1.2781819224357605, "sampling/importance_sampling_ratio/mean": 1.000171458721161, "sampling/importance_sampling_ratio/min": 0.7931345343589783, "sampling/sampling_logp_difference/max": 0.2554804563522339, "sampling/sampling_logp_difference/mean": 0.021068674325942994, "step": 2210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.1, "completions/max_terminated_length": 306.1, "completions/mean_length": 218.0, "completions/mean_terminated_length": 218.0, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.1212183482944966, "epoch": 0.20410039532959456, "frac_reward_zero_std": 0.5, "grad_norm": 0.11511164903640747, "kl": 0.0098762410110794, "learning_rate": 4.995577554310363e-06, "loss": 0.0008, "num_tokens": 9645250.0, "reward": -0.45, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2596561074256898, "sampling/importance_sampling_ratio/mean": 1.000476711988449, "sampling/importance_sampling_ratio/min": 0.7693795919418335, "sampling/sampling_logp_difference/max": 0.2891555309295654, "sampling/sampling_logp_difference/mean": 0.021183535642921924, "step": 2220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 330.5, "completions/max_terminated_length": 320.2, "completions/mean_length": 226.1125, "completions/mean_terminated_length": 221.18333435058594, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0108953446149826, "epoch": 0.20501976647972786, "frac_reward_zero_std": 0.4, "grad_norm": 0.10604795813560486, "kl": 0.010174993035616354, "learning_rate": 4.995425637617027e-06, "loss": 0.0004, "num_tokens": 9688299.0, "reward": -0.0125, "reward_std": 0.46416853070259095, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.46416855454444883, "sampling/importance_sampling_ratio/max": 1.2615553975105285, "sampling/importance_sampling_ratio/mean": 1.0004300773143768, "sampling/importance_sampling_ratio/min": 0.7889519810676575, "sampling/sampling_logp_difference/max": 0.26501508951187136, "sampling/sampling_logp_difference/mean": 0.02010820796713233, "step": 2230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.2, "completions/max_terminated_length": 300.2, "completions/mean_length": 205.35, "completions/mean_terminated_length": 205.35, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.1300479792058469, "epoch": 0.20593913762986119, "frac_reward_zero_std": 0.2, "grad_norm": 6.740416526794434, "kl": 0.011183674086350948, "learning_rate": 4.995271157921529e-06, "loss": 0.0002, "num_tokens": 9729431.0, "reward": 0.09375, "reward_std": 0.7056808531284332, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.7056808710098267, "sampling/importance_sampling_ratio/max": 1.231994354724884, "sampling/importance_sampling_ratio/mean": 0.9993794322013855, "sampling/importance_sampling_ratio/min": 0.765828150510788, "sampling/sampling_logp_difference/max": 0.2802165746688843, "sampling/sampling_logp_difference/mean": 0.02215910069644451, "step": 2240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.1, "completions/max_terminated_length": 318.5, "completions/mean_length": 219.4, "completions/mean_terminated_length": 216.51607360839844, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.022788287885487, "epoch": 0.20685850877999448, "frac_reward_zero_std": 0.3, "grad_norm": 6.603946685791016, "kl": 0.010940103541361167, "learning_rate": 4.995114115382532e-06, "loss": 0.001, "num_tokens": 9773903.0, "reward": -0.50625, "reward_std": 0.5840818628668785, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.5840818986296654, "sampling/importance_sampling_ratio/max": 1.28565856218338, "sampling/importance_sampling_ratio/mean": 0.9997080743312836, "sampling/importance_sampling_ratio/min": 0.7406872153282166, "sampling/sampling_logp_difference/max": 0.32908267378807066, "sampling/sampling_logp_difference/mean": 0.020690289605408908, "step": 2250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.6, "completions/max_terminated_length": 283.6, "completions/mean_length": 194.7625, "completions/mean_terminated_length": 194.7625, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0276084169745445, "epoch": 0.20777787993012778, "frac_reward_zero_std": 0.2, "grad_norm": 7.0738301277160645, "kl": 0.010275126568740234, "learning_rate": 4.994954510161331e-06, "loss": 0.0007, "num_tokens": 9812244.0, "reward": 0.15, "reward_std": 0.7002923905849456, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.7002924025058747, "sampling/importance_sampling_ratio/max": 1.317594587802887, "sampling/importance_sampling_ratio/mean": 1.0002803802490234, "sampling/importance_sampling_ratio/min": 0.7714057505130768, "sampling/sampling_logp_difference/max": 0.3138373255729675, "sampling/sampling_logp_difference/mean": 0.02113717086613178, "step": 2260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.6, "completions/max_terminated_length": 295.6, "completions/mean_length": 198.0, "completions/mean_terminated_length": 198.0, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.02365155890584, "epoch": 0.2086972510802611, "frac_reward_zero_std": 0.7, "grad_norm": 0.11521594971418381, "kl": 0.011075348156737164, "learning_rate": 4.994792342421852e-06, "loss": 0.0009, "num_tokens": 9854492.0, "reward": 0.05, "reward_std": 0.30639100074768066, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.30639100074768066, "sampling/importance_sampling_ratio/max": 1.2581071972846984, "sampling/importance_sampling_ratio/mean": 0.999862688779831, "sampling/importance_sampling_ratio/min": 0.7617414176464081, "sampling/sampling_logp_difference/max": 0.28340089321136475, "sampling/sampling_logp_difference/mean": 0.020963709242641924, "step": 2270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 224.25, "completions/mean_terminated_length": 224.25, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.071159054338932, "epoch": 0.2096166222303944, "frac_reward_zero_std": 0.3, "grad_norm": 5.894285202026367, "kl": 0.009414625854697078, "learning_rate": 4.994627612330655e-06, "loss": 0.0007, "num_tokens": 9904320.0, "reward": -0.025, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.279772734642029, "sampling/importance_sampling_ratio/mean": 0.9998757779598236, "sampling/importance_sampling_ratio/min": 0.7365552365779877, "sampling/sampling_logp_difference/max": 0.32968672513961794, "sampling/sampling_logp_difference/mean": 0.020506482757627963, "step": 2280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 223.225, "completions/mean_terminated_length": 220.52857360839843, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.0226584650576114, "epoch": 0.21053599338052773, "frac_reward_zero_std": 0.2, "grad_norm": 5.900231838226318, "kl": 0.009197174847940914, "learning_rate": 4.994460320056932e-06, "loss": 0.0002, "num_tokens": 9949010.0, "reward": -0.20625, "reward_std": 0.7757517218589782, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.7757517576217652, "sampling/importance_sampling_ratio/max": 1.2958923935890199, "sampling/importance_sampling_ratio/mean": 1.0001584827899932, "sampling/importance_sampling_ratio/min": 0.7811731994152069, "sampling/sampling_logp_difference/max": 0.288016140460968, "sampling/sampling_logp_difference/mean": 0.020023071300238372, "step": 2290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.2, "completions/max_terminated_length": 314.2, "completions/mean_length": 213.3625, "completions/mean_terminated_length": 213.3625, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.1088243633508683, "epoch": 0.21145536453066102, "frac_reward_zero_std": 0.6, "grad_norm": 0.07762943208217621, "kl": 0.010659564658999443, "learning_rate": 4.9942904657725044e-06, "loss": 0.0004, "num_tokens": 9992383.0, "reward": -0.3, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2767234206199647, "sampling/importance_sampling_ratio/mean": 0.9993786454200745, "sampling/importance_sampling_ratio/min": 0.7802468001842499, "sampling/sampling_logp_difference/max": 0.28068501949310304, "sampling/sampling_logp_difference/mean": 0.02136251423507929, "step": 2300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 220.925, "completions/mean_terminated_length": 220.925, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.095242941379547, "epoch": 0.21237473568079435, "frac_reward_zero_std": 0.5, "grad_norm": 0.11877265572547913, "kl": 0.01342452034004964, "learning_rate": 4.9941180496518265e-06, "loss": 0.0007, "num_tokens": 10040121.0, "reward": -0.15, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2812605381011963, "sampling/importance_sampling_ratio/mean": 1.000442123413086, "sampling/importance_sampling_ratio/min": 0.7842645943164825, "sampling/sampling_logp_difference/max": 0.27679802775382994, "sampling/sampling_logp_difference/mean": 0.0211794326081872, "step": 2310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.2, "completions/max_terminated_length": 294.2, "completions/mean_length": 206.2375, "completions/mean_terminated_length": 206.2375, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.1397470109164716, "epoch": 0.21329410683092764, "frac_reward_zero_std": 0.2, "grad_norm": 0.1568358689546585, "kl": 0.011108913674252107, "learning_rate": 4.993943071871984e-06, "loss": 0.0006, "num_tokens": 10084212.0, "reward": -0.25, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.278085994720459, "sampling/importance_sampling_ratio/mean": 0.999704647064209, "sampling/importance_sampling_ratio/min": 0.7767283201217652, "sampling/sampling_logp_difference/max": 0.2946625709533691, "sampling/sampling_logp_difference/mean": 0.0220833420753479, "step": 2320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.9, "completions/max_terminated_length": 300.9, "completions/mean_length": 222.2, "completions/mean_terminated_length": 222.2, "completions/min_length": 151.4, "completions/min_terminated_length": 151.4, "entropy": 1.0567740835249424, "epoch": 0.21421347798106097, "frac_reward_zero_std": 0.4, "grad_norm": 5.945573806762695, "kl": 0.011036351876100525, "learning_rate": 4.993765532612692e-06, "loss": 0.0004, "num_tokens": 10131108.0, "reward": -0.175, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.2286440134048462, "sampling/importance_sampling_ratio/mean": 1.0001855790615082, "sampling/importance_sampling_ratio/min": 0.7589503645896911, "sampling/sampling_logp_difference/max": 0.2917361855506897, "sampling/sampling_logp_difference/mean": 0.0207851130515337, "step": 2330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.4, "completions/max_terminated_length": 316.4, "completions/mean_length": 216.65, "completions/mean_terminated_length": 216.65, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.0723932914435863, "epoch": 0.21513284913119426, "frac_reward_zero_std": 0.7, "grad_norm": 0.11017471551895142, "kl": 0.011248399940086529, "learning_rate": 4.993585432056299e-06, "loss": 0.0004, "num_tokens": 10173528.0, "reward": -0.675, "reward_std": 0.2886738538742065, "rewards/reward_fn/mean": -0.675, "rewards/reward_fn/std": 0.2886738657951355, "sampling/importance_sampling_ratio/max": 1.271712851524353, "sampling/importance_sampling_ratio/mean": 1.000087523460388, "sampling/importance_sampling_ratio/min": 0.7742648005485535, "sampling/sampling_logp_difference/max": 0.2740587055683136, "sampling/sampling_logp_difference/mean": 0.021255279332399367, "step": 2340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.4, "completions/max_terminated_length": 336.4, "completions/mean_length": 229.3875, "completions/mean_terminated_length": 229.3875, "completions/min_length": 135.4, "completions/min_terminated_length": 135.4, "entropy": 1.0739058203995229, "epoch": 0.21605222028132756, "frac_reward_zero_std": 0.2, "grad_norm": 5.15317440032959, "kl": 0.010353507092804647, "learning_rate": 4.993402770387782e-06, "loss": 0.0014, "num_tokens": 10216175.0, "reward": -0.125, "reward_std": 0.7440193653106689, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.7440194010734558, "sampling/importance_sampling_ratio/max": 1.2781758427619934, "sampling/importance_sampling_ratio/mean": 1.000088208913803, "sampling/importance_sampling_ratio/min": 0.7820380389690399, "sampling/sampling_logp_difference/max": 0.2731472015380859, "sampling/sampling_logp_difference/mean": 0.020743687450885773, "step": 2350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 216.4875, "completions/mean_terminated_length": 216.4875, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.0812455102801324, "epoch": 0.2169715914314609, "frac_reward_zero_std": 0.5, "grad_norm": 0.19482657313346863, "kl": 0.011492405075114221, "learning_rate": 4.993217547794748e-06, "loss": 0.0003, "num_tokens": 10261182.0, "reward": -0.2, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.255460548400879, "sampling/importance_sampling_ratio/mean": 0.9997986614704132, "sampling/importance_sampling_ratio/min": 0.7558029353618622, "sampling/sampling_logp_difference/max": 0.28722001016139986, "sampling/sampling_logp_difference/mean": 0.02172707561403513, "step": 2360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.3, "completions/max_terminated_length": 290.3, "completions/mean_length": 215.15, "completions/mean_terminated_length": 215.15, "completions/min_length": 153.1, "completions/min_terminated_length": 153.1, "entropy": 1.1137432523071766, "epoch": 0.21789096258159418, "frac_reward_zero_std": 0.5, "grad_norm": 5.620314121246338, "kl": 0.010604341316502541, "learning_rate": 4.993029764467438e-06, "loss": 0.0007, "num_tokens": 10304882.0, "reward": -0.10625, "reward_std": 0.46320008039474486, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.4632001042366028, "sampling/importance_sampling_ratio/max": 1.3321283459663391, "sampling/importance_sampling_ratio/mean": 1.0001504361629485, "sampling/importance_sampling_ratio/min": 0.7508431315422058, "sampling/sampling_logp_difference/max": 0.33485397398471833, "sampling/sampling_logp_difference/mean": 0.021807814389467238, "step": 2370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 214.375, "completions/mean_terminated_length": 214.375, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.0290556006133555, "epoch": 0.2188103337317275, "frac_reward_zero_std": 0.3, "grad_norm": 0.11104036867618561, "kl": 0.010015181597555056, "learning_rate": 4.992839420598719e-06, "loss": 0.0003, "num_tokens": 10351464.0, "reward": 0.05, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2188105583190918, "sampling/importance_sampling_ratio/mean": 0.9997400104999542, "sampling/importance_sampling_ratio/min": 0.769007408618927, "sampling/sampling_logp_difference/max": 0.2647926926612854, "sampling/sampling_logp_difference/mean": 0.019805621169507504, "step": 2380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.7, "completions/max_terminated_length": 273.7, "completions/mean_length": 196.475, "completions/mean_terminated_length": 196.475, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0501971252262592, "epoch": 0.2197297048818608, "frac_reward_zero_std": 0.6, "grad_norm": 0.11176208406686783, "kl": 0.01021952269366011, "learning_rate": 4.992646516384089e-06, "loss": 0.0006, "num_tokens": 10393406.0, "reward": -0.4, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.27602881193161, "sampling/importance_sampling_ratio/mean": 0.9999139130115509, "sampling/importance_sampling_ratio/min": 0.7839913487434387, "sampling/sampling_logp_difference/max": 0.26058757305145264, "sampling/sampling_logp_difference/mean": 0.020774257369339466, "step": 2390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 281.9, "completions/max_terminated_length": 280.0, "completions/mean_length": 213.3375, "completions/mean_terminated_length": 211.0625, "completions/min_length": 158.3, "completions/min_terminated_length": 158.3, "entropy": 1.0626112777739762, "epoch": 0.22064907603199413, "frac_reward_zero_std": 0.7, "grad_norm": 0.09549110382795334, "kl": 0.010987402705359273, "learning_rate": 4.992451052021676e-06, "loss": 0.0006, "num_tokens": 10437145.0, "reward": -0.1125, "reward_std": 0.24406498670578003, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.24406498670578003, "sampling/importance_sampling_ratio/max": 1.242991304397583, "sampling/importance_sampling_ratio/mean": 1.0008363842964172, "sampling/importance_sampling_ratio/min": 0.774161183834076, "sampling/sampling_logp_difference/max": 0.2667845726013184, "sampling/sampling_logp_difference/mean": 0.020968471001833678, "step": 2400 }, { "epoch": 0.22064907603199413, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 304.2739965095986, "eval_completions/max_terminated_length": 301.10296684118674, "eval_completions/mean_length": 213.16884816753927, "eval_completions/mean_terminated_length": 212.31466632197248, "eval_completions/min_length": 141.20244328097732, "eval_completions/min_terminated_length": 141.20244328097732, "eval_entropy": 1.1061905301574222, "eval_frac_reward_zero_std": 0.46596858638743455, "eval_kl": 0.010792956411201397, "eval_loss": 0.00048375115147791803, "eval_num_tokens": 10437145.0, "eval_reward": -0.14092495636998253, "eval_reward_std": 0.47299190500123756, "eval_rewards/reward_fn/mean": -0.14092495636998253, "eval_rewards/reward_fn/std": 0.47299192112465804, "eval_runtime": 1765.4789, "eval_samples_per_second": 0.325, "eval_sampling/importance_sampling_ratio/max": 1.2844815543392774, "eval_sampling/importance_sampling_ratio/mean": 1.0000747459929236, "eval_sampling/importance_sampling_ratio/min": 0.7767876932758311, "eval_sampling/sampling_logp_difference/max": 0.2918822692534894, "eval_sampling/sampling_logp_difference/mean": 0.021156296893843754, "eval_steps_per_second": 0.041, "step": 2400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 212.2, "completions/mean_terminated_length": 212.2, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.012891410291195, "epoch": 0.22156844718212743, "frac_reward_zero_std": 0.4, "grad_norm": 0.09080097079277039, "kl": 0.010210090436157771, "learning_rate": 4.992253027712238e-06, "loss": 0.0005, "num_tokens": 10481777.0, "reward": -0.35625, "reward_std": 0.5544095754623413, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.5544095873832703, "sampling/importance_sampling_ratio/max": 1.2732836961746217, "sampling/importance_sampling_ratio/mean": 1.0004545986652373, "sampling/importance_sampling_ratio/min": 0.7873417735099792, "sampling/sampling_logp_difference/max": 0.2558030247688293, "sampling/sampling_logp_difference/mean": 0.020424568466842173, "step": 2410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 203.2125, "completions/mean_terminated_length": 203.2125, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.0779230050742625, "epoch": 0.22248781833226072, "frac_reward_zero_std": 0.4, "grad_norm": 0.17877258360385895, "kl": 0.010291895939735696, "learning_rate": 4.992052443659163e-06, "loss": 0.0004, "num_tokens": 10521474.0, "reward": -0.075, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2814240217208863, "sampling/importance_sampling_ratio/mean": 1.0004507303237915, "sampling/importance_sampling_ratio/min": 0.7950044631958008, "sampling/sampling_logp_difference/max": 0.2773540496826172, "sampling/sampling_logp_difference/mean": 0.02139242775738239, "step": 2420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.0, "completions/max_terminated_length": 272.0, "completions/mean_length": 195.0375, "completions/mean_terminated_length": 195.0375, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0953298382461072, "epoch": 0.22340718948239405, "frac_reward_zero_std": 0.7, "grad_norm": 5.323483943939209, "kl": 0.009298543210024945, "learning_rate": 4.9918493000684664e-06, "loss": 0.0005, "num_tokens": 10566093.0, "reward": 0.275, "reward_std": 0.3105294942855835, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.31052953004837036, "sampling/importance_sampling_ratio/max": 1.2274282336235047, "sampling/importance_sampling_ratio/mean": 1.0001396238803864, "sampling/importance_sampling_ratio/min": 0.7838948667049408, "sampling/sampling_logp_difference/max": 0.245469868183136, "sampling/sampling_logp_difference/mean": 0.02208430152386427, "step": 2430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.3, "completions/max_terminated_length": 327.3, "completions/mean_length": 218.45, "completions/mean_terminated_length": 218.45, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.1585793197154999, "epoch": 0.22432656063252734, "frac_reward_zero_std": 0.3, "grad_norm": 6.084267616271973, "kl": 0.011713143746601418, "learning_rate": 4.991643597148792e-06, "loss": 0.0002, "num_tokens": 10611793.0, "reward": -0.275, "reward_std": 0.6657598376274109, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6657598733901977, "sampling/importance_sampling_ratio/max": 1.3506702661514283, "sampling/importance_sampling_ratio/mean": 0.999789547920227, "sampling/importance_sampling_ratio/min": 0.7382666528224945, "sampling/sampling_logp_difference/max": 0.39910115003585817, "sampling/sampling_logp_difference/mean": 0.022686055302619933, "step": 2440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 351.3, "completions/max_terminated_length": 324.8, "completions/mean_length": 224.775, "completions/mean_terminated_length": 221.15178680419922, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0674375340342521, "epoch": 0.22524593178266067, "frac_reward_zero_std": 0.5, "grad_norm": 6.151426792144775, "kl": 0.011184162937570363, "learning_rate": 4.991435335111413e-06, "loss": 0.0006, "num_tokens": 10656855.0, "reward": -0.6125, "reward_std": 0.33495701253414156, "rewards/reward_fn/mean": -0.6125, "rewards/reward_fn/std": 0.33495703637599944, "sampling/importance_sampling_ratio/max": 1.3381534218788147, "sampling/importance_sampling_ratio/mean": 0.9998226761817932, "sampling/importance_sampling_ratio/min": 0.7321114182472229, "sampling/sampling_logp_difference/max": 0.3585946261882782, "sampling/sampling_logp_difference/mean": 0.021139444410800935, "step": 2450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 352.4, "completions/max_terminated_length": 345.1, "completions/mean_length": 242.55, "completions/mean_terminated_length": 240.15714416503906, "completions/min_length": 169.6, "completions/min_terminated_length": 169.6, "entropy": 1.145337277650833, "epoch": 0.22616530293279397, "frac_reward_zero_std": 0.4, "grad_norm": 6.1155900955200195, "kl": 0.00998126352787949, "learning_rate": 4.991224514170232e-06, "loss": 0.001, "num_tokens": 10702027.0, "reward": -0.38125, "reward_std": 0.4149737223982811, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.4149737223982811, "sampling/importance_sampling_ratio/max": 1.2451242804527283, "sampling/importance_sampling_ratio/mean": 0.9997696101665496, "sampling/importance_sampling_ratio/min": 0.7817505836486817, "sampling/sampling_logp_difference/max": 0.24983067512512208, "sampling/sampling_logp_difference/mean": 0.022040662728250025, "step": 2460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 288.9, "completions/max_terminated_length": 275.4, "completions/mean_length": 204.9375, "completions/mean_terminated_length": 202.03928833007814, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.0806695990264417, "epoch": 0.2270846740829273, "frac_reward_zero_std": 0.5, "grad_norm": 7.714158058166504, "kl": 0.011774570029228926, "learning_rate": 4.9910111345417785e-06, "loss": 0.0002, "num_tokens": 10745462.0, "reward": -0.23125, "reward_std": 0.32239171117544174, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.32239171117544174, "sampling/importance_sampling_ratio/max": 1.3346067547798157, "sampling/importance_sampling_ratio/mean": 1.0000242352485658, "sampling/importance_sampling_ratio/min": 0.6988933742046356, "sampling/sampling_logp_difference/max": 0.4221360445022583, "sampling/sampling_logp_difference/mean": 0.02174580655992031, "step": 2470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 210.4375, "completions/mean_terminated_length": 210.4375, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.0609476953744887, "epoch": 0.2280040452330606, "frac_reward_zero_std": 0.4, "grad_norm": 6.853574752807617, "kl": 0.011474801710573957, "learning_rate": 4.990795196445212e-06, "loss": 0.0005, "num_tokens": 10785913.0, "reward": -0.1, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.2924336552619935, "sampling/importance_sampling_ratio/mean": 0.9996351897716522, "sampling/importance_sampling_ratio/min": 0.7725027203559875, "sampling/sampling_logp_difference/max": 0.29560717940330505, "sampling/sampling_logp_difference/mean": 0.02085524294525385, "step": 2480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.1, "completions/max_terminated_length": 332.1, "completions/mean_length": 203.3, "completions/mean_terminated_length": 203.3, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.0484023958444595, "epoch": 0.22892341638319388, "frac_reward_zero_std": 0.6, "grad_norm": 0.11710735410451889, "kl": 0.009766783157829195, "learning_rate": 4.990576700102316e-06, "loss": 0.0008, "num_tokens": 10826689.0, "reward": -0.175, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2637648582458496, "sampling/importance_sampling_ratio/mean": 0.9993664860725403, "sampling/importance_sampling_ratio/min": 0.7644260466098786, "sampling/sampling_logp_difference/max": 0.28748432397842405, "sampling/sampling_logp_difference/mean": 0.020741592906415463, "step": 2490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.0, "completions/max_terminated_length": 247.0, "completions/mean_length": 182.4, "completions/mean_terminated_length": 182.4, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 0.9901092275977135, "epoch": 0.2298427875333272, "frac_reward_zero_std": 0.3, "grad_norm": 7.0520758628845215, "kl": 0.01061692349612713, "learning_rate": 4.990355645737506e-06, "loss": 0.0004, "num_tokens": 10866697.0, "reward": 0.025, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.286341631412506, "sampling/importance_sampling_ratio/mean": 1.0005799055099487, "sampling/importance_sampling_ratio/min": 0.7804164230823517, "sampling/sampling_logp_difference/max": 0.2599011421203613, "sampling/sampling_logp_difference/mean": 0.020651878044009208, "step": 2500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.2, "completions/max_terminated_length": 247.2, "completions/mean_length": 180.2, "completions/mean_terminated_length": 180.2, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.010568869113922, "epoch": 0.2307621586834605, "frac_reward_zero_std": 0.4, "grad_norm": 5.519500255584717, "kl": 0.012951495757442899, "learning_rate": 4.99013203357782e-06, "loss": 0.0006, "num_tokens": 10907969.0, "reward": -0.275, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.3255655646324158, "sampling/importance_sampling_ratio/mean": 0.9998658239841461, "sampling/importance_sampling_ratio/min": 0.7827211439609527, "sampling/sampling_logp_difference/max": 0.3064261794090271, "sampling/sampling_logp_difference/mean": 0.02040474358946085, "step": 2510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 204.6125, "completions/mean_terminated_length": 204.6125, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.068831381946802, "epoch": 0.23168152983359383, "frac_reward_zero_std": 0.4, "grad_norm": 4.786029815673828, "kl": 0.012454909808002412, "learning_rate": 4.989905863852928e-06, "loss": 0.001, "num_tokens": 10948906.0, "reward": -0.375, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.3508333683013916, "sampling/importance_sampling_ratio/mean": 1.0003152549266816, "sampling/importance_sampling_ratio/min": 0.7295522838830948, "sampling/sampling_logp_difference/max": 0.3845064163208008, "sampling/sampling_logp_difference/mean": 0.02151664812117815, "step": 2520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.1, "completions/max_terminated_length": 265.1, "completions/mean_length": 186.0625, "completions/mean_terminated_length": 186.0625, "completions/min_length": 115.6, "completions/min_terminated_length": 115.6, "entropy": 1.0076603196561336, "epoch": 0.23260090098372713, "frac_reward_zero_std": 0.4, "grad_norm": 6.266746520996094, "kl": 0.010516808024840429, "learning_rate": 4.989677136795123e-06, "loss": 0.0002, "num_tokens": 10988127.0, "reward": -0.225, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.315231728553772, "sampling/importance_sampling_ratio/mean": 0.9997446894645691, "sampling/importance_sampling_ratio/min": 0.7838646709918976, "sampling/sampling_logp_difference/max": 0.28645175099372866, "sampling/sampling_logp_difference/mean": 0.02137715145945549, "step": 2530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 355.3, "completions/max_terminated_length": 349.1, "completions/mean_length": 241.35, "completions/mean_terminated_length": 238.1267868041992, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.1024777010083198, "epoch": 0.23352027213386045, "frac_reward_zero_std": 0.3, "grad_norm": 5.375761985778809, "kl": 0.010802687163231895, "learning_rate": 4.989445852639328e-06, "loss": 0.0007, "num_tokens": 11037507.0, "reward": -0.6125, "reward_std": 0.5045932337641716, "rewards/reward_fn/mean": -0.6125, "rewards/reward_fn/std": 0.5045932397246361, "sampling/importance_sampling_ratio/max": 1.2515447020530701, "sampling/importance_sampling_ratio/mean": 0.9992856025695801, "sampling/importance_sampling_ratio/min": 0.7298306107521058, "sampling/sampling_logp_difference/max": 0.33117915391922, "sampling/sampling_logp_difference/mean": 0.02149066757410765, "step": 2540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.7, "completions/max_terminated_length": 332.7, "completions/mean_length": 223.4, "completions/mean_terminated_length": 223.4, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.0407434470951558, "epoch": 0.23443964328399375, "frac_reward_zero_std": 0.3, "grad_norm": 5.2555437088012695, "kl": 0.010391264443751425, "learning_rate": 4.989212011623087e-06, "loss": 0.0004, "num_tokens": 11083235.0, "reward": -0.1, "reward_std": 0.6589862048625946, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6589862406253815, "sampling/importance_sampling_ratio/max": 1.2722551822662354, "sampling/importance_sampling_ratio/mean": 1.0003579378128051, "sampling/importance_sampling_ratio/min": 0.7531703352928162, "sampling/sampling_logp_difference/max": 0.308605295419693, "sampling/sampling_logp_difference/mean": 0.021066922321915628, "step": 2550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 204.375, "completions/mean_terminated_length": 204.375, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0401474073529244, "epoch": 0.23535901443412705, "frac_reward_zero_std": 0.5, "grad_norm": 4.736339092254639, "kl": 0.010939698299625888, "learning_rate": 4.988975613986575e-06, "loss": 0.0005, "num_tokens": 11127249.0, "reward": 0.075, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.2526020646095275, "sampling/importance_sampling_ratio/mean": 0.9999776303768158, "sampling/importance_sampling_ratio/min": 0.7463765978813172, "sampling/sampling_logp_difference/max": 0.29964451789855956, "sampling/sampling_logp_difference/mean": 0.02115418128669262, "step": 2560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 203.625, "completions/mean_terminated_length": 203.625, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 0.9791261278092861, "epoch": 0.23627838558426037, "frac_reward_zero_std": 0.4, "grad_norm": 4.7393693923950195, "kl": 0.011097777530085295, "learning_rate": 4.988736659972593e-06, "loss": 0.0007, "num_tokens": 11171859.0, "reward": -0.5625, "reward_std": 0.43558403104543686, "rewards/reward_fn/mean": -0.5625, "rewards/reward_fn/std": 0.43558403104543686, "sampling/importance_sampling_ratio/max": 1.3075982570648192, "sampling/importance_sampling_ratio/mean": 0.9993849635124207, "sampling/importance_sampling_ratio/min": 0.7677158713340759, "sampling/sampling_logp_difference/max": 0.30457318425178526, "sampling/sampling_logp_difference/mean": 0.0206495501101017, "step": 2570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.2, "completions/max_terminated_length": 267.2, "completions/mean_length": 188.4875, "completions/mean_terminated_length": 188.4875, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 0.9516775116324425, "epoch": 0.23719775673439367, "frac_reward_zero_std": 0.4, "grad_norm": 6.211798667907715, "kl": 0.011422870605019852, "learning_rate": 4.988495149826563e-06, "loss": 0.0005, "num_tokens": 11215930.0, "reward": 0.25, "reward_std": 0.5294508516788483, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.5294508635997772, "sampling/importance_sampling_ratio/max": 1.2587588429450989, "sampling/importance_sampling_ratio/mean": 1.0000974833965302, "sampling/importance_sampling_ratio/min": 0.7788911402225495, "sampling/sampling_logp_difference/max": 0.26046243906021116, "sampling/sampling_logp_difference/mean": 0.019774654507637025, "step": 2580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 185.375, "completions/mean_terminated_length": 185.375, "completions/min_length": 118.7, "completions/min_terminated_length": 118.7, "entropy": 0.9108937297016382, "epoch": 0.238117127884527, "frac_reward_zero_std": 0.3, "grad_norm": 5.342586517333984, "kl": 0.010309187424718402, "learning_rate": 4.988251083796539e-06, "loss": 0.0007, "num_tokens": 11258080.0, "reward": -0.175, "reward_std": 0.5933722019195556, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5933722376823425, "sampling/importance_sampling_ratio/max": 1.2969301104545594, "sampling/importance_sampling_ratio/mean": 0.9997437119483947, "sampling/importance_sampling_ratio/min": 0.7654163122177124, "sampling/sampling_logp_difference/max": 0.31805499494075773, "sampling/sampling_logp_difference/mean": 0.019591809157282113, "step": 2590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 199.0375, "completions/mean_terminated_length": 199.0375, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0034102484583856, "epoch": 0.2390364990346603, "frac_reward_zero_std": 0.4, "grad_norm": 0.13150428235530853, "kl": 0.01085330477799289, "learning_rate": 4.988004462133192e-06, "loss": 0.0004, "num_tokens": 11301235.0, "reward": -0.025, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.2738400220870971, "sampling/importance_sampling_ratio/mean": 1.0002694249153137, "sampling/importance_sampling_ratio/min": 0.7865205645561218, "sampling/sampling_logp_difference/max": 0.2567598342895508, "sampling/sampling_logp_difference/mean": 0.020776056870818137, "step": 2600 }, { "epoch": 0.2390364990346603, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 286.29493891797557, "eval_completions/max_terminated_length": 284.90575916230364, "eval_completions/mean_length": 200.99258289703315, "eval_completions/mean_terminated_length": 200.7225443788432, "eval_completions/min_length": 130.95811518324606, "eval_completions/min_terminated_length": 130.95811518324606, "eval_entropy": 1.0417593708213087, "eval_frac_reward_zero_std": 0.43455497382198954, "eval_kl": 0.012034385823474037, "eval_loss": 0.0006278703222051263, "eval_num_tokens": 11301235.0, "eval_reward": -0.11976439790575916, "eval_reward_std": 0.5010741019124136, "eval_rewards/reward_fn/mean": -0.11976439790575916, "eval_rewards/reward_fn/std": 0.501074119492143, "eval_runtime": 1675.0426, "eval_samples_per_second": 0.342, "eval_sampling/importance_sampling_ratio/max": 1.26646597722438, "eval_sampling/importance_sampling_ratio/mean": 0.9999966176184476, "eval_sampling/importance_sampling_ratio/min": 0.7705182895766503, "eval_sampling/sampling_logp_difference/max": 0.29292503717057994, "eval_sampling/sampling_logp_difference/mean": 0.02112447244503631, "eval_steps_per_second": 0.043, "step": 2600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.3, "completions/max_terminated_length": 274.3, "completions/mean_length": 193.2, "completions/mean_terminated_length": 193.2, "completions/min_length": 118.7, "completions/min_terminated_length": 118.7, "entropy": 0.9342095389962196, "epoch": 0.2399558701847936, "frac_reward_zero_std": 0.5, "grad_norm": 5.481683731079102, "kl": 0.011370484164217487, "learning_rate": 4.9877552850898255e-06, "loss": 0.0003, "num_tokens": 11338755.0, "reward": 0.05, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2823458433151245, "sampling/importance_sampling_ratio/mean": 1.0002991020679475, "sampling/importance_sampling_ratio/min": 0.7793822586536407, "sampling/sampling_logp_difference/max": 0.28696462512016296, "sampling/sampling_logp_difference/mean": 0.019730937294662, "step": 2610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 207.625, "completions/mean_terminated_length": 207.625, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 0.9615633614361286, "epoch": 0.2408752413349269, "frac_reward_zero_std": 0.2, "grad_norm": 5.937769412994385, "kl": 0.011780701664974913, "learning_rate": 4.987503552922363e-06, "loss": 0.0004, "num_tokens": 11381453.0, "reward": 0.05, "reward_std": 0.79868985414505, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.7986899018287659, "sampling/importance_sampling_ratio/max": 1.2621464610099793, "sampling/importance_sampling_ratio/mean": 1.0001003086566924, "sampling/importance_sampling_ratio/min": 0.7352514147758484, "sampling/sampling_logp_difference/max": 0.3316683769226074, "sampling/sampling_logp_difference/mean": 0.019993729330599307, "step": 2620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.7, "completions/max_terminated_length": 272.7, "completions/mean_length": 195.4375, "completions/mean_terminated_length": 195.4375, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.0105532675981521, "epoch": 0.2417946124850602, "frac_reward_zero_std": 0.5, "grad_norm": 6.371341705322266, "kl": 0.012454901641467587, "learning_rate": 4.987249265889355e-06, "loss": 0.0007, "num_tokens": 11424616.0, "reward": -0.25, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2932299733161927, "sampling/importance_sampling_ratio/mean": 1.0000790357589722, "sampling/importance_sampling_ratio/min": 0.7492846131324769, "sampling/sampling_logp_difference/max": 0.33617364168167113, "sampling/sampling_logp_difference/mean": 0.020268334820866585, "step": 2630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 210.6375, "completions/mean_terminated_length": 210.6375, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "entropy": 1.0500336833298207, "epoch": 0.24271398363519353, "frac_reward_zero_std": 0.5, "grad_norm": 0.0916207954287529, "kl": 0.012070966837927698, "learning_rate": 4.9869924242519715e-06, "loss": 0.0009, "num_tokens": 11467819.0, "reward": -0.575, "reward_std": 0.42594102025032043, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.42594102025032043, "sampling/importance_sampling_ratio/max": 1.2540933609008789, "sampling/importance_sampling_ratio/mean": 1.0000102996826172, "sampling/importance_sampling_ratio/min": 0.783137995004654, "sampling/sampling_logp_difference/max": 0.2580528974533081, "sampling/sampling_logp_difference/mean": 0.02088972311466932, "step": 2640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 188.9375, "completions/mean_terminated_length": 188.9375, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.1493305429816245, "epoch": 0.24363335478532683, "frac_reward_zero_std": 0.3, "grad_norm": 7.070753574371338, "kl": 0.012486370949773118, "learning_rate": 4.986733028274013e-06, "loss": 0.0004, "num_tokens": 11509942.0, "reward": -0.05625, "reward_std": 0.605814391374588, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.605814403295517, "sampling/importance_sampling_ratio/max": 1.2649933457374574, "sampling/importance_sampling_ratio/mean": 0.9999022781848907, "sampling/importance_sampling_ratio/min": 0.7746386528015137, "sampling/sampling_logp_difference/max": 0.2880581557750702, "sampling/sampling_logp_difference/mean": 0.022317446209490298, "step": 2650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.5, "completions/max_terminated_length": 263.5, "completions/mean_length": 189.7375, "completions/mean_terminated_length": 189.7375, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0661252021789551, "epoch": 0.24455272593546015, "frac_reward_zero_std": 0.7, "grad_norm": 0.12563222646713257, "kl": 0.010568220540881157, "learning_rate": 4.986471078221898e-06, "loss": 0.0008, "num_tokens": 11555481.0, "reward": -0.175, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.297947108745575, "sampling/importance_sampling_ratio/mean": 1.0002429723739623, "sampling/importance_sampling_ratio/min": 0.7493366897106171, "sampling/sampling_logp_difference/max": 0.34448201656341554, "sampling/sampling_logp_difference/mean": 0.02102298103272915, "step": 2660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.4, "completions/max_terminated_length": 257.4, "completions/mean_length": 197.8375, "completions/mean_terminated_length": 197.8375, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0197341497987509, "epoch": 0.24547209708559345, "frac_reward_zero_std": 0.3, "grad_norm": 5.0218400955200195, "kl": 0.01131791535008233, "learning_rate": 4.986206574364671e-06, "loss": 0.0006, "num_tokens": 11601156.0, "reward": 0.1, "reward_std": 0.5824443817138671, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5824444055557251, "sampling/importance_sampling_ratio/max": 1.325544548034668, "sampling/importance_sampling_ratio/mean": 1.0002475202083587, "sampling/importance_sampling_ratio/min": 0.7989067554473877, "sampling/sampling_logp_difference/max": 0.33252166509628295, "sampling/sampling_logp_difference/mean": 0.020492627378553153, "step": 2670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 208.8875, "completions/mean_terminated_length": 208.8875, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0235033169388772, "epoch": 0.24639146823572677, "frac_reward_zero_std": 0.6, "grad_norm": 6.504693984985352, "kl": 0.012164646032033488, "learning_rate": 4.985939516973996e-06, "loss": 0.0006, "num_tokens": 11642915.0, "reward": -0.05625, "reward_std": 0.3585855305194855, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.35858553647994995, "sampling/importance_sampling_ratio/max": 1.2570522904396058, "sampling/importance_sampling_ratio/mean": 0.9999192655086517, "sampling/importance_sampling_ratio/min": 0.7578364491462708, "sampling/sampling_logp_difference/max": 0.28611356019973755, "sampling/sampling_logp_difference/mean": 0.020751627534627913, "step": 2680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.3, "completions/max_terminated_length": 271.3, "completions/mean_length": 197.15, "completions/mean_terminated_length": 197.15, "completions/min_length": 120.7, "completions/min_terminated_length": 120.7, "entropy": 1.0009728901088237, "epoch": 0.24731083938586007, "frac_reward_zero_std": 0.6, "grad_norm": 14.591148376464844, "kl": 0.010814885358558967, "learning_rate": 4.985669906324167e-06, "loss": 0.0001, "num_tokens": 11684655.0, "reward": 0.125, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.3763219237327575, "sampling/importance_sampling_ratio/mean": 0.9991706192493439, "sampling/importance_sampling_ratio/min": 0.7130636066198349, "sampling/sampling_logp_difference/max": 0.4927516579627991, "sampling/sampling_logp_difference/mean": 0.021281799674034117, "step": 2690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.1, "completions/max_terminated_length": 328.1, "completions/mean_length": 227.4125, "completions/mean_terminated_length": 227.4125, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.057347372174263, "epoch": 0.24823021053599337, "frac_reward_zero_std": 0.5, "grad_norm": 4.734621524810791, "kl": 0.010881451691966504, "learning_rate": 4.985397742692093e-06, "loss": 0.0004, "num_tokens": 11730120.0, "reward": -0.55, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2496525526046753, "sampling/importance_sampling_ratio/mean": 1.0003575265407563, "sampling/importance_sampling_ratio/min": 0.7743078649044037, "sampling/sampling_logp_difference/max": 0.29007959365844727, "sampling/sampling_logp_difference/mean": 0.020857038721442223, "step": 2700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.9, "completions/max_terminated_length": 292.9, "completions/mean_length": 201.0, "completions/mean_terminated_length": 201.0, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 0.9746059857308864, "epoch": 0.2491495816861267, "frac_reward_zero_std": 0.4, "grad_norm": 5.4649882316589355, "kl": 0.009764757449738681, "learning_rate": 4.985123026357308e-06, "loss": 0.0001, "num_tokens": 11771208.0, "reward": 0.15, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2702165484428405, "sampling/importance_sampling_ratio/mean": 1.000883662700653, "sampling/importance_sampling_ratio/min": 0.7850229263305664, "sampling/sampling_logp_difference/max": 0.2656403839588165, "sampling/sampling_logp_difference/mean": 0.0197589922696352, "step": 2710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.3, "completions/max_terminated_length": 279.3, "completions/mean_length": 201.375, "completions/mean_terminated_length": 201.375, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 0.8491265572607517, "epoch": 0.25006895283626, "frac_reward_zero_std": 0.4, "grad_norm": 0.23459449410438538, "kl": 0.01035195978183765, "learning_rate": 4.9848457576019685e-06, "loss": 0.0003, "num_tokens": 11814470.0, "reward": 0.05, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2525699138641357, "sampling/importance_sampling_ratio/mean": 1.0002151608467102, "sampling/importance_sampling_ratio/min": 0.6929373525083065, "sampling/sampling_logp_difference/max": 0.5292375922203064, "sampling/sampling_logp_difference/mean": 0.01922444812953472, "step": 2720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.8, "completions/max_terminated_length": 277.8, "completions/mean_length": 189.6, "completions/mean_terminated_length": 189.6, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0223263390362263, "epoch": 0.2509883239863933, "frac_reward_zero_std": 0.5, "grad_norm": 0.22966916859149933, "kl": 0.012335234187776222, "learning_rate": 4.98456593671085e-06, "loss": 0.0006, "num_tokens": 11855926.0, "reward": -0.275, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.3075085878372192, "sampling/importance_sampling_ratio/mean": 0.9996864795684814, "sampling/importance_sampling_ratio/min": 0.7377306252717972, "sampling/sampling_logp_difference/max": 0.40763256549835203, "sampling/sampling_logp_difference/mean": 0.021413985453546048, "step": 2730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.2, "completions/max_terminated_length": 326.2, "completions/mean_length": 228.2125, "completions/mean_terminated_length": 228.2125, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.0789699487388134, "epoch": 0.2519076951365266, "frac_reward_zero_std": 0.3, "grad_norm": 5.385793685913086, "kl": 0.011377238971181213, "learning_rate": 4.9842835639713525e-06, "loss": 0.0003, "num_tokens": 11898471.0, "reward": -0.275, "reward_std": 0.5967825591564179, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5967825591564179, "sampling/importance_sampling_ratio/max": 1.2995437622070312, "sampling/importance_sampling_ratio/mean": 1.0008242964744567, "sampling/importance_sampling_ratio/min": 0.76954225897789, "sampling/sampling_logp_difference/max": 0.3047028720378876, "sampling/sampling_logp_difference/mean": 0.021605622209608554, "step": 2740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 218.3125, "completions/mean_terminated_length": 218.3125, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1044444486498832, "epoch": 0.2528270662866599, "frac_reward_zero_std": 0.6, "grad_norm": 0.08391159772872925, "kl": 0.009758189955027774, "learning_rate": 4.983998639673496e-06, "loss": 0.0005, "num_tokens": 11946072.0, "reward": -0.25, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2679354310035706, "sampling/importance_sampling_ratio/mean": 1.0001905798912047, "sampling/importance_sampling_ratio/min": 0.7918044626712799, "sampling/sampling_logp_difference/max": 0.2518913745880127, "sampling/sampling_logp_difference/mean": 0.021429184824228287, "step": 2750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.3, "completions/max_terminated_length": 289.3, "completions/mean_length": 211.025, "completions/mean_terminated_length": 211.025, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.0102364595979452, "epoch": 0.25374643743679326, "frac_reward_zero_std": 0.5, "grad_norm": 0.16279277205467224, "kl": 0.010006821595015935, "learning_rate": 4.983711164109919e-06, "loss": 0.0006, "num_tokens": 11991466.0, "reward": -0.05, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.2472301244735717, "sampling/importance_sampling_ratio/mean": 1.0002774834632873, "sampling/importance_sampling_ratio/min": 0.7967600166797638, "sampling/sampling_logp_difference/max": 0.2502105712890625, "sampling/sampling_logp_difference/mean": 0.020141740888357164, "step": 2760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 343.4, "completions/max_terminated_length": 343.4, "completions/mean_length": 237.1375, "completions/mean_terminated_length": 237.1375, "completions/min_length": 164.4, "completions/min_terminated_length": 164.4, "entropy": 1.0597155183553695, "epoch": 0.25466580858692656, "frac_reward_zero_std": 0.5, "grad_norm": 0.12030981481075287, "kl": 0.011525061377324165, "learning_rate": 4.983421137575882e-06, "loss": 0.0006, "num_tokens": 12037853.0, "reward": -0.08125, "reward_std": 0.4935488224029541, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.49354884028434753, "sampling/importance_sampling_ratio/max": 1.291707408428192, "sampling/importance_sampling_ratio/mean": 0.9999069333076477, "sampling/importance_sampling_ratio/min": 0.7892123937606812, "sampling/sampling_logp_difference/max": 0.27344837188720705, "sampling/sampling_logp_difference/mean": 0.02076991256326437, "step": 2770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 212.75, "completions/mean_terminated_length": 212.75, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.0555035427212716, "epoch": 0.25558517973705985, "frac_reward_zero_std": 0.7, "grad_norm": 0.11179179698228836, "kl": 0.011389269348001108, "learning_rate": 4.983128560369267e-06, "loss": 0.0007, "num_tokens": 12079185.0, "reward": 0.45, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.2432140350341796, "sampling/importance_sampling_ratio/mean": 0.9999784052371978, "sampling/importance_sampling_ratio/min": 0.7587709248065948, "sampling/sampling_logp_difference/max": 0.2916135787963867, "sampling/sampling_logp_difference/mean": 0.020691538602113722, "step": 2780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 200.825, "completions/mean_terminated_length": 200.825, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 0.9693934060633183, "epoch": 0.25650455088719315, "frac_reward_zero_std": 0.5, "grad_norm": 7.554068088531494, "kl": 0.009981666947714985, "learning_rate": 4.982833432790573e-06, "loss": 0.0002, "num_tokens": 12117555.0, "reward": -0.15, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.2588335394859314, "sampling/importance_sampling_ratio/mean": 0.9994102954864502, "sampling/importance_sampling_ratio/min": 0.7465343534946441, "sampling/sampling_logp_difference/max": 0.303021514415741, "sampling/sampling_logp_difference/mean": 0.020233062095940113, "step": 2790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 218.575, "completions/mean_terminated_length": 218.575, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.0754728108644485, "epoch": 0.25742392203732645, "frac_reward_zero_std": 0.4, "grad_norm": 0.1442117989063263, "kl": 0.01055901226354763, "learning_rate": 4.982535755142921e-06, "loss": 0.0005, "num_tokens": 12160273.0, "reward": -0.225, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2408118724822998, "sampling/importance_sampling_ratio/mean": 0.9997322857379913, "sampling/importance_sampling_ratio/min": 0.79299236536026, "sampling/sampling_logp_difference/max": 0.24245622158050537, "sampling/sampling_logp_difference/mean": 0.021280864253640175, "step": 2800 }, { "epoch": 0.25742392203732645, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 299.11343804537523, "eval_completions/max_terminated_length": 296.3089005235602, "eval_completions/mean_length": 207.445462478185, "eval_completions/mean_terminated_length": 206.7672651999908, "eval_completions/min_length": 134.34904013961605, "eval_completions/min_terminated_length": 134.34904013961605, "eval_entropy": 1.0612048893178736, "eval_frac_reward_zero_std": 0.45898778359511344, "eval_kl": 0.011325637639271028, "eval_loss": 0.000535906758159399, "eval_num_tokens": 12160273.0, "eval_reward": -0.13154450261780104, "eval_reward_std": 0.48129275235726154, "eval_rewards/reward_fn/mean": -0.13154450261780104, "eval_rewards/reward_fn/std": 0.4812927679605717, "eval_runtime": 1740.1271, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.261911700949411, "eval_sampling/importance_sampling_ratio/mean": 1.0000402471782026, "eval_sampling/importance_sampling_ratio/min": 0.7662798052110805, "eval_sampling/sampling_logp_difference/max": 0.2914000768311985, "eval_sampling/sampling_logp_difference/mean": 0.021733401273336606, "eval_steps_per_second": 0.041, "step": 2800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.8, "completions/max_terminated_length": 245.8, "completions/mean_length": 181.775, "completions/mean_terminated_length": 181.775, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 0.9969171993434429, "epoch": 0.2583432931874598, "frac_reward_zero_std": 0.6, "grad_norm": 5.810840129852295, "kl": 0.009882683400064707, "learning_rate": 4.9822355277320475e-06, "loss": 0.0006, "num_tokens": 12200703.0, "reward": 0.125, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.273535168170929, "sampling/importance_sampling_ratio/mean": 1.0000237107276917, "sampling/importance_sampling_ratio/min": 0.7586061239242554, "sampling/sampling_logp_difference/max": 0.2983745813369751, "sampling/sampling_logp_difference/mean": 0.020835849083960058, "step": 2810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.7, "completions/max_terminated_length": 292.7, "completions/mean_length": 207.4375, "completions/mean_terminated_length": 207.4375, "completions/min_length": 113.9, "completions/min_terminated_length": 113.9, "entropy": 0.9921106960624456, "epoch": 0.2592626643375931, "frac_reward_zero_std": 0.3, "grad_norm": 0.09508863836526871, "kl": 0.009128287207568064, "learning_rate": 4.981932750866313e-06, "loss": 0.0007, "num_tokens": 12243874.0, "reward": -0.075, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.271787202358246, "sampling/importance_sampling_ratio/mean": 1.0004277527332306, "sampling/importance_sampling_ratio/min": 0.7598743975162506, "sampling/sampling_logp_difference/max": 0.29424619376659394, "sampling/sampling_logp_difference/mean": 0.02066533751785755, "step": 2820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 306.8, "completions/max_terminated_length": 291.6, "completions/mean_length": 209.4875, "completions/mean_terminated_length": 206.78035888671874, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0811770483851433, "epoch": 0.2601820354877264, "frac_reward_zero_std": 0.3, "grad_norm": 0.1054266095161438, "kl": 0.010062046919483692, "learning_rate": 4.981627424856691e-06, "loss": 0.0006, "num_tokens": 12286881.0, "reward": -0.15625, "reward_std": 0.5542315781116486, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.554231595993042, "sampling/importance_sampling_ratio/max": 1.2486117362976075, "sampling/importance_sampling_ratio/mean": 0.9996690809726715, "sampling/importance_sampling_ratio/min": 0.7404702603816986, "sampling/sampling_logp_difference/max": 0.3086616039276123, "sampling/sampling_logp_difference/mean": 0.02192587573081255, "step": 2830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 214.85, "completions/mean_terminated_length": 214.85, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.082757508009672, "epoch": 0.2611014066378597, "frac_reward_zero_std": 0.5, "grad_norm": 4.894346237182617, "kl": 0.011362863157410174, "learning_rate": 4.981319550016777e-06, "loss": 0.0004, "num_tokens": 12335125.0, "reward": -0.28125, "reward_std": 0.486899334192276, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.48689934611320496, "sampling/importance_sampling_ratio/max": 1.2341540575027465, "sampling/importance_sampling_ratio/mean": 0.9999154031276702, "sampling/importance_sampling_ratio/min": 0.7624978482723236, "sampling/sampling_logp_difference/max": 0.28746441602706907, "sampling/sampling_logp_difference/mean": 0.021222647465765476, "step": 2840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.2, "completions/max_terminated_length": 324.2, "completions/mean_length": 220.175, "completions/mean_terminated_length": 220.175, "completions/min_length": 134.7, "completions/min_terminated_length": 134.7, "entropy": 1.080984604358673, "epoch": 0.262020777787993, "frac_reward_zero_std": 0.2, "grad_norm": 6.280346393585205, "kl": 0.011071531620109454, "learning_rate": 4.9810091266627835e-06, "loss": 0.0007, "num_tokens": 12379811.0, "reward": -0.2, "reward_std": 0.7692696690559387, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.7692697167396545, "sampling/importance_sampling_ratio/max": 1.2414828896522523, "sampling/importance_sampling_ratio/mean": 0.9998758435249329, "sampling/importance_sampling_ratio/min": 0.7494953408837318, "sampling/sampling_logp_difference/max": 0.3915313184261322, "sampling/sampling_logp_difference/mean": 0.021006784588098525, "step": 2850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 218.3125, "completions/mean_terminated_length": 218.3125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0200559992343188, "epoch": 0.26294014893812634, "frac_reward_zero_std": 0.5, "grad_norm": 5.998976230621338, "kl": 0.010215351678198203, "learning_rate": 4.980696155113539e-06, "loss": 0.0004, "num_tokens": 12424564.0, "reward": -0.225, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2611293554306031, "sampling/importance_sampling_ratio/mean": 1.0002988874912262, "sampling/importance_sampling_ratio/min": 0.7499540686607361, "sampling/sampling_logp_difference/max": 0.3093577861785889, "sampling/sampling_logp_difference/mean": 0.02039481196552515, "step": 2860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 206.5125, "completions/mean_terminated_length": 206.5125, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.0704832725226878, "epoch": 0.26385952008825964, "frac_reward_zero_std": 0.6, "grad_norm": 0.18089254200458527, "kl": 0.010373964661266655, "learning_rate": 4.980380635690492e-06, "loss": 0.0003, "num_tokens": 12469229.0, "reward": 0.05, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.294747269153595, "sampling/importance_sampling_ratio/mean": 1.0003302097320557, "sampling/importance_sampling_ratio/min": 0.7580967843532562, "sampling/sampling_logp_difference/max": 0.3189679503440857, "sampling/sampling_logp_difference/mean": 0.021761940978467464, "step": 2870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 211.225, "completions/mean_terminated_length": 211.225, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0888272903859615, "epoch": 0.26477889123839293, "frac_reward_zero_std": 0.4, "grad_norm": 5.428631782531738, "kl": 0.011663316248450429, "learning_rate": 4.980062568717704e-06, "loss": 0.0007, "num_tokens": 12510727.0, "reward": 0.26875, "reward_std": 0.542016153037548, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.542016188800335, "sampling/importance_sampling_ratio/max": 1.2912502408027648, "sampling/importance_sampling_ratio/mean": 1.0002261996269226, "sampling/importance_sampling_ratio/min": 0.7573240756988525, "sampling/sampling_logp_difference/max": 0.29352726936340334, "sampling/sampling_logp_difference/mean": 0.02136547416448593, "step": 2880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.8, "completions/max_terminated_length": 297.0, "completions/mean_length": 206.875, "completions/mean_terminated_length": 203.93392944335938, "completions/min_length": 117.7, "completions/min_terminated_length": 117.7, "entropy": 1.0042827047407628, "epoch": 0.26569826238852623, "frac_reward_zero_std": 0.2, "grad_norm": 0.14426466822624207, "kl": 0.010598230280447752, "learning_rate": 4.979741954521857e-06, "loss": 0.0008, "num_tokens": 12552765.0, "reward": 0.06875, "reward_std": 0.7158065617084504, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.7158065915107727, "sampling/importance_sampling_ratio/max": 1.2832030773162841, "sampling/importance_sampling_ratio/mean": 0.9991556465625763, "sampling/importance_sampling_ratio/min": 0.7449377059936524, "sampling/sampling_logp_difference/max": 0.3168556451797485, "sampling/sampling_logp_difference/mean": 0.0210710683837533, "step": 2890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.3, "completions/max_terminated_length": 315.3, "completions/mean_length": 205.875, "completions/mean_terminated_length": 205.875, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 0.9581081978976727, "epoch": 0.2666176335386596, "frac_reward_zero_std": 0.4, "grad_norm": 6.229625701904297, "kl": 0.009211961278924718, "learning_rate": 4.979418793432248e-06, "loss": 0.0002, "num_tokens": 12597971.0, "reward": -0.175, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2463905334472656, "sampling/importance_sampling_ratio/mean": 1.00039901137352, "sampling/importance_sampling_ratio/min": 0.7871528029441833, "sampling/sampling_logp_difference/max": 0.2564482927322388, "sampling/sampling_logp_difference/mean": 0.019513036869466305, "step": 2900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.9, "completions/max_terminated_length": 323.9, "completions/mean_length": 227.275, "completions/mean_terminated_length": 227.275, "completions/min_length": 154.4, "completions/min_terminated_length": 154.4, "entropy": 1.1268527768552303, "epoch": 0.2675370046887929, "frac_reward_zero_std": 0.6, "grad_norm": 0.0948585793375969, "kl": 0.011739477032097056, "learning_rate": 4.979093085780788e-06, "loss": 0.0009, "num_tokens": 12645625.0, "reward": -0.125, "reward_std": 0.40311150550842284, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.40311154127120974, "sampling/importance_sampling_ratio/max": 1.2249012470245362, "sampling/importance_sampling_ratio/mean": 0.999481838941574, "sampling/importance_sampling_ratio/min": 0.7517392098903656, "sampling/sampling_logp_difference/max": 0.2970536768436432, "sampling/sampling_logp_difference/mean": 0.021054331958293915, "step": 2910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.0, "completions/max_terminated_length": 319.2, "completions/mean_length": 232.425, "completions/mean_terminated_length": 228.8750015258789, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.0713797256350517, "epoch": 0.2684563758389262, "frac_reward_zero_std": 0.4, "grad_norm": 6.538857936859131, "kl": 0.0109710234683007, "learning_rate": 4.978764831902005e-06, "loss": 0.0003, "num_tokens": 12691763.0, "reward": -0.18125, "reward_std": 0.43682936280965806, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.43682938665151594, "sampling/importance_sampling_ratio/max": 1.2820359945297242, "sampling/importance_sampling_ratio/mean": 0.999939215183258, "sampling/importance_sampling_ratio/min": 0.7851763844490052, "sampling/sampling_logp_difference/max": 0.2873277187347412, "sampling/sampling_logp_difference/mean": 0.020752864703536034, "step": 2920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 201.65, "completions/mean_terminated_length": 201.65, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.088754814118147, "epoch": 0.2693757469890595, "frac_reward_zero_std": 0.9, "grad_norm": 0.13672152161598206, "kl": 0.010542893002275377, "learning_rate": 4.978434032133042e-06, "loss": 0.0006, "num_tokens": 12731111.0, "reward": -0.45, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.2433626413345338, "sampling/importance_sampling_ratio/mean": 0.9999507904052735, "sampling/importance_sampling_ratio/min": 0.7971227705478668, "sampling/sampling_logp_difference/max": 0.23736850023269654, "sampling/sampling_logp_difference/mean": 0.02111559398472309, "step": 2930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.9, "completions/max_terminated_length": 299.9, "completions/mean_length": 215.575, "completions/mean_terminated_length": 215.575, "completions/min_length": 148.7, "completions/min_terminated_length": 148.7, "entropy": 1.0638048097491264, "epoch": 0.27029511813919277, "frac_reward_zero_std": 0.4, "grad_norm": 5.916285991668701, "kl": 0.011021789122605697, "learning_rate": 4.978100686813658e-06, "loss": 0.0007, "num_tokens": 12773989.0, "reward": 0.01875, "reward_std": 0.5263776004314422, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.5263776004314422, "sampling/importance_sampling_ratio/max": 1.2562369585037232, "sampling/importance_sampling_ratio/mean": 0.9999050915241241, "sampling/importance_sampling_ratio/min": 0.7789143025875092, "sampling/sampling_logp_difference/max": 0.2657563924789429, "sampling/sampling_logp_difference/mean": 0.020680586248636244, "step": 2940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.3, "completions/max_terminated_length": 327.3, "completions/mean_length": 238.025, "completions/mean_terminated_length": 238.025, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 1.1678918331861496, "epoch": 0.2712144892893261, "frac_reward_zero_std": 0.7, "grad_norm": 0.10988866537809372, "kl": 0.010816188127500936, "learning_rate": 4.977764796286224e-06, "loss": 0.0005, "num_tokens": 12820607.0, "reward": -0.40625, "reward_std": 0.30449501276016233, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.30449501276016233, "sampling/importance_sampling_ratio/max": 1.2785430431365967, "sampling/importance_sampling_ratio/mean": 1.000512808561325, "sampling/importance_sampling_ratio/min": 0.7877653241157532, "sampling/sampling_logp_difference/max": 0.2692512035369873, "sampling/sampling_logp_difference/mean": 0.021344553492963315, "step": 2950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 205.2375, "completions/mean_terminated_length": 205.2375, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0760528162121772, "epoch": 0.2721338604394594, "frac_reward_zero_std": 0.6, "grad_norm": 0.104570671916008, "kl": 0.01007548721972853, "learning_rate": 4.977426360895727e-06, "loss": 0.0008, "num_tokens": 12863474.0, "reward": -0.575, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2387065768241883, "sampling/importance_sampling_ratio/mean": 0.9999704897403717, "sampling/importance_sampling_ratio/min": 0.7906269669532776, "sampling/sampling_logp_difference/max": 0.2436307907104492, "sampling/sampling_logp_difference/mean": 0.021230327524244787, "step": 2960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.4, "completions/max_terminated_length": 326.4, "completions/mean_length": 222.3875, "completions/mean_terminated_length": 222.3875, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 0.9755946032702922, "epoch": 0.2730532315895927, "frac_reward_zero_std": 0.4, "grad_norm": 5.570831775665283, "kl": 0.009411396941868589, "learning_rate": 4.977085380989768e-06, "loss": 0.0003, "num_tokens": 12906905.0, "reward": 0.15, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2805304050445556, "sampling/importance_sampling_ratio/mean": 1.000033438205719, "sampling/importance_sampling_ratio/min": 0.7765223205089569, "sampling/sampling_logp_difference/max": 0.27893441915512085, "sampling/sampling_logp_difference/mean": 0.01940320134162903, "step": 2970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 210.3125, "completions/mean_terminated_length": 210.3125, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0299917601048947, "epoch": 0.273972602739726, "frac_reward_zero_std": 0.4, "grad_norm": 7.4183430671691895, "kl": 0.00915182302123867, "learning_rate": 4.9767418569185585e-06, "loss": 0.0001, "num_tokens": 12948418.0, "reward": -0.35, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.3150875329971314, "sampling/importance_sampling_ratio/mean": 1.0003524005413056, "sampling/importance_sampling_ratio/min": 0.7881732285022736, "sampling/sampling_logp_difference/max": 0.28403730392456056, "sampling/sampling_logp_difference/mean": 0.020351105928421022, "step": 2980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.6, "completions/max_terminated_length": 323.6, "completions/mean_length": 221.0875, "completions/mean_terminated_length": 221.0875, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0554384544491768, "epoch": 0.2748919738898593, "frac_reward_zero_std": 0.6, "grad_norm": 0.1127392128109932, "kl": 0.010386332881171256, "learning_rate": 4.9763957890349284e-06, "loss": 0.0006, "num_tokens": 12993977.0, "reward": 0.225, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2369151353836059, "sampling/importance_sampling_ratio/mean": 0.9995321989059448, "sampling/importance_sampling_ratio/min": 0.8009593427181244, "sampling/sampling_logp_difference/max": 0.23444077968597413, "sampling/sampling_logp_difference/mean": 0.020738713815808295, "step": 2990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.7, "completions/max_terminated_length": 287.7, "completions/mean_length": 199.9875, "completions/mean_terminated_length": 199.9875, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.051791501790285, "epoch": 0.27581134503999266, "frac_reward_zero_std": 0.4, "grad_norm": 0.08508328348398209, "kl": 0.010128949990030379, "learning_rate": 4.976047177694315e-06, "loss": 0.001, "num_tokens": 13038440.0, "reward": -0.05, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.2392753124237061, "sampling/importance_sampling_ratio/mean": 0.9997134566307068, "sampling/importance_sampling_ratio/min": 0.7880199790000916, "sampling/sampling_logp_difference/max": 0.25580153465270994, "sampling/sampling_logp_difference/mean": 0.020783905312418936, "step": 3000 }, { "epoch": 0.27581134503999266, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 288.39441535776615, "eval_completions/max_terminated_length": 287.09424083769636, "eval_completions/mean_length": 201.3778359511344, "eval_completions/mean_terminated_length": 201.0552856019238, "eval_completions/min_length": 129.19022687609075, "eval_completions/min_terminated_length": 129.19022687609075, "eval_entropy": 1.056019762163595, "eval_frac_reward_zero_std": 0.450261780104712, "eval_kl": 0.011142948977194115, "eval_loss": 0.0005557667464017868, "eval_num_tokens": 13038440.0, "eval_reward": -0.12663612565445026, "eval_reward_std": 0.4793417890398498, "eval_rewards/reward_fn/mean": -0.12663612565445026, "eval_rewards/reward_fn/std": 0.4793418051632703, "eval_runtime": 1684.818, "eval_samples_per_second": 0.34, "eval_sampling/importance_sampling_ratio/max": 1.2766486258614957, "eval_sampling/importance_sampling_ratio/mean": 0.9999948123154632, "eval_sampling/importance_sampling_ratio/min": 0.7757992508835817, "eval_sampling/sampling_logp_difference/max": 0.2880360097161138, "eval_sampling/sampling_logp_difference/mean": 0.02089104160380405, "eval_steps_per_second": 0.043, "step": 3000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 196.425, "completions/mean_terminated_length": 196.425, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.0188458539545535, "epoch": 0.27673071619012596, "frac_reward_zero_std": 0.4, "grad_norm": 5.345553398132324, "kl": 0.011334045563125983, "learning_rate": 4.975696023254771e-06, "loss": 0.0008, "num_tokens": 13076994.0, "reward": -0.025, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2262628316879272, "sampling/importance_sampling_ratio/mean": 0.999360191822052, "sampling/importance_sampling_ratio/min": 0.7642592847347259, "sampling/sampling_logp_difference/max": 0.28010780811309816, "sampling/sampling_logp_difference/mean": 0.02115575075149536, "step": 3010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.8, "completions/max_terminated_length": 302.8, "completions/mean_length": 217.8125, "completions/mean_terminated_length": 217.8125, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.062905741482973, "epoch": 0.27765008734025926, "frac_reward_zero_std": 0.5, "grad_norm": 4.948235988616943, "kl": 0.012143132509663701, "learning_rate": 4.97534232607696e-06, "loss": 0.0005, "num_tokens": 13120731.0, "reward": -0.35625, "reward_std": 0.402328197658062, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.402328197658062, "sampling/importance_sampling_ratio/max": 1.2337971806526185, "sampling/importance_sampling_ratio/mean": 0.9998822271823883, "sampling/importance_sampling_ratio/min": 0.7904243528842926, "sampling/sampling_logp_difference/max": 0.24389398097991943, "sampling/sampling_logp_difference/mean": 0.020279762614518405, "step": 3020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.3, "completions/max_terminated_length": 297.3, "completions/mean_length": 207.2, "completions/mean_terminated_length": 207.2, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.0539379820227623, "epoch": 0.27856945849039255, "frac_reward_zero_std": 0.6, "grad_norm": 0.12932275235652924, "kl": 0.012297289568232372, "learning_rate": 4.974986086524156e-06, "loss": 0.0001, "num_tokens": 13162587.0, "reward": -0.175, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.3197848439216613, "sampling/importance_sampling_ratio/mean": 1.0001338601112366, "sampling/importance_sampling_ratio/min": 0.7521610021591186, "sampling/sampling_logp_difference/max": 0.3393049716949463, "sampling/sampling_logp_difference/mean": 0.0205106008797884, "step": 3030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.4, "completions/max_terminated_length": 316.4, "completions/mean_length": 216.85, "completions/mean_terminated_length": 216.85, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.0526421062648297, "epoch": 0.2794888296405259, "frac_reward_zero_std": 0.5, "grad_norm": 5.728741645812988, "kl": 0.011953950283350423, "learning_rate": 4.974627304962247e-06, "loss": 0.001, "num_tokens": 13206303.0, "reward": 0.225, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2739320993423462, "sampling/importance_sampling_ratio/mean": 1.0002655267715455, "sampling/importance_sampling_ratio/min": 0.7645920276641845, "sampling/sampling_logp_difference/max": 0.2968616008758545, "sampling/sampling_logp_difference/mean": 0.020452052168548108, "step": 3040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 217.0625, "completions/mean_terminated_length": 217.0625, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.053018044680357, "epoch": 0.2804082007906592, "frac_reward_zero_std": 0.4, "grad_norm": 5.266042232513428, "kl": 0.01012190593755804, "learning_rate": 4.97426598175973e-06, "loss": 0.0007, "num_tokens": 13250036.0, "reward": -0.38125, "reward_std": 0.5919759094715118, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.5919759333133697, "sampling/importance_sampling_ratio/max": 1.2971499681472778, "sampling/importance_sampling_ratio/mean": 1.0000174522399903, "sampling/importance_sampling_ratio/min": 0.7311173707246781, "sampling/sampling_logp_difference/max": 0.38111563920974734, "sampling/sampling_logp_difference/mean": 0.021086253970861436, "step": 3050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.5, "completions/max_terminated_length": 332.5, "completions/mean_length": 220.65, "completions/mean_terminated_length": 220.65, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0620789401233197, "epoch": 0.2813275719407925, "frac_reward_zero_std": 0.6, "grad_norm": 5.832259654998779, "kl": 0.012095266836695373, "learning_rate": 4.973902117287714e-06, "loss": 0.0005, "num_tokens": 13296120.0, "reward": -0.575, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.350371742248535, "sampling/importance_sampling_ratio/mean": 1.0000187754631042, "sampling/importance_sampling_ratio/min": 0.783443135023117, "sampling/sampling_logp_difference/max": 0.30533404350280763, "sampling/sampling_logp_difference/mean": 0.02092981543391943, "step": 3060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 202.0625, "completions/mean_terminated_length": 202.0625, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.066781011596322, "epoch": 0.2822469430909258, "frac_reward_zero_std": 0.8, "grad_norm": 6.490849018096924, "kl": 0.012840493186376988, "learning_rate": 4.9735357119199154e-06, "loss": 0.0006, "num_tokens": 13339957.0, "reward": 0.19375, "reward_std": 0.15909902453422547, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.15909903049468993, "sampling/importance_sampling_ratio/max": 1.2710317373275757, "sampling/importance_sampling_ratio/mean": 1.0000608384609222, "sampling/importance_sampling_ratio/min": 0.7919386208057404, "sampling/sampling_logp_difference/max": 0.25711435079574585, "sampling/sampling_logp_difference/mean": 0.020889974758028985, "step": 3070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.4, "completions/max_terminated_length": 265.4, "completions/mean_length": 195.2, "completions/mean_terminated_length": 195.2, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.0795501686632634, "epoch": 0.2831663142410591, "frac_reward_zero_std": 0.5, "grad_norm": 0.0838383212685585, "kl": 0.011272014968562871, "learning_rate": 4.9731667660326625e-06, "loss": 0.0004, "num_tokens": 13380525.0, "reward": -0.125, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2417298793792724, "sampling/importance_sampling_ratio/mean": 1.0002350807189941, "sampling/importance_sampling_ratio/min": 0.785903787612915, "sampling/sampling_logp_difference/max": 0.2601369619369507, "sampling/sampling_logp_difference/mean": 0.02098796647042036, "step": 3080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.4, "completions/max_terminated_length": 273.4, "completions/mean_length": 196.8375, "completions/mean_terminated_length": 196.8375, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0344906814396382, "epoch": 0.28408568539119244, "frac_reward_zero_std": 0.4, "grad_norm": 5.066373825073242, "kl": 0.011268875800305977, "learning_rate": 4.972795280004893e-06, "loss": 0.0006, "num_tokens": 13423544.0, "reward": -0.15, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.3177303194999694, "sampling/importance_sampling_ratio/mean": 1.0001079857349395, "sampling/importance_sampling_ratio/min": 0.808626937866211, "sampling/sampling_logp_difference/max": 0.27572024166584014, "sampling/sampling_logp_difference/mean": 0.02037345850840211, "step": 3090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.1, "completions/max_terminated_length": 300.1, "completions/mean_length": 218.1875, "completions/mean_terminated_length": 218.1875, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 1.182720448076725, "epoch": 0.28500505654132574, "frac_reward_zero_std": 0.3, "grad_norm": 5.720422744750977, "kl": 0.013414640055270866, "learning_rate": 4.972421254218151e-06, "loss": 0.0005, "num_tokens": 13466975.0, "reward": -0.23125, "reward_std": 0.5294113740324974, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5294113978743553, "sampling/importance_sampling_ratio/max": 1.358827817440033, "sampling/importance_sampling_ratio/mean": 0.9995778977870942, "sampling/importance_sampling_ratio/min": 0.7840069174766541, "sampling/sampling_logp_difference/max": 0.31704181432724, "sampling/sampling_logp_difference/mean": 0.022340040653944016, "step": 3100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 199.35, "completions/mean_terminated_length": 199.35, "completions/min_length": 112.6, "completions/min_terminated_length": 112.6, "entropy": 1.0489048443734645, "epoch": 0.28592442769145904, "frac_reward_zero_std": 0.3, "grad_norm": 5.59348726272583, "kl": 0.011125909688416868, "learning_rate": 4.972044689056593e-06, "loss": 0.0004, "num_tokens": 13511267.0, "reward": 0.225, "reward_std": 0.5967825591564179, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5967825591564179, "sampling/importance_sampling_ratio/max": 1.2390315890312196, "sampling/importance_sampling_ratio/mean": 1.0006732523441315, "sampling/importance_sampling_ratio/min": 0.8011610329151153, "sampling/sampling_logp_difference/max": 0.2395630180835724, "sampling/sampling_logp_difference/mean": 0.020879417657852173, "step": 3110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 204.7, "completions/mean_terminated_length": 204.7, "completions/min_length": 124.7, "completions/min_terminated_length": 124.7, "entropy": 1.004054180532694, "epoch": 0.28684379884159233, "frac_reward_zero_std": 0.6, "grad_norm": 0.19764213263988495, "kl": 0.01198892883840017, "learning_rate": 4.97166558490698e-06, "loss": 0.0008, "num_tokens": 13553171.0, "reward": -0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.265267276763916, "sampling/importance_sampling_ratio/mean": 1.0000044226646423, "sampling/importance_sampling_ratio/min": 0.7602647244930267, "sampling/sampling_logp_difference/max": 0.3025314211845398, "sampling/sampling_logp_difference/mean": 0.019929283298552036, "step": 3120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.2, "completions/max_terminated_length": 288.2, "completions/mean_length": 191.5375, "completions/mean_terminated_length": 191.5375, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 0.9890505358576774, "epoch": 0.2877631699917257, "frac_reward_zero_std": 0.6, "grad_norm": 0.11215851455926895, "kl": 0.01097930904943496, "learning_rate": 4.971283942158684e-06, "loss": 0.0003, "num_tokens": 13594078.0, "reward": -0.225, "reward_std": 0.41743398904800416, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.417434024810791, "sampling/importance_sampling_ratio/max": 1.297709858417511, "sampling/importance_sampling_ratio/mean": 0.9999955058097839, "sampling/importance_sampling_ratio/min": 0.7713634729385376, "sampling/sampling_logp_difference/max": 0.29228028655052185, "sampling/sampling_logp_difference/mean": 0.019664808921515942, "step": 3130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.8, "completions/max_terminated_length": 285.8, "completions/mean_length": 204.925, "completions/mean_terminated_length": 204.925, "completions/min_length": 134.1, "completions/min_terminated_length": 134.1, "entropy": 1.0525304466485976, "epoch": 0.288682541141859, "frac_reward_zero_std": 0.4, "grad_norm": 0.11893165111541748, "kl": 0.01006021184148267, "learning_rate": 4.9708997612036805e-06, "loss": 0.0007, "num_tokens": 13639688.0, "reward": -0.125, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2871218919754028, "sampling/importance_sampling_ratio/mean": 0.9999821603298187, "sampling/importance_sampling_ratio/min": 0.7891266644001007, "sampling/sampling_logp_difference/max": 0.2773348867893219, "sampling/sampling_logp_difference/mean": 0.02096028160303831, "step": 3140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 198.3875, "completions/mean_terminated_length": 198.3875, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.024985944852233, "epoch": 0.2896019122919923, "frac_reward_zero_std": 0.6, "grad_norm": 0.13026563823223114, "kl": 0.010901391744846478, "learning_rate": 4.970513042436556e-06, "loss": 0.0004, "num_tokens": 13681663.0, "reward": 0.35, "reward_std": 0.3846505284309387, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.3846505284309387, "sampling/importance_sampling_ratio/max": 1.3337575554847718, "sampling/importance_sampling_ratio/mean": 0.9994821071624755, "sampling/importance_sampling_ratio/min": 0.7529596269130707, "sampling/sampling_logp_difference/max": 0.35747418403625486, "sampling/sampling_logp_difference/mean": 0.02126753982156515, "step": 3150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.6, "completions/max_terminated_length": 321.6, "completions/mean_length": 218.7375, "completions/mean_terminated_length": 218.7375, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.0834360912442207, "epoch": 0.2905212834421256, "frac_reward_zero_std": 0.5, "grad_norm": 4.169060230255127, "kl": 0.01249639653833583, "learning_rate": 4.970123786254502e-06, "loss": 0.0006, "num_tokens": 13725554.0, "reward": 0.0, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2687109231948852, "sampling/importance_sampling_ratio/mean": 1.0003963708877563, "sampling/importance_sampling_ratio/min": 0.7492340385913849, "sampling/sampling_logp_difference/max": 0.3026998400688171, "sampling/sampling_logp_difference/mean": 0.020945898629724978, "step": 3160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.3, "completions/max_terminated_length": 279.3, "completions/mean_length": 209.9875, "completions/mean_terminated_length": 209.9875, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.0871562115848064, "epoch": 0.2914406545922589, "frac_reward_zero_std": 0.1, "grad_norm": 5.530633926391602, "kl": 0.011550526757491752, "learning_rate": 4.969731993057313e-06, "loss": 0.0005, "num_tokens": 13772865.0, "reward": 0.1, "reward_std": 0.7491317331790924, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.7491317451000213, "sampling/importance_sampling_ratio/max": 1.3216771721839904, "sampling/importance_sampling_ratio/mean": 0.9997056603431702, "sampling/importance_sampling_ratio/min": 0.7589674085378647, "sampling/sampling_logp_difference/max": 0.32447296380996704, "sampling/sampling_logp_difference/mean": 0.021278670616447927, "step": 3170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.8, "completions/max_terminated_length": 252.8, "completions/mean_length": 188.0, "completions/mean_terminated_length": 188.0, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.0557625055313111, "epoch": 0.2923600257423922, "frac_reward_zero_std": 0.3, "grad_norm": 6.271437168121338, "kl": 0.010816453362349421, "learning_rate": 4.969337663247395e-06, "loss": 0.0003, "num_tokens": 13816929.0, "reward": 0.05, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2486080408096314, "sampling/importance_sampling_ratio/mean": 1.000100553035736, "sampling/importance_sampling_ratio/min": 0.7621223628520966, "sampling/sampling_logp_difference/max": 0.2772038221359253, "sampling/sampling_logp_difference/mean": 0.021052446588873862, "step": 3180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.9, "completions/max_terminated_length": 321.9, "completions/mean_length": 220.55, "completions/mean_terminated_length": 220.55, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0706253074109555, "epoch": 0.2932793968925255, "frac_reward_zero_std": 0.4, "grad_norm": 5.764314651489258, "kl": 0.011181587778264657, "learning_rate": 4.968940797229756e-06, "loss": 0.0006, "num_tokens": 13862053.0, "reward": -0.275, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.3323882579803468, "sampling/importance_sampling_ratio/mean": 0.999704760313034, "sampling/importance_sampling_ratio/min": 0.7645407855510712, "sampling/sampling_logp_difference/max": 0.3144256234169006, "sampling/sampling_logp_difference/mean": 0.02126467041671276, "step": 3190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.6, "completions/max_terminated_length": 276.6, "completions/mean_length": 204.35, "completions/mean_terminated_length": 204.35, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.103306395560503, "epoch": 0.2941987680426588, "frac_reward_zero_std": 0.3, "grad_norm": 0.10013232380151749, "kl": 0.011012861737981439, "learning_rate": 4.9685413954120064e-06, "loss": 0.0006, "num_tokens": 13907401.0, "reward": -0.25625, "reward_std": 0.5542315781116486, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.554231595993042, "sampling/importance_sampling_ratio/max": 1.2490511775016784, "sampling/importance_sampling_ratio/mean": 1.0001957416534424, "sampling/importance_sampling_ratio/min": 0.7905869722366333, "sampling/sampling_logp_difference/max": 0.24698617458343505, "sampling/sampling_logp_difference/mean": 0.021341325342655183, "step": 3200 }, { "epoch": 0.2941987680426588, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 305.4450261780105, "eval_completions/max_terminated_length": 301.69458987783594, "eval_completions/mean_length": 212.63176265270505, "eval_completions/mean_terminated_length": 211.9231490955719, "eval_completions/min_length": 139.91448516579408, "eval_completions/min_terminated_length": 139.91448516579408, "eval_entropy": 1.07979557360238, "eval_frac_reward_zero_std": 0.462478184991274, "eval_kl": 0.011909419512229909, "eval_loss": 0.000602230429649353, "eval_num_tokens": 13907401.0, "eval_reward": -0.1087478184991274, "eval_reward_std": 0.4743207133981362, "eval_rewards/reward_fn/mean": -0.1087478184991274, "eval_rewards/reward_fn/std": 0.47432072900144634, "eval_runtime": 1772.7705, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2702920243378084, "eval_sampling/importance_sampling_ratio/mean": 1.0000068790833572, "eval_sampling/importance_sampling_ratio/min": 0.7663298889760571, "eval_sampling/sampling_logp_difference/max": 0.29583951445982304, "eval_sampling/sampling_logp_difference/mean": 0.020718347427498607, "eval_steps_per_second": 0.041, "step": 3200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.1, "completions/max_terminated_length": 278.1, "completions/mean_length": 199.225, "completions/mean_terminated_length": 199.225, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.0713184885680676, "epoch": 0.2951181391927921, "frac_reward_zero_std": 0.3, "grad_norm": 0.13751593232154846, "kl": 0.011631828220561146, "learning_rate": 4.9681394582043676e-06, "loss": 0.0003, "num_tokens": 13949163.0, "reward": -0.225, "reward_std": 0.5933722019195556, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5933722376823425, "sampling/importance_sampling_ratio/max": 1.2478226900100708, "sampling/importance_sampling_ratio/mean": 1.000209814310074, "sampling/importance_sampling_ratio/min": 0.7491489887237549, "sampling/sampling_logp_difference/max": 0.2999851703643799, "sampling/sampling_logp_difference/mean": 0.021413533575832845, "step": 3210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 224.6375, "completions/mean_terminated_length": 224.6375, "completions/min_length": 163.9, "completions/min_terminated_length": 163.9, "entropy": 1.0797304689884186, "epoch": 0.2960375103429254, "frac_reward_zero_std": 0.6, "grad_norm": 6.091808319091797, "kl": 0.012616083171451464, "learning_rate": 4.967734986019659e-06, "loss": 0.0005, "num_tokens": 13994278.0, "reward": -0.05, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2721864223480224, "sampling/importance_sampling_ratio/mean": 0.9996244847774506, "sampling/importance_sampling_ratio/min": 0.7474463820457459, "sampling/sampling_logp_difference/max": 0.3084326505661011, "sampling/sampling_logp_difference/mean": 0.021365855261683463, "step": 3220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.4, "completions/max_terminated_length": 260.4, "completions/mean_length": 193.1875, "completions/mean_terminated_length": 193.1875, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.1218849606812, "epoch": 0.29695688149305877, "frac_reward_zero_std": 0.5, "grad_norm": 5.689598083496094, "kl": 0.012880048871738835, "learning_rate": 4.967327979273307e-06, "loss": 0.0005, "num_tokens": 14038517.0, "reward": 0.0, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2504512786865234, "sampling/importance_sampling_ratio/mean": 1.0000479221343994, "sampling/importance_sampling_ratio/min": 0.7659060478210449, "sampling/sampling_logp_difference/max": 0.27408944070339203, "sampling/sampling_logp_difference/mean": 0.022167630679905414, "step": 3230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 198.0125, "completions/mean_terminated_length": 198.0125, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0333550348877907, "epoch": 0.29787625264319206, "frac_reward_zero_std": 0.6, "grad_norm": 0.18589849770069122, "kl": 0.011907252366654575, "learning_rate": 4.966918438383338e-06, "loss": 0.0006, "num_tokens": 14078094.0, "reward": -0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.4140948891639709, "sampling/importance_sampling_ratio/mean": 1.0004902780056, "sampling/importance_sampling_ratio/min": 0.7780568242073059, "sampling/sampling_logp_difference/max": 0.3804423987865448, "sampling/sampling_logp_difference/mean": 0.020931327342987062, "step": 3240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.0, "completions/max_terminated_length": 309.0, "completions/mean_length": 223.4375, "completions/mean_terminated_length": 223.4375, "completions/min_length": 153.4, "completions/min_terminated_length": 153.4, "entropy": 1.0211482021957636, "epoch": 0.29879562379332536, "frac_reward_zero_std": 0.4, "grad_norm": 0.1653199940919876, "kl": 0.012070708285318688, "learning_rate": 4.966506363770387e-06, "loss": 0.0005, "num_tokens": 14120737.0, "reward": 0.1375, "reward_std": 0.3619407147169113, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.3619407147169113, "sampling/importance_sampling_ratio/max": 1.2491608142852784, "sampling/importance_sampling_ratio/mean": 1.0000177800655365, "sampling/importance_sampling_ratio/min": 0.8093985974788666, "sampling/sampling_logp_difference/max": 0.24325743913650513, "sampling/sampling_logp_difference/mean": 0.01996495109051466, "step": 3250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.4, "completions/max_terminated_length": 318.4, "completions/mean_length": 222.6, "completions/mean_terminated_length": 222.6, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.0609354861080647, "epoch": 0.29971499494345866, "frac_reward_zero_std": 0.5, "grad_norm": 0.15014250576496124, "kl": 0.011479726998368278, "learning_rate": 4.966091755857685e-06, "loss": 0.0004, "num_tokens": 14165697.0, "reward": -0.53125, "reward_std": 0.4162048876285553, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.4162048935890198, "sampling/importance_sampling_ratio/max": 1.3457364797592164, "sampling/importance_sampling_ratio/mean": 0.999638843536377, "sampling/importance_sampling_ratio/min": 0.7873004019260407, "sampling/sampling_logp_difference/max": 0.3175753951072693, "sampling/sampling_logp_difference/mean": 0.020647377148270606, "step": 3260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 387.7, "completions/max_terminated_length": 387.2, "completions/mean_length": 267.0125, "completions/mean_terminated_length": 263.87083435058594, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 1.0616928614675998, "epoch": 0.300634366093592, "frac_reward_zero_std": 0.6, "grad_norm": 6.785939693450928, "kl": 0.010851484577869996, "learning_rate": 4.965674615071069e-06, "loss": 0.0005, "num_tokens": 14214866.0, "reward": -0.55625, "reward_std": 0.3925050973892212, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.3925050973892212, "sampling/importance_sampling_ratio/max": 1.288062047958374, "sampling/importance_sampling_ratio/mean": 0.9994681894779205, "sampling/importance_sampling_ratio/min": 0.7548766136169434, "sampling/sampling_logp_difference/max": 0.32626240253448485, "sampling/sampling_logp_difference/mean": 0.02010455261915922, "step": 3270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 201.0875, "completions/mean_terminated_length": 201.0875, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.109372877329588, "epoch": 0.3015537372437253, "frac_reward_zero_std": 0.6, "grad_norm": 0.20438767969608307, "kl": 0.012639928690623492, "learning_rate": 4.965254941838974e-06, "loss": 0.0006, "num_tokens": 14256425.0, "reward": -0.45, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2706104636192321, "sampling/importance_sampling_ratio/mean": 0.9992230057716369, "sampling/importance_sampling_ratio/min": 0.7576861023902893, "sampling/sampling_logp_difference/max": 0.2993146538734436, "sampling/sampling_logp_difference/mean": 0.02183995395898819, "step": 3280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.1, "completions/max_terminated_length": 268.1, "completions/mean_length": 203.425, "completions/mean_terminated_length": 203.425, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.0981516316533089, "epoch": 0.3024731083938586, "frac_reward_zero_std": 0.4, "grad_norm": 0.230949267745018, "kl": 0.012444810644956306, "learning_rate": 4.964832736592441e-06, "loss": 0.0003, "num_tokens": 14303355.0, "reward": -0.15, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.341007161140442, "sampling/importance_sampling_ratio/mean": 1.0005682826042175, "sampling/importance_sampling_ratio/min": 0.739646977186203, "sampling/sampling_logp_difference/max": 0.36877630949020385, "sampling/sampling_logp_difference/mean": 0.02188901137560606, "step": 3290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.1, "completions/max_terminated_length": 285.1, "completions/mean_length": 189.7, "completions/mean_terminated_length": 189.7, "completions/min_length": 110.3, "completions/min_terminated_length": 110.3, "entropy": 1.0456227704882621, "epoch": 0.3033924795439919, "frac_reward_zero_std": 0.2, "grad_norm": 5.891936302185059, "kl": 0.011145039735129102, "learning_rate": 4.964407999765105e-06, "loss": 0.0001, "num_tokens": 14341107.0, "reward": 0.125, "reward_std": 0.664098572731018, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.664098584651947, "sampling/importance_sampling_ratio/max": 1.247208344936371, "sampling/importance_sampling_ratio/mean": 0.9998607754707336, "sampling/importance_sampling_ratio/min": 0.7931545674800873, "sampling/sampling_logp_difference/max": 0.25273489356040957, "sampling/sampling_logp_difference/mean": 0.021050767786800862, "step": 3300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.5, "completions/max_terminated_length": 260.5, "completions/mean_length": 196.45, "completions/mean_terminated_length": 196.45, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.055165495723486, "epoch": 0.3043118506941252, "frac_reward_zero_std": 0.7, "grad_norm": 5.946415424346924, "kl": 0.011382727598538622, "learning_rate": 4.963980731793208e-06, "loss": 0.0006, "num_tokens": 14381735.0, "reward": -0.05, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2715867161750793, "sampling/importance_sampling_ratio/mean": 0.9999899625778198, "sampling/importance_sampling_ratio/min": 0.7625855326652526, "sampling/sampling_logp_difference/max": 0.29745934009552, "sampling/sampling_logp_difference/mean": 0.020833195373415946, "step": 3310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.2, "completions/max_terminated_length": 255.2, "completions/mean_length": 194.3125, "completions/mean_terminated_length": 194.3125, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0850521691143513, "epoch": 0.30523122184425855, "frac_reward_zero_std": 0.4, "grad_norm": 6.563942909240723, "kl": 0.01240106942714192, "learning_rate": 4.963550933115587e-06, "loss": 0.0007, "num_tokens": 14425592.0, "reward": 0.15, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.4110400676727295, "sampling/importance_sampling_ratio/mean": 0.9998840630054474, "sampling/importance_sampling_ratio/min": 0.7839056491851807, "sampling/sampling_logp_difference/max": 0.3392579674720764, "sampling/sampling_logp_difference/mean": 0.021363977529108524, "step": 3320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.3, "completions/max_terminated_length": 315.3, "completions/mean_length": 209.5125, "completions/mean_terminated_length": 209.5125, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.1237838573753833, "epoch": 0.30615059299439185, "frac_reward_zero_std": 0.5, "grad_norm": 0.12437228858470917, "kl": 0.013775202562101185, "learning_rate": 4.96311860417368e-06, "loss": 0.0006, "num_tokens": 14469673.0, "reward": -0.35, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2817110896110535, "sampling/importance_sampling_ratio/mean": 1.0003365218639373, "sampling/importance_sampling_ratio/min": 0.7969170153141022, "sampling/sampling_logp_difference/max": 0.25361756682395936, "sampling/sampling_logp_difference/mean": 0.02157805971801281, "step": 3330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 300.6, "completions/max_terminated_length": 293.7, "completions/mean_length": 215.8375, "completions/mean_terminated_length": 213.20357360839844, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 0.9898960784077644, "epoch": 0.30706996414452514, "frac_reward_zero_std": 0.4, "grad_norm": 5.734639644622803, "kl": 0.011176029316266068, "learning_rate": 4.962683745411524e-06, "loss": 0.0005, "num_tokens": 14513188.0, "reward": -0.55625, "reward_std": 0.5381913900375366, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.538191419839859, "sampling/importance_sampling_ratio/max": 1.29407856464386, "sampling/importance_sampling_ratio/mean": 1.000000524520874, "sampling/importance_sampling_ratio/min": 0.7596995413303376, "sampling/sampling_logp_difference/max": 0.3033494710922241, "sampling/sampling_logp_difference/mean": 0.019634569808840752, "step": 3340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.5, "completions/max_terminated_length": 292.5, "completions/mean_length": 207.0625, "completions/mean_terminated_length": 207.0625, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.1229946061968803, "epoch": 0.30798933529465844, "frac_reward_zero_std": 0.7, "grad_norm": 7.5488176345825195, "kl": 0.01192045393981971, "learning_rate": 4.9622463572757525e-06, "loss": 0.0008, "num_tokens": 14557913.0, "reward": -0.65, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2484214425086975, "sampling/importance_sampling_ratio/mean": 0.9998448669910431, "sampling/importance_sampling_ratio/min": 0.7816978693008423, "sampling/sampling_logp_difference/max": 0.25361243486404417, "sampling/sampling_logp_difference/mean": 0.021760937571525574, "step": 3350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.9, "completions/max_terminated_length": 278.9, "completions/mean_length": 200.4375, "completions/mean_terminated_length": 200.4375, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.0253915898501873, "epoch": 0.30890870644479174, "frac_reward_zero_std": 0.4, "grad_norm": 0.12784092128276825, "kl": 0.010556973336497322, "learning_rate": 4.961806440215599e-06, "loss": 0.0005, "num_tokens": 14602460.0, "reward": 0.075, "reward_std": 0.5773320138454437, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5773320615291595, "sampling/importance_sampling_ratio/max": 1.223389208316803, "sampling/importance_sampling_ratio/mean": 0.9997777163982391, "sampling/importance_sampling_ratio/min": 0.7466545283794404, "sampling/sampling_logp_difference/max": 0.3090578556060791, "sampling/sampling_logp_difference/mean": 0.02027799729257822, "step": 3360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.8, "completions/max_terminated_length": 298.8, "completions/mean_length": 226.9125, "completions/mean_terminated_length": 226.9125, "completions/min_length": 162.5, "completions/min_terminated_length": 162.5, "entropy": 1.0943810299038887, "epoch": 0.3098280775949251, "frac_reward_zero_std": 0.5, "grad_norm": 5.9767985343933105, "kl": 0.010377160669304431, "learning_rate": 4.9613639946828925e-06, "loss": 0.0002, "num_tokens": 14647333.0, "reward": -0.25, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.2944650411605836, "sampling/importance_sampling_ratio/mean": 0.9998953938484192, "sampling/importance_sampling_ratio/min": 0.785211443901062, "sampling/sampling_logp_difference/max": 0.270890998840332, "sampling/sampling_logp_difference/mean": 0.021239372715353966, "step": 3370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.0, "completions/max_terminated_length": 322.0, "completions/mean_length": 208.5875, "completions/mean_terminated_length": 208.5875, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 0.9948768369853497, "epoch": 0.3107474487450584, "frac_reward_zero_std": 0.1, "grad_norm": 5.086149215698242, "kl": 0.010246444132644684, "learning_rate": 4.960919021132062e-06, "loss": 0.0005, "num_tokens": 14688356.0, "reward": 0.175, "reward_std": 0.7019944012165069, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.7019944250583648, "sampling/importance_sampling_ratio/max": 1.2642218947410584, "sampling/importance_sampling_ratio/mean": 0.9998965084552764, "sampling/importance_sampling_ratio/min": 0.7544682919979095, "sampling/sampling_logp_difference/max": 0.311410927772522, "sampling/sampling_logp_difference/mean": 0.020227492973208427, "step": 3380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.8, "completions/max_terminated_length": 319.8, "completions/mean_length": 226.9375, "completions/mean_terminated_length": 226.9375, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.005763664841652, "epoch": 0.3116668198951917, "frac_reward_zero_std": 0.3, "grad_norm": 6.581082344055176, "kl": 0.01019047963200137, "learning_rate": 4.96047152002013e-06, "loss": 0.0006, "num_tokens": 14735695.0, "reward": -0.025, "reward_std": 0.6152592301368713, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6152592420578002, "sampling/importance_sampling_ratio/max": 1.2259103178977966, "sampling/importance_sampling_ratio/mean": 0.9999107420444489, "sampling/importance_sampling_ratio/min": 0.7746140420436859, "sampling/sampling_logp_difference/max": 0.2605891853570938, "sampling/sampling_logp_difference/mean": 0.020189375057816504, "step": 3390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.3, "completions/max_terminated_length": 266.3, "completions/mean_length": 202.4375, "completions/mean_terminated_length": 202.4375, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 0.9857037253677845, "epoch": 0.312586191045325, "frac_reward_zero_std": 0.2, "grad_norm": 5.546913146972656, "kl": 0.01238949082326144, "learning_rate": 4.960021491806716e-06, "loss": 0.0007, "num_tokens": 14776378.0, "reward": -0.575, "reward_std": 0.6312994182109832, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.6312994182109832, "sampling/importance_sampling_ratio/max": 1.2476478695869446, "sampling/importance_sampling_ratio/mean": 0.999132114648819, "sampling/importance_sampling_ratio/min": 0.7598730385303497, "sampling/sampling_logp_difference/max": 0.2915363788604736, "sampling/sampling_logp_difference/mean": 0.019973271153867246, "step": 3400 }, { "epoch": 0.312586191045325, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 305.70680628272254, "eval_completions/max_terminated_length": 303.16230366492147, "eval_completions/mean_length": 214.95789703315882, "eval_completions/mean_terminated_length": 214.37693248702178, "eval_completions/min_length": 142.8848167539267, "eval_completions/min_terminated_length": 142.8848167539267, "eval_entropy": 1.046874632542046, "eval_frac_reward_zero_std": 0.4607329842931937, "eval_kl": 0.011844595803198183, "eval_loss": 0.0006042275344952941, "eval_num_tokens": 14776378.0, "eval_reward": -0.12467277486910995, "eval_reward_std": 0.4662950683066566, "eval_rewards/reward_fn/mean": -0.12467277486910995, "eval_rewards/reward_fn/std": 0.4662950824536578, "eval_runtime": 1774.9477, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2730144054269708, "eval_sampling/importance_sampling_ratio/mean": 1.000060549373194, "eval_sampling/importance_sampling_ratio/min": 0.7696107243160095, "eval_sampling/sampling_logp_difference/max": 0.29194521929997214, "eval_sampling/sampling_logp_difference/mean": 0.020687427207580517, "eval_steps_per_second": 0.041, "step": 3400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 216.1375, "completions/mean_terminated_length": 216.1375, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 1.104987846314907, "epoch": 0.31350556219545833, "frac_reward_zero_std": 0.4, "grad_norm": 0.19174346327781677, "kl": 0.014094382262555882, "learning_rate": 4.959568936954036e-06, "loss": 0.0007, "num_tokens": 14820437.0, "reward": -0.475, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2874980568885803, "sampling/importance_sampling_ratio/mean": 1.0001987397670746, "sampling/importance_sampling_ratio/min": 0.7802516996860505, "sampling/sampling_logp_difference/max": 0.2907045781612396, "sampling/sampling_logp_difference/mean": 0.021651615388691425, "step": 3410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 350.7, "completions/max_terminated_length": 350.7, "completions/mean_length": 236.6375, "completions/mean_terminated_length": 236.6375, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0706845991313458, "epoch": 0.31442493334559163, "frac_reward_zero_std": 0.5, "grad_norm": 5.893774032592773, "kl": 0.013176489010220394, "learning_rate": 4.959113855926898e-06, "loss": 0.0005, "num_tokens": 14866976.0, "reward": -0.15, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.294789469242096, "sampling/importance_sampling_ratio/mean": 0.9994373977184295, "sampling/importance_sampling_ratio/min": 0.762947803735733, "sampling/sampling_logp_difference/max": 0.29134219884872437, "sampling/sampling_logp_difference/mean": 0.021479264833033085, "step": 3420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 207.2, "completions/mean_terminated_length": 207.2, "completions/min_length": 122.7, "completions/min_terminated_length": 122.7, "entropy": 0.9512211792171001, "epoch": 0.3153443044957249, "frac_reward_zero_std": 0.5, "grad_norm": 0.11604711413383484, "kl": 0.011376893770648166, "learning_rate": 4.958656249192707e-06, "loss": 0.0007, "num_tokens": 14908184.0, "reward": -0.4, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.3375632643699646, "sampling/importance_sampling_ratio/mean": 1.0004499197006225, "sampling/importance_sampling_ratio/min": 0.7322974532842637, "sampling/sampling_logp_difference/max": 0.38893351554870603, "sampling/sampling_logp_difference/mean": 0.020050607807934284, "step": 3430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 213.3625, "completions/mean_terminated_length": 213.3625, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 0.9730191469192505, "epoch": 0.3162636756458582, "frac_reward_zero_std": 0.5, "grad_norm": 6.823051452636719, "kl": 0.012337391095934435, "learning_rate": 4.958196117221464e-06, "loss": 0.0005, "num_tokens": 14951149.0, "reward": -0.125, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2793813228607178, "sampling/importance_sampling_ratio/mean": 0.9999080836772919, "sampling/importance_sampling_ratio/min": 0.7554189741611481, "sampling/sampling_logp_difference/max": 0.31240421533584595, "sampling/sampling_logp_difference/mean": 0.020315217785537243, "step": 3440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.2, "completions/max_terminated_length": 297.2, "completions/mean_length": 205.2125, "completions/mean_terminated_length": 205.2125, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0016306631267071, "epoch": 0.3171830467959915, "frac_reward_zero_std": 0.5, "grad_norm": 0.2582768499851227, "kl": 0.011092762177577242, "learning_rate": 4.95773346048576e-06, "loss": 0.0005, "num_tokens": 14995334.0, "reward": -0.00625, "reward_std": 0.3770622000098228, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.3770622119307518, "sampling/importance_sampling_ratio/max": 1.298619031906128, "sampling/importance_sampling_ratio/mean": 1.0002946317195893, "sampling/importance_sampling_ratio/min": 0.7850292325019836, "sampling/sampling_logp_difference/max": 0.28630497455596926, "sampling/sampling_logp_difference/mean": 0.02018632087856531, "step": 3450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.3, "completions/max_terminated_length": 294.3, "completions/mean_length": 209.9875, "completions/mean_terminated_length": 209.9875, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.1378321073949338, "epoch": 0.31810241794612487, "frac_reward_zero_std": 0.5, "grad_norm": 0.13169017434120178, "kl": 0.014590884529752657, "learning_rate": 4.95726827946078e-06, "loss": 0.0007, "num_tokens": 15038645.0, "reward": -0.3, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2903668999671936, "sampling/importance_sampling_ratio/mean": 1.0002513945102691, "sampling/importance_sampling_ratio/min": 0.7798754274845123, "sampling/sampling_logp_difference/max": 0.278341943025589, "sampling/sampling_logp_difference/mean": 0.022294539771974087, "step": 3460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.9, "completions/max_terminated_length": 284.9, "completions/mean_length": 221.6875, "completions/mean_terminated_length": 221.6875, "completions/min_length": 154.0, "completions/min_terminated_length": 154.0, "entropy": 1.0628562785685063, "epoch": 0.31902178909625817, "frac_reward_zero_std": 0.5, "grad_norm": 0.1290055811405182, "kl": 0.011304972326615825, "learning_rate": 4.956800574624303e-06, "loss": 0.0005, "num_tokens": 15081772.0, "reward": 0.075, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2210638999938965, "sampling/importance_sampling_ratio/mean": 0.9996226131916046, "sampling/importance_sampling_ratio/min": 0.7727429270744324, "sampling/sampling_logp_difference/max": 0.2730405330657959, "sampling/sampling_logp_difference/mean": 0.021016532368957997, "step": 3470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.2, "completions/max_terminated_length": 274.2, "completions/mean_length": 194.5625, "completions/mean_terminated_length": 194.5625, "completions/min_length": 122.1, "completions/min_terminated_length": 122.1, "entropy": 1.0143702436238526, "epoch": 0.31994116024639147, "frac_reward_zero_std": 0.4, "grad_norm": 6.120928764343262, "kl": 0.010623425655649044, "learning_rate": 4.9563303464567e-06, "loss": 0.0004, "num_tokens": 15120761.0, "reward": 0.2, "reward_std": 0.5554606795310975, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5554607272148132, "sampling/importance_sampling_ratio/max": 1.2443084359169005, "sampling/importance_sampling_ratio/mean": 0.999220359325409, "sampling/importance_sampling_ratio/min": 0.7687348306179047, "sampling/sampling_logp_difference/max": 0.27268773317337036, "sampling/sampling_logp_difference/mean": 0.02080503236502409, "step": 3480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.8, "completions/max_terminated_length": 337.8, "completions/mean_length": 219.65, "completions/mean_terminated_length": 219.65, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.0878363780677318, "epoch": 0.32086053139652476, "frac_reward_zero_std": 0.4, "grad_norm": 0.10947508364915848, "kl": 0.012415934016462415, "learning_rate": 4.955857595440931e-06, "loss": 0.0007, "num_tokens": 15167957.0, "reward": 0.15, "reward_std": 0.49665169715881347, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.49665169715881347, "sampling/importance_sampling_ratio/max": 1.2405983328819274, "sampling/importance_sampling_ratio/mean": 0.9997793316841126, "sampling/importance_sampling_ratio/min": 0.7272709250450134, "sampling/sampling_logp_difference/max": 0.33355992436409, "sampling/sampling_logp_difference/mean": 0.021491809748113155, "step": 3490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 205.4, "completions/mean_terminated_length": 205.4, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.0390953730791808, "epoch": 0.32177990254665806, "frac_reward_zero_std": 0.4, "grad_norm": 7.744321823120117, "kl": 0.01315739516285248, "learning_rate": 4.955382322062551e-06, "loss": 0.0008, "num_tokens": 15212605.0, "reward": -0.275, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.287436807155609, "sampling/importance_sampling_ratio/mean": 0.9998999953269958, "sampling/importance_sampling_ratio/min": 0.7758686482906342, "sampling/sampling_logp_difference/max": 0.30030460357666017, "sampling/sampling_logp_difference/mean": 0.021055490337312223, "step": 3500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.8, "completions/max_terminated_length": 257.8, "completions/mean_length": 186.325, "completions/mean_terminated_length": 186.325, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0239496409893036, "epoch": 0.3226992736967914, "frac_reward_zero_std": 0.5, "grad_norm": 7.300314426422119, "kl": 0.01183616027701646, "learning_rate": 4.954904526809704e-06, "loss": 0.0005, "num_tokens": 15252439.0, "reward": -0.275, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2759687662124635, "sampling/importance_sampling_ratio/mean": 0.9997684895992279, "sampling/importance_sampling_ratio/min": 0.6940703392028809, "sampling/sampling_logp_difference/max": 0.40875532627105715, "sampling/sampling_logp_difference/mean": 0.02114726062864065, "step": 3510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.6, "completions/max_terminated_length": 273.6, "completions/mean_length": 189.05, "completions/mean_terminated_length": 189.05, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0074752181768418, "epoch": 0.3236186448469247, "frac_reward_zero_std": 0.6, "grad_norm": 0.09589658677577972, "kl": 0.011094226542627438, "learning_rate": 4.954424210173122e-06, "loss": 0.0005, "num_tokens": 15294283.0, "reward": 0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2803358078002929, "sampling/importance_sampling_ratio/mean": 1.0002174019813537, "sampling/importance_sampling_ratio/min": 0.8095321714878082, "sampling/sampling_logp_difference/max": 0.25745233297348025, "sampling/sampling_logp_difference/mean": 0.02037270274013281, "step": 3520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.7, "completions/max_terminated_length": 304.1, "completions/mean_length": 215.725, "completions/mean_terminated_length": 212.7375, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.0664041571319103, "epoch": 0.324538015997058, "frac_reward_zero_std": 0.3, "grad_norm": 6.630002498626709, "kl": 0.014995229063788428, "learning_rate": 4.95394137264613e-06, "loss": 0.0008, "num_tokens": 15336349.0, "reward": -0.2125, "reward_std": 0.5361612245440484, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.5361612424254417, "sampling/importance_sampling_ratio/max": 1.24510338306427, "sampling/importance_sampling_ratio/mean": 0.999545031785965, "sampling/importance_sampling_ratio/min": 0.6833582311868668, "sampling/sampling_logp_difference/max": 0.4157314717769623, "sampling/sampling_logp_difference/mean": 0.020948414131999014, "step": 3530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.3, "completions/max_terminated_length": 262.3, "completions/mean_length": 179.075, "completions/mean_terminated_length": 179.075, "completions/min_length": 108.6, "completions/min_terminated_length": 108.6, "entropy": 1.0016712386161088, "epoch": 0.3254573871471913, "frac_reward_zero_std": 0.5, "grad_norm": 0.20167100429534912, "kl": 0.011517896421719343, "learning_rate": 4.953456014724643e-06, "loss": 0.0002, "num_tokens": 15373411.0, "reward": 0.075, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2978383779525757, "sampling/importance_sampling_ratio/mean": 1.0002139866352082, "sampling/importance_sampling_ratio/min": 0.7719148337841034, "sampling/sampling_logp_difference/max": 0.2998048007488251, "sampling/sampling_logp_difference/mean": 0.020172739587724208, "step": 3540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 242.9, "completions/max_terminated_length": 242.9, "completions/mean_length": 177.125, "completions/mean_terminated_length": 177.125, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 0.9937149345874786, "epoch": 0.32637675829732465, "frac_reward_zero_std": 0.5, "grad_norm": 0.19431963562965393, "kl": 0.01249188241781667, "learning_rate": 4.952968136907158e-06, "loss": 0.0012, "num_tokens": 15412573.0, "reward": -0.3, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3881375670433045, "sampling/importance_sampling_ratio/mean": 1.000268131494522, "sampling/importance_sampling_ratio/min": 0.7659147083759308, "sampling/sampling_logp_difference/max": 0.3612779378890991, "sampling/sampling_logp_difference/mean": 0.020688130147755147, "step": 3550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.3, "completions/max_terminated_length": 323.0, "completions/mean_length": 232.0375, "completions/mean_terminated_length": 229.63214416503905, "completions/min_length": 154.6, "completions/min_terminated_length": 154.6, "entropy": 1.0296704567968846, "epoch": 0.32729612944745795, "frac_reward_zero_std": 0.3, "grad_norm": 0.13242831826210022, "kl": 0.011174838209990412, "learning_rate": 4.952477739694768e-06, "loss": 0.0007, "num_tokens": 15459792.0, "reward": 0.11875, "reward_std": 0.5936936140060425, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.5936936259269714, "sampling/importance_sampling_ratio/max": 1.3590788125991822, "sampling/importance_sampling_ratio/mean": 1.0003484904766082, "sampling/importance_sampling_ratio/min": 0.7825342178344726, "sampling/sampling_logp_difference/max": 0.31433509588241576, "sampling/sampling_logp_difference/mean": 0.020928164198994636, "step": 3560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 296.4, "completions/max_terminated_length": 281.9, "completions/mean_length": 207.725, "completions/mean_terminated_length": 204.8589294433594, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.1179012343287469, "epoch": 0.32821550059759125, "frac_reward_zero_std": 0.4, "grad_norm": 4.940199375152588, "kl": 0.011389413650613278, "learning_rate": 4.951984823591148e-06, "loss": 0.0006, "num_tokens": 15503682.0, "reward": -0.30625, "reward_std": 0.5229829370975494, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5229829490184784, "sampling/importance_sampling_ratio/max": 1.3222885847091674, "sampling/importance_sampling_ratio/mean": 0.9998194932937622, "sampling/importance_sampling_ratio/min": 0.8057673096656799, "sampling/sampling_logp_difference/max": 0.2863261878490448, "sampling/sampling_logp_difference/mean": 0.020976205170154572, "step": 3570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 204.9, "completions/mean_terminated_length": 204.9, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0537150751799345, "epoch": 0.32913487174772454, "frac_reward_zero_std": 0.4, "grad_norm": 6.648979663848877, "kl": 0.01108229480450973, "learning_rate": 4.951489389102564e-06, "loss": 0.0006, "num_tokens": 15545978.0, "reward": -0.475, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2356521487236023, "sampling/importance_sampling_ratio/mean": 0.9997918367385864, "sampling/importance_sampling_ratio/min": 0.765489912033081, "sampling/sampling_logp_difference/max": 0.28154668807983396, "sampling/sampling_logp_difference/mean": 0.021404032409191132, "step": 3580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.9, "completions/max_terminated_length": 286.9, "completions/mean_length": 217.6375, "completions/mean_terminated_length": 215.49642944335938, "completions/min_length": 150.5, "completions/min_terminated_length": 150.5, "entropy": 1.0331536792218685, "epoch": 0.33005424289785784, "frac_reward_zero_std": 0.4, "grad_norm": 4.512575149536133, "kl": 0.010969623760320246, "learning_rate": 4.950991436737868e-06, "loss": 0.0008, "num_tokens": 15591269.0, "reward": 0.11875, "reward_std": 0.48736135810613634, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.48736135810613634, "sampling/importance_sampling_ratio/max": 1.2530445456504822, "sampling/importance_sampling_ratio/mean": 0.9996728777885437, "sampling/importance_sampling_ratio/min": 0.73861163854599, "sampling/sampling_logp_difference/max": 0.3141424059867859, "sampling/sampling_logp_difference/mean": 0.021030213125050067, "step": 3590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.3, "completions/max_terminated_length": 281.3, "completions/mean_length": 196.0875, "completions/mean_terminated_length": 196.0875, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 1.0028023455291986, "epoch": 0.3309736140479912, "frac_reward_zero_std": 0.4, "grad_norm": 0.15529422461986542, "kl": 0.01243047786410898, "learning_rate": 4.9504909670084934e-06, "loss": 0.0007, "num_tokens": 15634500.0, "reward": -0.63125, "reward_std": 0.43682936280965806, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.43682938665151594, "sampling/importance_sampling_ratio/max": 1.2834877252578736, "sampling/importance_sampling_ratio/mean": 0.9998450875282288, "sampling/importance_sampling_ratio/min": 0.7898684084415436, "sampling/sampling_logp_difference/max": 0.2665059566497803, "sampling/sampling_logp_difference/mean": 0.020914800092577936, "step": 3600 }, { "epoch": 0.3309736140479912, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 290.67539267015707, "eval_completions/max_terminated_length": 288.94240837696333, "eval_completions/mean_length": 203.49760034904014, "eval_completions/mean_terminated_length": 203.02476536083387, "eval_completions/min_length": 128.69633507853402, "eval_completions/min_terminated_length": 128.69633507853402, "eval_entropy": 1.0631092742579562, "eval_frac_reward_zero_std": 0.4956369982547993, "eval_kl": 0.011741269656975492, "eval_loss": 0.0006005754112266004, "eval_num_tokens": 15634500.0, "eval_reward": -0.11267452006980803, "eval_reward_std": 0.4439167717364446, "eval_rewards/reward_fn/mean": -0.11267452006980803, "eval_rewards/reward_fn/std": 0.44391678598746787, "eval_runtime": 1696.9916, "eval_samples_per_second": 0.338, "eval_sampling/importance_sampling_ratio/max": 1.2670826262828567, "eval_sampling/importance_sampling_ratio/mean": 0.9999941284743903, "eval_sampling/importance_sampling_ratio/min": 0.7740164943962613, "eval_sampling/sampling_logp_difference/max": 0.2838432843980573, "eval_sampling/sampling_logp_difference/mean": 0.020830457981344294, "eval_steps_per_second": 0.042, "step": 3600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.2, "completions/max_terminated_length": 306.2, "completions/mean_length": 213.75, "completions/mean_terminated_length": 213.75, "completions/min_length": 147.1, "completions/min_terminated_length": 147.1, "entropy": 1.0776903115212917, "epoch": 0.3318929851981245, "frac_reward_zero_std": 0.4, "grad_norm": 5.620420932769775, "kl": 0.01364452107809484, "learning_rate": 4.949987980428466e-06, "loss": 0.0009, "num_tokens": 15682568.0, "reward": 0.05, "reward_std": 0.5226615250110627, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5226615607738495, "sampling/importance_sampling_ratio/max": 1.2493976354599, "sampling/importance_sampling_ratio/mean": 1.000603038072586, "sampling/importance_sampling_ratio/min": 0.7673669815063476, "sampling/sampling_logp_difference/max": 0.2822683095932007, "sampling/sampling_logp_difference/mean": 0.021432445012032987, "step": 3610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.9, "completions/max_terminated_length": 312.9, "completions/mean_length": 211.5375, "completions/mean_terminated_length": 211.5375, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.1095087677240372, "epoch": 0.3328123563482578, "frac_reward_zero_std": 0.4, "grad_norm": 8.087228775024414, "kl": 0.012184544617775827, "learning_rate": 4.949482477514391e-06, "loss": 0.0004, "num_tokens": 15726307.0, "reward": -0.30625, "reward_std": 0.5310883328318596, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5310883566737175, "sampling/importance_sampling_ratio/max": 1.2980021238327026, "sampling/importance_sampling_ratio/mean": 1.0000895023345948, "sampling/importance_sampling_ratio/min": 0.7876690626144409, "sampling/sampling_logp_difference/max": 0.28760193586349486, "sampling/sampling_logp_difference/mean": 0.02085013557225466, "step": 3620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 303.8, "completions/max_terminated_length": 292.7, "completions/mean_length": 206.2875, "completions/mean_terminated_length": 195.4875, "completions/min_length": 110.8, "completions/min_terminated_length": 110.8, "entropy": 0.9763751167804002, "epoch": 0.3337317274983911, "frac_reward_zero_std": 0.3, "grad_norm": 8.270167350769043, "kl": 0.023923257499700413, "learning_rate": 4.948974458785463e-06, "loss": 0.0007, "num_tokens": 15766682.0, "reward": 0.1, "reward_std": 0.6287176668643951, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6287176668643951, "sampling/importance_sampling_ratio/max": 1.3638510823249816, "sampling/importance_sampling_ratio/mean": 1.0001223683357239, "sampling/importance_sampling_ratio/min": 0.7290257334709167, "sampling/sampling_logp_difference/max": 0.4327288746833801, "sampling/sampling_logp_difference/mean": 0.020251426752656697, "step": 3630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.8, "completions/max_terminated_length": 294.8, "completions/mean_length": 201.7, "completions/mean_terminated_length": 201.7, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.1100360110402108, "epoch": 0.33465109864852444, "frac_reward_zero_std": 0.8, "grad_norm": 0.13998104631900787, "kl": 0.011279226845363154, "learning_rate": 4.9484639247634555e-06, "loss": 0.0005, "num_tokens": 15811226.0, "reward": 0.1, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2599603056907653, "sampling/importance_sampling_ratio/mean": 0.9996344923973084, "sampling/importance_sampling_ratio/min": 0.7226696729660034, "sampling/sampling_logp_difference/max": 0.3424621820449829, "sampling/sampling_logp_difference/mean": 0.0223692974075675, "step": 3640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 213.8625, "completions/mean_terminated_length": 213.8625, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.1652512215077877, "epoch": 0.33557046979865773, "frac_reward_zero_std": 0.3, "grad_norm": 5.744229793548584, "kl": 0.012660046754172071, "learning_rate": 4.947950875972729e-06, "loss": 0.0006, "num_tokens": 15857495.0, "reward": -0.15, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.296852970123291, "sampling/importance_sampling_ratio/mean": 0.9999665021896362, "sampling/importance_sampling_ratio/min": 0.7885059118270874, "sampling/sampling_logp_difference/max": 0.27971017360687256, "sampling/sampling_logp_difference/mean": 0.02225772850215435, "step": 3650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.5, "completions/max_terminated_length": 330.5, "completions/mean_length": 221.35, "completions/mean_terminated_length": 221.35, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.063965267688036, "epoch": 0.33648984094879103, "frac_reward_zero_std": 0.5, "grad_norm": 6.414453029632568, "kl": 0.013420826906803995, "learning_rate": 4.947435312940226e-06, "loss": 0.0007, "num_tokens": 15901683.0, "reward": -0.33125, "reward_std": 0.5221937894821167, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.5221938073635102, "sampling/importance_sampling_ratio/max": 1.3630035638809204, "sampling/importance_sampling_ratio/mean": 1.0001299023628234, "sampling/importance_sampling_ratio/min": 0.7677088856697083, "sampling/sampling_logp_difference/max": 0.35501052141189576, "sampling/sampling_logp_difference/mean": 0.020311651192605494, "step": 3660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 200.9875, "completions/mean_terminated_length": 200.9875, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.0612401463091374, "epoch": 0.3374092120989243, "frac_reward_zero_std": 0.5, "grad_norm": 6.166656970977783, "kl": 0.011865037918323652, "learning_rate": 4.94691723619547e-06, "loss": 0.0007, "num_tokens": 15944546.0, "reward": 0.2, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.3284626245498656, "sampling/importance_sampling_ratio/mean": 1.0003156304359435, "sampling/importance_sampling_ratio/min": 0.7697577059268952, "sampling/sampling_logp_difference/max": 0.2948136806488037, "sampling/sampling_logp_difference/mean": 0.020989071018993853, "step": 3670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.2, "completions/max_terminated_length": 313.2, "completions/mean_length": 210.4375, "completions/mean_terminated_length": 210.4375, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0553309857845306, "epoch": 0.3383285832490576, "frac_reward_zero_std": 0.5, "grad_norm": 7.613981246948242, "kl": 0.012788902362808585, "learning_rate": 4.946396646270569e-06, "loss": 0.001, "num_tokens": 15993037.0, "reward": 0.06875, "reward_std": 0.4964326679706573, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.4964326679706573, "sampling/importance_sampling_ratio/max": 1.2283290505409241, "sampling/importance_sampling_ratio/mean": 0.9993171274662018, "sampling/importance_sampling_ratio/min": 0.781339693069458, "sampling/sampling_logp_difference/max": 0.2534042954444885, "sampling/sampling_logp_difference/mean": 0.020393354073166847, "step": 3680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 188.15, "completions/mean_terminated_length": 188.15, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0934713736176491, "epoch": 0.339247954399191, "frac_reward_zero_std": 0.5, "grad_norm": 6.5257887840271, "kl": 0.012660630559548736, "learning_rate": 4.945873543700209e-06, "loss": 0.0006, "num_tokens": 16033969.0, "reward": 0.3, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2522456049919128, "sampling/importance_sampling_ratio/mean": 0.9993742287158967, "sampling/importance_sampling_ratio/min": 0.7641173243522644, "sampling/sampling_logp_difference/max": 0.29059739112854005, "sampling/sampling_logp_difference/mean": 0.02115203980356455, "step": 3690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 188.3625, "completions/mean_terminated_length": 188.3625, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 1.0892113648355006, "epoch": 0.3401673255493243, "frac_reward_zero_std": 0.4, "grad_norm": 6.124208450317383, "kl": 0.014292460423894227, "learning_rate": 4.945347929021659e-06, "loss": 0.0013, "num_tokens": 16076670.0, "reward": 0.1, "reward_std": 0.5916701912879944, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5916702151298523, "sampling/importance_sampling_ratio/max": 1.2820717453956605, "sampling/importance_sampling_ratio/mean": 0.9998752772808075, "sampling/importance_sampling_ratio/min": 0.7247937530279159, "sampling/sampling_logp_difference/max": 0.36658614575862886, "sampling/sampling_logp_difference/mean": 0.021411561779677867, "step": 3700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.9, "completions/max_terminated_length": 292.9, "completions/mean_length": 205.0875, "completions/mean_terminated_length": 205.0875, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.0957892641425133, "epoch": 0.34108669669945757, "frac_reward_zero_std": 0.6, "grad_norm": 0.12415610998868942, "kl": 0.010919964360073208, "learning_rate": 4.944819802774767e-06, "loss": 0.0005, "num_tokens": 16117253.0, "reward": -0.125, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.2568305611610413, "sampling/importance_sampling_ratio/mean": 0.9990845620632172, "sampling/importance_sampling_ratio/min": 0.7436662197113038, "sampling/sampling_logp_difference/max": 0.3065265893936157, "sampling/sampling_logp_difference/mean": 0.021982208453118803, "step": 3710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.7, "completions/max_terminated_length": 318.7, "completions/mean_length": 217.8625, "completions/mean_terminated_length": 217.8625, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.0760253839194776, "epoch": 0.34200606784959087, "frac_reward_zero_std": 0.6, "grad_norm": 5.269711971282959, "kl": 0.011005713883787394, "learning_rate": 4.944289165501961e-06, "loss": 0.0006, "num_tokens": 16161362.0, "reward": -0.325, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2983101606369019, "sampling/importance_sampling_ratio/mean": 1.0000386714935303, "sampling/importance_sampling_ratio/min": 0.7776707649230957, "sampling/sampling_logp_difference/max": 0.28872924447059634, "sampling/sampling_logp_difference/mean": 0.02106418814510107, "step": 3720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.8, "completions/max_terminated_length": 298.8, "completions/mean_length": 205.825, "completions/mean_terminated_length": 205.825, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 0.9833491265773773, "epoch": 0.34292543899972416, "frac_reward_zero_std": 0.5, "grad_norm": 5.233916759490967, "kl": 0.011344584997277706, "learning_rate": 4.9437560177482494e-06, "loss": 0.001, "num_tokens": 16201820.0, "reward": -0.2, "reward_std": 0.5066213369369507, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5066213846206665, "sampling/importance_sampling_ratio/max": 1.2467018485069274, "sampling/importance_sampling_ratio/mean": 0.9997506558895111, "sampling/importance_sampling_ratio/min": 0.7226564109325408, "sampling/sampling_logp_difference/max": 0.3378041505813599, "sampling/sampling_logp_difference/mean": 0.020489361137151718, "step": 3730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.3, "completions/max_terminated_length": 274.3, "completions/mean_length": 203.7125, "completions/mean_terminated_length": 203.7125, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0581568896770477, "epoch": 0.3438448101498575, "frac_reward_zero_std": 0.4, "grad_norm": 4.897265434265137, "kl": 0.01225677010952495, "learning_rate": 4.943220360061215e-06, "loss": 0.0001, "num_tokens": 16246597.0, "reward": -0.28125, "reward_std": 0.5163200557231903, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.5163200855255127, "sampling/importance_sampling_ratio/max": 1.2618929028511048, "sampling/importance_sampling_ratio/mean": 1.0007981061935425, "sampling/importance_sampling_ratio/min": 0.7582690089941024, "sampling/sampling_logp_difference/max": 0.3016495227813721, "sampling/sampling_logp_difference/mean": 0.020428329519927502, "step": 3740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.3, "completions/max_terminated_length": 308.3, "completions/mean_length": 225.0375, "completions/mean_terminated_length": 225.0375, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0653736129403115, "epoch": 0.3447641812999908, "frac_reward_zero_std": 0.3, "grad_norm": 0.13490329682826996, "kl": 0.013224344892660157, "learning_rate": 4.942682192991024e-06, "loss": 0.0006, "num_tokens": 16290464.0, "reward": -0.125, "reward_std": 0.6589705109596252, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.65897057056427, "sampling/importance_sampling_ratio/max": 1.2665727972984313, "sampling/importance_sampling_ratio/mean": 1.0000663161277772, "sampling/importance_sampling_ratio/min": 0.7756862103939056, "sampling/sampling_logp_difference/max": 0.26750117540359497, "sampling/sampling_logp_difference/mean": 0.021077648736536503, "step": 3750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.5, "completions/max_terminated_length": 328.5, "completions/mean_length": 222.9875, "completions/mean_terminated_length": 222.9875, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.0250547751784325, "epoch": 0.3456835524501241, "frac_reward_zero_std": 0.3, "grad_norm": 5.286283016204834, "kl": 0.011447757779387757, "learning_rate": 4.942141517090415e-06, "loss": 0.0007, "num_tokens": 16335375.0, "reward": 0.0125, "reward_std": 0.6194513082504273, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.6194513082504273, "sampling/importance_sampling_ratio/max": 1.2484618067741393, "sampling/importance_sampling_ratio/mean": 0.9998645424842835, "sampling/importance_sampling_ratio/min": 0.7800793468952179, "sampling/sampling_logp_difference/max": 0.2651719212532043, "sampling/sampling_logp_difference/mean": 0.020452835038304328, "step": 3760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.6, "completions/max_terminated_length": 291.6, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.061372798681259, "epoch": 0.3466029236002574, "frac_reward_zero_std": 0.7, "grad_norm": 6.4758429527282715, "kl": 0.01170982233597897, "learning_rate": 4.941598332914707e-06, "loss": 0.0001, "num_tokens": 16379709.0, "reward": -0.075, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.3065367579460143, "sampling/importance_sampling_ratio/mean": 0.9996950149536132, "sampling/importance_sampling_ratio/min": 0.7354342490434647, "sampling/sampling_logp_difference/max": 0.36794450879096985, "sampling/sampling_logp_difference/mean": 0.020779002457857132, "step": 3770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.6, "completions/max_terminated_length": 284.6, "completions/mean_length": 214.075, "completions/mean_terminated_length": 214.075, "completions/min_length": 154.1, "completions/min_terminated_length": 154.1, "entropy": 1.0027343519032001, "epoch": 0.34752229475039076, "frac_reward_zero_std": 0.6, "grad_norm": 6.662972927093506, "kl": 0.010331993058207445, "learning_rate": 4.941052641021793e-06, "loss": 0.0005, "num_tokens": 16426667.0, "reward": -0.15, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.317011260986328, "sampling/importance_sampling_ratio/mean": 0.9999153196811676, "sampling/importance_sampling_ratio/min": 0.7873839795589447, "sampling/sampling_logp_difference/max": 0.2949887990951538, "sampling/sampling_logp_difference/mean": 0.02042299062013626, "step": 3780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.4, "completions/max_terminated_length": 322.4, "completions/mean_length": 234.05, "completions/mean_terminated_length": 234.05, "completions/min_length": 159.4, "completions/min_terminated_length": 159.4, "entropy": 1.0661455377936364, "epoch": 0.34844166590052406, "frac_reward_zero_std": 0.2, "grad_norm": 4.628396034240723, "kl": 0.012924973771441728, "learning_rate": 4.940504441972142e-06, "loss": 0.0002, "num_tokens": 16471623.0, "reward": -0.05625, "reward_std": 0.7376778960227967, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.737677925825119, "sampling/importance_sampling_ratio/max": 1.2688018560409546, "sampling/importance_sampling_ratio/mean": 0.999770563840866, "sampling/importance_sampling_ratio/min": 0.7569762468338013, "sampling/sampling_logp_difference/max": 0.30265371799468993, "sampling/sampling_logp_difference/mean": 0.02079704087227583, "step": 3790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 191.075, "completions/mean_terminated_length": 191.075, "completions/min_length": 120.9, "completions/min_terminated_length": 120.9, "entropy": 1.0654788322746753, "epoch": 0.34936103705065735, "frac_reward_zero_std": 0.6, "grad_norm": 0.5529763698577881, "kl": 0.012940020021051168, "learning_rate": 4.939953736328799e-06, "loss": 0.0008, "num_tokens": 16511821.0, "reward": -0.125, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.3109470009803772, "sampling/importance_sampling_ratio/mean": 1.000136423110962, "sampling/importance_sampling_ratio/min": 0.7860559165477753, "sampling/sampling_logp_difference/max": 0.2964987516403198, "sampling/sampling_logp_difference/mean": 0.02147068940103054, "step": 3800 }, { "epoch": 0.34936103705065735, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 296.91623036649213, "eval_completions/max_terminated_length": 295.09424083769636, "eval_completions/mean_length": 206.3542757417103, "eval_completions/mean_terminated_length": 205.9815197157402, "eval_completions/min_length": 132.8062827225131, "eval_completions/min_terminated_length": 132.8062827225131, "eval_entropy": 1.0885215709034686, "eval_frac_reward_zero_std": 0.4886561954624782, "eval_kl": 0.01152203748259453, "eval_loss": 0.0005615889676846564, "eval_num_tokens": 16511821.0, "eval_reward": -0.13568935427574172, "eval_reward_std": 0.45218269269503847, "eval_rewards/reward_fn/mean": -0.13568935427574172, "eval_rewards/reward_fn/std": 0.45218270736215005, "eval_runtime": 1728.4772, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.2687158309976467, "eval_sampling/importance_sampling_ratio/mean": 1.0000270777763913, "eval_sampling/importance_sampling_ratio/min": 0.7707268085467254, "eval_sampling/sampling_logp_difference/max": 0.2926208122030395, "eval_sampling/sampling_logp_difference/mean": 0.020735366539753337, "eval_steps_per_second": 0.042, "step": 3800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.2, "completions/max_terminated_length": 289.2, "completions/mean_length": 202.4125, "completions/mean_terminated_length": 202.4125, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.053345050662756, "epoch": 0.35028040820079065, "frac_reward_zero_std": 0.7, "grad_norm": 0.09794866293668747, "kl": 0.011145023658173159, "learning_rate": 4.9394005246573826e-06, "loss": 0.0006, "num_tokens": 16555134.0, "reward": -0.25, "reward_std": 0.31392415761947634, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3139241814613342, "sampling/importance_sampling_ratio/max": 1.238322627544403, "sampling/importance_sampling_ratio/mean": 0.9993649840354919, "sampling/importance_sampling_ratio/min": 0.7733029067516327, "sampling/sampling_logp_difference/max": 0.2610157191753387, "sampling/sampling_logp_difference/mean": 0.021475919894874095, "step": 3810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 208.425, "completions/mean_terminated_length": 208.425, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.0193090073764324, "epoch": 0.35119977935092395, "frac_reward_zero_std": 0.5, "grad_norm": 0.12642347812652588, "kl": 0.010904795106034726, "learning_rate": 4.938844807526085e-06, "loss": 0.0003, "num_tokens": 16597288.0, "reward": 0.3, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.259182369709015, "sampling/importance_sampling_ratio/mean": 1.000695925951004, "sampling/importance_sampling_ratio/min": 0.7958551585674286, "sampling/sampling_logp_difference/max": 0.2510819673538208, "sampling/sampling_logp_difference/mean": 0.020689052529633045, "step": 3820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.3, "completions/max_terminated_length": 310.3, "completions/mean_length": 220.275, "completions/mean_terminated_length": 220.275, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 1.054525362700224, "epoch": 0.3521191505010573, "frac_reward_zero_std": 0.7, "grad_norm": 0.12457771599292755, "kl": 0.01070829654345289, "learning_rate": 4.938286585505673e-06, "loss": 0.0003, "num_tokens": 16643150.0, "reward": -0.175, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.286538803577423, "sampling/importance_sampling_ratio/mean": 1.0002843022346497, "sampling/importance_sampling_ratio/min": 0.7652734637260437, "sampling/sampling_logp_difference/max": 0.30440855026245117, "sampling/sampling_logp_difference/mean": 0.020538173988461493, "step": 3830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 248.7, "completions/max_terminated_length": 248.7, "completions/mean_length": 175.85, "completions/mean_terminated_length": 175.85, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.045484734326601, "epoch": 0.3530385216511906, "frac_reward_zero_std": 0.3, "grad_norm": 7.262127876281738, "kl": 0.012519288790645077, "learning_rate": 4.9377258591694854e-06, "loss": 0.0008, "num_tokens": 16683306.0, "reward": -0.15, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2556553125381469, "sampling/importance_sampling_ratio/mean": 0.9997143983840943, "sampling/importance_sampling_ratio/min": 0.763152152299881, "sampling/sampling_logp_difference/max": 0.29271273612976073, "sampling/sampling_logp_difference/mean": 0.02135893814265728, "step": 3840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.9, "completions/max_terminated_length": 296.9, "completions/mean_length": 210.5375, "completions/mean_terminated_length": 210.5375, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.07700899168849, "epoch": 0.3539578928013239, "frac_reward_zero_std": 0.6, "grad_norm": 0.12488441914319992, "kl": 0.010444469872163608, "learning_rate": 4.937162629093433e-06, "loss": 0.0008, "num_tokens": 16726517.0, "reward": -0.1, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2700573682785035, "sampling/importance_sampling_ratio/mean": 0.9996520042419433, "sampling/importance_sampling_ratio/min": 0.8016027808189392, "sampling/sampling_logp_difference/max": 0.2512554407119751, "sampling/sampling_logp_difference/mean": 0.020532961934804916, "step": 3850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 215.5625, "completions/mean_terminated_length": 215.5625, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.1459747046232223, "epoch": 0.3548772639514572, "frac_reward_zero_std": 0.6, "grad_norm": 0.1956474632024765, "kl": 0.012508171331137418, "learning_rate": 4.936596895855998e-06, "loss": 0.0008, "num_tokens": 16770874.0, "reward": -0.25, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2894083499908446, "sampling/importance_sampling_ratio/mean": 0.9998260021209717, "sampling/importance_sampling_ratio/min": 0.7349626421928406, "sampling/sampling_logp_difference/max": 0.3440129578113556, "sampling/sampling_logp_difference/mean": 0.02144880760461092, "step": 3860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 344.9, "completions/max_terminated_length": 344.9, "completions/mean_length": 237.95, "completions/mean_terminated_length": 237.95, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.084223533421755, "epoch": 0.3557966351015905, "frac_reward_zero_std": 0.2, "grad_norm": 4.430074691772461, "kl": 0.010195340798236429, "learning_rate": 4.9360286600382326e-06, "loss": 0.0007, "num_tokens": 16815342.0, "reward": 0.01875, "reward_std": 0.6924362659454346, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.6924362897872924, "sampling/importance_sampling_ratio/max": 1.2486694216728211, "sampling/importance_sampling_ratio/mean": 0.9998777151107788, "sampling/importance_sampling_ratio/min": 0.7804706037044525, "sampling/sampling_logp_difference/max": 0.254573255777359, "sampling/sampling_logp_difference/mean": 0.020658917725086212, "step": 3870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 332.8, "completions/max_terminated_length": 319.0, "completions/mean_length": 220.9, "completions/mean_terminated_length": 217.78571472167968, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0467714592814445, "epoch": 0.35671600625172384, "frac_reward_zero_std": 0.5, "grad_norm": 7.400862693786621, "kl": 0.012360867077950389, "learning_rate": 4.9354579222237624e-06, "loss": 0.0004, "num_tokens": 16861862.0, "reward": -0.00625, "reward_std": 0.48030423521995547, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.48030425906181334, "sampling/importance_sampling_ratio/max": 1.271393394470215, "sampling/importance_sampling_ratio/mean": 0.9997876405715942, "sampling/importance_sampling_ratio/min": 0.7547622144222259, "sampling/sampling_logp_difference/max": 0.29249796867370603, "sampling/sampling_logp_difference/mean": 0.0197792349383235, "step": 3880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.6, "completions/max_terminated_length": 281.6, "completions/mean_length": 202.475, "completions/mean_terminated_length": 202.475, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.0956706143915653, "epoch": 0.35763537740185714, "frac_reward_zero_std": 0.7, "grad_norm": 5.225960731506348, "kl": 0.011294832645216957, "learning_rate": 4.934884682998781e-06, "loss": 0.0005, "num_tokens": 16903892.0, "reward": -0.075, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.21480712890625, "sampling/importance_sampling_ratio/mean": 0.9994322121143341, "sampling/importance_sampling_ratio/min": 0.7887655615806579, "sampling/sampling_logp_difference/max": 0.23971204757690429, "sampling/sampling_logp_difference/mean": 0.021209538355469702, "step": 3890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.1, "completions/max_terminated_length": 303.4, "completions/mean_length": 226.725, "completions/mean_terminated_length": 223.30178680419922, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "entropy": 1.0722710236907005, "epoch": 0.35855474855199043, "frac_reward_zero_std": 0.5, "grad_norm": 1.2538424730300903, "kl": 0.01138078909716569, "learning_rate": 4.934308942952049e-06, "loss": 0.0004, "num_tokens": 16948366.0, "reward": -0.23125, "reward_std": 0.5010819792747497, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5010820209980011, "sampling/importance_sampling_ratio/max": 1.333158302307129, "sampling/importance_sampling_ratio/mean": 1.0001471102237702, "sampling/importance_sampling_ratio/min": 0.7634155929088593, "sampling/sampling_logp_difference/max": 0.3129764825105667, "sampling/sampling_logp_difference/mean": 0.02130358051508665, "step": 3900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 209.45, "completions/mean_terminated_length": 209.45, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.0320186376571656, "epoch": 0.35947411970212373, "frac_reward_zero_std": 0.8, "grad_norm": 0.10857973992824554, "kl": 0.0110542036825791, "learning_rate": 4.933730702674899e-06, "loss": 0.0005, "num_tokens": 16993338.0, "reward": -0.4, "reward_std": 0.1851640224456787, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.1851640224456787, "sampling/importance_sampling_ratio/max": 1.3026119351387024, "sampling/importance_sampling_ratio/mean": 1.0000069379806518, "sampling/importance_sampling_ratio/min": 0.765450245141983, "sampling/sampling_logp_difference/max": 0.31453957557678225, "sampling/sampling_logp_difference/mean": 0.020485105738043786, "step": 3910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 202.3, "completions/mean_terminated_length": 202.3, "completions/min_length": 150.2, "completions/min_terminated_length": 150.2, "entropy": 1.0709520630538463, "epoch": 0.3603934908522571, "frac_reward_zero_std": 0.5, "grad_norm": 5.5692315101623535, "kl": 0.010571346123469993, "learning_rate": 4.933149962761231e-06, "loss": 0.0003, "num_tokens": 17035994.0, "reward": 0.25, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2415111184120178, "sampling/importance_sampling_ratio/mean": 0.999776816368103, "sampling/importance_sampling_ratio/min": 0.7365941286087037, "sampling/sampling_logp_difference/max": 0.3241949677467346, "sampling/sampling_logp_difference/mean": 0.02140331417322159, "step": 3920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 190.7125, "completions/mean_terminated_length": 190.7125, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 1.041239431500435, "epoch": 0.3613128620023904, "frac_reward_zero_std": 0.2, "grad_norm": 6.399746417999268, "kl": 0.012641590950079263, "learning_rate": 4.93256672380751e-06, "loss": 0.0003, "num_tokens": 17078603.0, "reward": -0.025, "reward_std": 0.6531550586223602, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6531550824642182, "sampling/importance_sampling_ratio/max": 1.3294009804725646, "sampling/importance_sampling_ratio/mean": 1.0005433082580566, "sampling/importance_sampling_ratio/min": 0.742558890581131, "sampling/sampling_logp_difference/max": 0.34329813718795776, "sampling/sampling_logp_difference/mean": 0.02211311347782612, "step": 3930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 195.0, "completions/mean_terminated_length": 195.0, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.075921406596899, "epoch": 0.3622322331525237, "frac_reward_zero_std": 0.6, "grad_norm": 6.750421524047852, "kl": 0.013065169972833246, "learning_rate": 4.931980986412769e-06, "loss": 0.0005, "num_tokens": 17120147.0, "reward": -0.075, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.3099331855773926, "sampling/importance_sampling_ratio/mean": 0.9998826444149017, "sampling/importance_sampling_ratio/min": 0.7376315742731094, "sampling/sampling_logp_difference/max": 0.3271780401468277, "sampling/sampling_logp_difference/mean": 0.02169000394642353, "step": 3940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 204.7375, "completions/mean_terminated_length": 204.7375, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 1.0515459820628166, "epoch": 0.36315160430265697, "frac_reward_zero_std": 0.5, "grad_norm": 6.761740207672119, "kl": 0.011576194560620933, "learning_rate": 4.931392751178608e-06, "loss": 0.0007, "num_tokens": 17165942.0, "reward": -0.36875, "reward_std": 0.4524163782596588, "rewards/reward_fn/mean": -0.36875, "rewards/reward_fn/std": 0.4524164080619812, "sampling/importance_sampling_ratio/max": 1.2561506867408752, "sampling/importance_sampling_ratio/mean": 0.9994567513465882, "sampling/importance_sampling_ratio/min": 0.7711267471313477, "sampling/sampling_logp_difference/max": 0.2731828272342682, "sampling/sampling_logp_difference/mean": 0.02056875955313444, "step": 3950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.3, "completions/max_terminated_length": 310.3, "completions/mean_length": 215.65, "completions/mean_terminated_length": 215.65, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 1.1009848780930043, "epoch": 0.36407097545279027, "frac_reward_zero_std": 0.5, "grad_norm": 5.518365383148193, "kl": 0.012111204717075452, "learning_rate": 4.930802018709192e-06, "loss": 0.0008, "num_tokens": 17209258.0, "reward": -0.30625, "reward_std": 0.40570716708898547, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.4057071790099144, "sampling/importance_sampling_ratio/max": 1.3127690315246583, "sampling/importance_sampling_ratio/mean": 0.9996513664722443, "sampling/importance_sampling_ratio/min": 0.787645012140274, "sampling/sampling_logp_difference/max": 0.29233012199401853, "sampling/sampling_logp_difference/mean": 0.0214562077075243, "step": 3960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.0, "completions/max_terminated_length": 239.0, "completions/mean_length": 169.6125, "completions/mean_terminated_length": 169.6125, "completions/min_length": 113.5, "completions/min_terminated_length": 113.5, "entropy": 0.9710375808179379, "epoch": 0.3649903466029236, "frac_reward_zero_std": 0.7, "grad_norm": 0.11539783328771591, "kl": 0.010889772899099625, "learning_rate": 4.9302087896112506e-06, "loss": 0.0006, "num_tokens": 17248235.0, "reward": -0.025, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2307886481285095, "sampling/importance_sampling_ratio/mean": 0.9995222389698029, "sampling/importance_sampling_ratio/min": 0.7796154677867889, "sampling/sampling_logp_difference/max": 0.24974602460861206, "sampling/sampling_logp_difference/mean": 0.020747668482363223, "step": 3970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.5, "completions/max_terminated_length": 285.5, "completions/mean_length": 189.6, "completions/mean_terminated_length": 189.6, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 0.9705347791314125, "epoch": 0.3659097177530569, "frac_reward_zero_std": 0.5, "grad_norm": 0.11269614845514297, "kl": 0.010621589963557198, "learning_rate": 4.929613064494076e-06, "loss": 0.0004, "num_tokens": 17286947.0, "reward": -0.3, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.357460880279541, "sampling/importance_sampling_ratio/mean": 1.0001745104789734, "sampling/importance_sampling_ratio/min": 0.7393416643142701, "sampling/sampling_logp_difference/max": 0.38856948018074033, "sampling/sampling_logp_difference/mean": 0.019864232279360294, "step": 3980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.8, "completions/max_terminated_length": 270.8, "completions/mean_length": 202.4625, "completions/mean_terminated_length": 202.4625, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.0416151754558087, "epoch": 0.3668290889031902, "frac_reward_zero_std": 0.7, "grad_norm": 0.09199123829603195, "kl": 0.012723285926040263, "learning_rate": 4.929014843969527e-06, "loss": 0.0008, "num_tokens": 17327792.0, "reward": 0.225, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.3348411440849304, "sampling/importance_sampling_ratio/mean": 0.9999898970127106, "sampling/importance_sampling_ratio/min": 0.7838741898536682, "sampling/sampling_logp_difference/max": 0.3182054042816162, "sampling/sampling_logp_difference/mean": 0.020807861536741256, "step": 3990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.2, "completions/max_terminated_length": 289.2, "completions/mean_length": 205.075, "completions/mean_terminated_length": 205.075, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.1055948160588742, "epoch": 0.3677484600533235, "frac_reward_zero_std": 0.5, "grad_norm": 9.79196548461914, "kl": 0.010652874858351424, "learning_rate": 4.9284141286520245e-06, "loss": 0.0008, "num_tokens": 17370670.0, "reward": -0.075, "reward_std": 0.45536120533943175, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.45536120533943175, "sampling/importance_sampling_ratio/max": 1.251124334335327, "sampling/importance_sampling_ratio/mean": 1.0001270949840546, "sampling/importance_sampling_ratio/min": 0.784620201587677, "sampling/sampling_logp_difference/max": 0.26241936087608336, "sampling/sampling_logp_difference/mean": 0.021410531364381313, "step": 4000 }, { "epoch": 0.3677484600533235, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0032722513089005235, "eval_completions/max_length": 306.804537521815, "eval_completions/max_terminated_length": 304.1413612565445, "eval_completions/mean_length": 211.55584642233856, "eval_completions/mean_terminated_length": 210.80035128768202, "eval_completions/min_length": 134.0977312390925, "eval_completions/min_terminated_length": 134.0977312390925, "eval_entropy": 1.0860569851040631, "eval_frac_reward_zero_std": 0.4677137870855148, "eval_kl": 0.01145308680936408, "eval_loss": 0.0005863785627298057, "eval_num_tokens": 17370670.0, "eval_reward": -0.11267452006980803, "eval_reward_std": 0.45785783938608454, "eval_rewards/reward_fn/mean": -0.11267452006980803, "eval_rewards/reward_fn/std": 0.45785785478135055, "eval_runtime": 1781.4865, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.2885760778323934, "eval_sampling/importance_sampling_ratio/mean": 1.0000189693602384, "eval_sampling/importance_sampling_ratio/min": 0.768478961567188, "eval_sampling/sampling_logp_difference/max": 0.30472261322106364, "eval_sampling/sampling_logp_difference/mean": 0.02125141721862461, "eval_steps_per_second": 0.04, "step": 4000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 214.6875, "completions/mean_terminated_length": 214.6875, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 0.9379271060228348, "epoch": 0.3686678312034568, "frac_reward_zero_std": 0.5, "grad_norm": 0.1326177567243576, "kl": 0.013651389826554805, "learning_rate": 4.92781091915855e-06, "loss": 0.0006, "num_tokens": 17413229.0, "reward": 0.2, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.2806268334388733, "sampling/importance_sampling_ratio/mean": 0.9997260510921478, "sampling/importance_sampling_ratio/min": 0.7807656586170196, "sampling/sampling_logp_difference/max": 0.2857259541749954, "sampling/sampling_logp_difference/mean": 0.01963389217853546, "step": 4010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.6, "completions/max_terminated_length": 290.4, "completions/mean_length": 224.625, "completions/mean_terminated_length": 221.05357208251954, "completions/min_length": 152.5, "completions/min_terminated_length": 152.5, "entropy": 1.1070492438971997, "epoch": 0.36958720235359016, "frac_reward_zero_std": 0.3, "grad_norm": 0.08208726346492767, "kl": 0.012464217148954048, "learning_rate": 4.927205216108649e-06, "loss": 0.0002, "num_tokens": 17459215.0, "reward": 0.05625, "reward_std": 0.6291591688990593, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.6291591867804527, "sampling/importance_sampling_ratio/max": 1.232274091243744, "sampling/importance_sampling_ratio/mean": 0.9997148275375366, "sampling/importance_sampling_ratio/min": 0.7674962699413299, "sampling/sampling_logp_difference/max": 0.2747806191444397, "sampling/sampling_logp_difference/mean": 0.02136263679713011, "step": 4020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.5, "completions/max_terminated_length": 313.3, "completions/mean_length": 220.35, "completions/mean_terminated_length": 217.04285888671876, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.1311936505138873, "epoch": 0.37050657350372346, "frac_reward_zero_std": 0.4, "grad_norm": 0.18091891705989838, "kl": 0.011017594014992938, "learning_rate": 4.926597020124428e-06, "loss": 0.0006, "num_tokens": 17506443.0, "reward": -0.4875, "reward_std": 0.48603986501693724, "rewards/reward_fn/mean": -0.4875, "rewards/reward_fn/std": 0.4860398888587952, "sampling/importance_sampling_ratio/max": 1.2518605709075927, "sampling/importance_sampling_ratio/mean": 1.0000186681747436, "sampling/importance_sampling_ratio/min": 0.7436865374445916, "sampling/sampling_logp_difference/max": 0.3696718573570251, "sampling/sampling_logp_difference/mean": 0.021227880008518697, "step": 4030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.6, "completions/max_terminated_length": 312.6, "completions/mean_length": 206.0875, "completions/mean_terminated_length": 206.0875, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.003322561085224, "epoch": 0.37142594465385675, "frac_reward_zero_std": 0.7, "grad_norm": 5.141659736633301, "kl": 0.01073904157965444, "learning_rate": 4.92598633183055e-06, "loss": 0.0006, "num_tokens": 17545978.0, "reward": -0.075, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2869314789772033, "sampling/importance_sampling_ratio/mean": 1.0004750430583953, "sampling/importance_sampling_ratio/min": 0.7708701968193055, "sampling/sampling_logp_difference/max": 0.2706547021865845, "sampling/sampling_logp_difference/mean": 0.02076073009520769, "step": 4040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.5, "completions/max_terminated_length": 270.5, "completions/mean_length": 194.5625, "completions/mean_terminated_length": 194.5625, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0719862148165702, "epoch": 0.37234531580399005, "frac_reward_zero_std": 0.4, "grad_norm": 6.145559310913086, "kl": 0.01108819105429575, "learning_rate": 4.925373151854244e-06, "loss": 0.0006, "num_tokens": 17586607.0, "reward": 0.55, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.55, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2270849227905274, "sampling/importance_sampling_ratio/mean": 1.000012856721878, "sampling/importance_sampling_ratio/min": 0.7706259727478028, "sampling/sampling_logp_difference/max": 0.2821680009365082, "sampling/sampling_logp_difference/mean": 0.021602493710815907, "step": 4050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.9, "completions/max_terminated_length": 327.9, "completions/mean_length": 207.5, "completions/mean_terminated_length": 207.5, "completions/min_length": 134.7, "completions/min_terminated_length": 134.7, "entropy": 1.0814829856157302, "epoch": 0.3732646869541234, "frac_reward_zero_std": 0.6, "grad_norm": 0.08712005615234375, "kl": 0.01206351889995858, "learning_rate": 4.924757480825292e-06, "loss": 0.0008, "num_tokens": 17627351.0, "reward": -0.45, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2508814215660096, "sampling/importance_sampling_ratio/mean": 1.0002462565898895, "sampling/importance_sampling_ratio/min": 0.7746960163116455, "sampling/sampling_logp_difference/max": 0.2708323299884796, "sampling/sampling_logp_difference/mean": 0.02106236182153225, "step": 4060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.3, "completions/max_terminated_length": 309.3, "completions/mean_length": 202.725, "completions/mean_terminated_length": 202.725, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0107815206050872, "epoch": 0.3741840581042567, "frac_reward_zero_std": 0.6, "grad_norm": 0.12375466525554657, "kl": 0.012315103638684377, "learning_rate": 4.92413931937604e-06, "loss": 0.0001, "num_tokens": 17671025.0, "reward": -0.55, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2825764179229737, "sampling/importance_sampling_ratio/mean": 0.9999566078186035, "sampling/importance_sampling_ratio/min": 0.7310586273670197, "sampling/sampling_logp_difference/max": 0.36065384149551394, "sampling/sampling_logp_difference/mean": 0.020056076534092426, "step": 4070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 229.125, "completions/mean_terminated_length": 229.125, "completions/min_length": 158.1, "completions/min_terminated_length": 158.1, "entropy": 1.2004849590361117, "epoch": 0.37510342925439, "frac_reward_zero_std": 0.6, "grad_norm": 0.1141628846526146, "kl": 0.011857985134702176, "learning_rate": 4.923518668141388e-06, "loss": 0.0008, "num_tokens": 17717083.0, "reward": -0.625, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2674716114997864, "sampling/importance_sampling_ratio/mean": 0.9997752428054809, "sampling/importance_sampling_ratio/min": 0.7614108204841614, "sampling/sampling_logp_difference/max": 0.2960360586643219, "sampling/sampling_logp_difference/mean": 0.02261939849704504, "step": 4080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 202.8375, "completions/mean_terminated_length": 202.8375, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.082180816680193, "epoch": 0.3760228004045233, "frac_reward_zero_std": 0.5, "grad_norm": 0.11117101460695267, "kl": 0.012720558274304494, "learning_rate": 4.922895527758795e-06, "loss": 0.0006, "num_tokens": 17760558.0, "reward": 0.1125, "reward_std": 0.505875951051712, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.505875962972641, "sampling/importance_sampling_ratio/max": 1.268252444267273, "sampling/importance_sampling_ratio/mean": 0.9994777500629425, "sampling/importance_sampling_ratio/min": 0.7630319416522979, "sampling/sampling_logp_difference/max": 0.29381234645843507, "sampling/sampling_logp_difference/mean": 0.021012330055236818, "step": 4090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.0, "completions/max_terminated_length": 286.0, "completions/mean_length": 198.1875, "completions/mean_terminated_length": 198.1875, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0959155306220054, "epoch": 0.3769421715546566, "frac_reward_zero_std": 0.4, "grad_norm": 0.09819050133228302, "kl": 0.011994760984089225, "learning_rate": 4.9222698988682775e-06, "loss": 0.0005, "num_tokens": 17803093.0, "reward": 0.325, "reward_std": 0.5622657001018524, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.5622657001018524, "sampling/importance_sampling_ratio/max": 1.246112596988678, "sampling/importance_sampling_ratio/mean": 1.000052458047867, "sampling/importance_sampling_ratio/min": 0.7790269792079926, "sampling/sampling_logp_difference/max": 0.2594437122344971, "sampling/sampling_logp_difference/mean": 0.021487616188824176, "step": 4100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 187.025, "completions/mean_terminated_length": 187.025, "completions/min_length": 119.9, "completions/min_terminated_length": 119.9, "entropy": 1.0451099932193757, "epoch": 0.37786154270478994, "frac_reward_zero_std": 0.5, "grad_norm": 0.12486845254898071, "kl": 0.012765026185661554, "learning_rate": 4.921641782112404e-06, "loss": 0.0006, "num_tokens": 17844831.0, "reward": -0.075, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.3876115083694458, "sampling/importance_sampling_ratio/mean": 0.9997731387615204, "sampling/importance_sampling_ratio/min": 0.7840146481990814, "sampling/sampling_logp_difference/max": 0.37284798324108126, "sampling/sampling_logp_difference/mean": 0.02120126448571682, "step": 4110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.8, "completions/max_terminated_length": 297.8, "completions/mean_length": 201.8875, "completions/mean_terminated_length": 201.8875, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 1.0580450899899005, "epoch": 0.37878091385492324, "frac_reward_zero_std": 0.6, "grad_norm": 0.15059125423431396, "kl": 0.01131479927280452, "learning_rate": 4.921011178136302e-06, "loss": 0.0008, "num_tokens": 17887398.0, "reward": 0.35, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2828755140304566, "sampling/importance_sampling_ratio/mean": 1.0002075850963592, "sampling/importance_sampling_ratio/min": 0.797919875383377, "sampling/sampling_logp_difference/max": 0.27245804071426394, "sampling/sampling_logp_difference/mean": 0.02044024523347616, "step": 4120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.8, "completions/max_terminated_length": 316.8, "completions/mean_length": 216.425, "completions/mean_terminated_length": 216.425, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.056341578066349, "epoch": 0.37970028500505654, "frac_reward_zero_std": 0.4, "grad_norm": 0.12152888625860214, "kl": 0.012885672663105651, "learning_rate": 4.920378087587652e-06, "loss": 0.0007, "num_tokens": 17931808.0, "reward": 0.025, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.2678850412368774, "sampling/importance_sampling_ratio/mean": 0.9997739970684052, "sampling/importance_sampling_ratio/min": 0.7948495268821716, "sampling/sampling_logp_difference/max": 0.25814027786254884, "sampling/sampling_logp_difference/mean": 0.021409266628324986, "step": 4130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.0, "completions/max_terminated_length": 320.0, "completions/mean_length": 209.2875, "completions/mean_terminated_length": 209.2875, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 1.0855748500674962, "epoch": 0.38061965615518983, "frac_reward_zero_std": 0.4, "grad_norm": 5.952367305755615, "kl": 0.010832431650487705, "learning_rate": 4.91974251111669e-06, "loss": 0.0004, "num_tokens": 17974431.0, "reward": -0.2, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.230693233013153, "sampling/importance_sampling_ratio/mean": 0.9997909784317016, "sampling/importance_sampling_ratio/min": 0.7730068027973175, "sampling/sampling_logp_difference/max": 0.2638783097267151, "sampling/sampling_logp_difference/mean": 0.02081681741401553, "step": 4140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 215.85, "completions/mean_terminated_length": 215.85, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 1.1054238021373748, "epoch": 0.3815390273053232, "frac_reward_zero_std": 0.6, "grad_norm": 0.1304997056722641, "kl": 0.011894141446100548, "learning_rate": 4.919104449376202e-06, "loss": 0.0007, "num_tokens": 18017291.0, "reward": -0.1, "reward_std": 0.38124017119407655, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3812402069568634, "sampling/importance_sampling_ratio/max": 1.2807276725769043, "sampling/importance_sampling_ratio/mean": 1.000231969356537, "sampling/importance_sampling_ratio/min": 0.7926554620265961, "sampling/sampling_logp_difference/max": 0.25929675698280336, "sampling/sampling_logp_difference/mean": 0.02155719920992851, "step": 4150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 219.075, "completions/mean_terminated_length": 219.075, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.1082587331533431, "epoch": 0.3824583984554565, "frac_reward_zero_std": 0.3, "grad_norm": 5.337881565093994, "kl": 0.013104151305742561, "learning_rate": 4.9184639030215295e-06, "loss": 0.0008, "num_tokens": 18062521.0, "reward": -0.2, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2977920055389405, "sampling/importance_sampling_ratio/mean": 0.9999036669731141, "sampling/importance_sampling_ratio/min": 0.7778182208538056, "sampling/sampling_logp_difference/max": 0.30054478645324706, "sampling/sampling_logp_difference/mean": 0.020784492790699004, "step": 4160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.7, "completions/max_terminated_length": 329.7, "completions/mean_length": 220.225, "completions/mean_terminated_length": 220.225, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.1240836285054683, "epoch": 0.3833777696055898, "frac_reward_zero_std": 0.4, "grad_norm": 0.12941135466098785, "kl": 0.012018622038885951, "learning_rate": 4.917820872710566e-06, "loss": 0.0005, "num_tokens": 18108147.0, "reward": -0.475, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2360854506492616, "sampling/importance_sampling_ratio/mean": 0.9998351395130157, "sampling/importance_sampling_ratio/min": 0.7328603953123093, "sampling/sampling_logp_difference/max": 0.3461729407310486, "sampling/sampling_logp_difference/mean": 0.021286931075155734, "step": 4170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 216.6125, "completions/mean_terminated_length": 216.6125, "completions/min_length": 145.3, "completions/min_terminated_length": 145.3, "entropy": 1.1534373931586743, "epoch": 0.3842971407557231, "frac_reward_zero_std": 0.4, "grad_norm": 6.234537601470947, "kl": 0.013226597604807466, "learning_rate": 4.917175359103753e-06, "loss": 0.0005, "num_tokens": 18152396.0, "reward": -0.6, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.3476779460906982, "sampling/importance_sampling_ratio/mean": 0.9999447286128997, "sampling/importance_sampling_ratio/min": 0.7461422502994537, "sampling/sampling_logp_difference/max": 0.37134329080581663, "sampling/sampling_logp_difference/mean": 0.02234238162636757, "step": 4180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 205.625, "completions/mean_terminated_length": 205.625, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.039468516409397, "epoch": 0.3852165119058564, "frac_reward_zero_std": 0.3, "grad_norm": 0.11548010259866714, "kl": 0.011638339233468287, "learning_rate": 4.9165273628640865e-06, "loss": 0.0004, "num_tokens": 18194374.0, "reward": 0.275, "reward_std": 0.5277738928794861, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.5277739048004151, "sampling/importance_sampling_ratio/max": 1.2770779013633728, "sampling/importance_sampling_ratio/mean": 1.0000529646873475, "sampling/importance_sampling_ratio/min": 0.7624103844165802, "sampling/sampling_logp_difference/max": 0.30358039140701293, "sampling/sampling_logp_difference/mean": 0.020696807838976383, "step": 4190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 188.0625, "completions/mean_terminated_length": 188.0625, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0592534735798835, "epoch": 0.3861358830559897, "frac_reward_zero_std": 0.4, "grad_norm": 0.09075111150741577, "kl": 0.013763274328084663, "learning_rate": 4.91587688465711e-06, "loss": 0.0005, "num_tokens": 18237851.0, "reward": 0.025, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.2451142191886901, "sampling/importance_sampling_ratio/mean": 1.0000199377536774, "sampling/importance_sampling_ratio/min": 0.7608016729354858, "sampling/sampling_logp_difference/max": 0.29567856788635255, "sampling/sampling_logp_difference/mean": 0.0212363101541996, "step": 4200 }, { "epoch": 0.3861358830559897, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.004363001745200698, "eval_completions/max_length": 307.06108202443284, "eval_completions/max_terminated_length": 303.20942408376965, "eval_completions/mean_length": 213.26003490401396, "eval_completions/mean_terminated_length": 212.2279151530374, "eval_completions/min_length": 138.9371727748691, "eval_completions/min_terminated_length": 138.9371727748691, "eval_entropy": 1.0945908834484859, "eval_frac_reward_zero_std": 0.45898778359511344, "eval_kl": 0.012057336564557741, "eval_loss": 0.0005685599753633142, "eval_num_tokens": 18237851.0, "eval_reward": -0.12358202443280977, "eval_reward_std": 0.4730130861827008, "eval_rewards/reward_fn/mean": -0.12358202443280977, "eval_rewards/reward_fn/std": 0.4730131013699227, "eval_runtime": 1777.9101, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.2748564254967123, "eval_sampling/importance_sampling_ratio/mean": 1.0000155767845234, "eval_sampling/importance_sampling_ratio/min": 0.7712137476519972, "eval_sampling/sampling_logp_difference/max": 0.2976636046098373, "eval_sampling/sampling_logp_difference/mean": 0.020921697617422328, "eval_steps_per_second": 0.04, "step": 4200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.3, "completions/max_terminated_length": 319.3, "completions/mean_length": 223.3, "completions/mean_terminated_length": 223.3, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0600866697728635, "epoch": 0.387055254206123, "frac_reward_zero_std": 0.5, "grad_norm": 0.10454141348600388, "kl": 0.01526711971964687, "learning_rate": 4.915223925150916e-06, "loss": 0.0006, "num_tokens": 18281115.0, "reward": -0.3375, "reward_std": 0.40861602276563647, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.40861602276563647, "sampling/importance_sampling_ratio/max": 1.2838784337043763, "sampling/importance_sampling_ratio/mean": 0.9997600138187408, "sampling/importance_sampling_ratio/min": 0.7799991607666016, "sampling/sampling_logp_difference/max": 0.2769541025161743, "sampling/sampling_logp_difference/mean": 0.020954769849777222, "step": 4210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.1, "completions/max_terminated_length": 299.1, "completions/mean_length": 212.25, "completions/mean_terminated_length": 212.25, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0776992693543435, "epoch": 0.3879746253562563, "frac_reward_zero_std": 0.6, "grad_norm": 7.177145004272461, "kl": 0.011730249074753374, "learning_rate": 4.914568485016148e-06, "loss": 0.0003, "num_tokens": 18322815.0, "reward": -0.35, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.2586533665657043, "sampling/importance_sampling_ratio/mean": 1.0000215113162993, "sampling/importance_sampling_ratio/min": 0.718192407488823, "sampling/sampling_logp_difference/max": 0.3951353311538696, "sampling/sampling_logp_difference/mean": 0.0211188904941082, "step": 4220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 374.3, "completions/max_terminated_length": 374.3, "completions/mean_length": 254.4375, "completions/mean_terminated_length": 254.4375, "completions/min_length": 162.4, "completions/min_terminated_length": 162.4, "entropy": 1.095973578840494, "epoch": 0.3888939965063896, "frac_reward_zero_std": 0.6, "grad_norm": 5.532623767852783, "kl": 0.01163805605028756, "learning_rate": 4.9139105649259945e-06, "loss": 0.0008, "num_tokens": 18372818.0, "reward": -0.55, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2825661420822143, "sampling/importance_sampling_ratio/mean": 1.0001707911491393, "sampling/importance_sampling_ratio/min": 0.8073943138122559, "sampling/sampling_logp_difference/max": 0.2594639778137207, "sampling/sampling_logp_difference/mean": 0.021088985726237297, "step": 4230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 223.3, "completions/mean_terminated_length": 223.3, "completions/min_length": 149.5, "completions/min_terminated_length": 149.5, "entropy": 1.1725053407251835, "epoch": 0.3898133676565229, "frac_reward_zero_std": 0.5, "grad_norm": 4.715742588043213, "kl": 0.012489797954913229, "learning_rate": 4.913250165556192e-06, "loss": 0.0007, "num_tokens": 18415786.0, "reward": -0.55, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2859978318214416, "sampling/importance_sampling_ratio/mean": 1.000122356414795, "sampling/importance_sampling_ratio/min": 0.7646305084228515, "sampling/sampling_logp_difference/max": 0.3154569029808044, "sampling/sampling_logp_difference/mean": 0.023009051755070686, "step": 4240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.3, "completions/max_terminated_length": 336.3, "completions/mean_length": 228.6625, "completions/mean_terminated_length": 228.6625, "completions/min_length": 157.1, "completions/min_terminated_length": 157.1, "entropy": 1.0988418139517306, "epoch": 0.39073273880665627, "frac_reward_zero_std": 0.7, "grad_norm": 0.1339138150215149, "kl": 0.011893614783184603, "learning_rate": 4.912587287585023e-06, "loss": 0.0005, "num_tokens": 18459735.0, "reward": 0.25, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.241512382030487, "sampling/importance_sampling_ratio/mean": 1.0004823923110961, "sampling/importance_sampling_ratio/min": 0.7782620012760162, "sampling/sampling_logp_difference/max": 0.2597755312919617, "sampling/sampling_logp_difference/mean": 0.02120252139866352, "step": 4250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 347.7, "completions/max_terminated_length": 347.7, "completions/mean_length": 249.1125, "completions/mean_terminated_length": 249.1125, "completions/min_length": 181.0, "completions/min_terminated_length": 181.0, "entropy": 1.155641546100378, "epoch": 0.39165210995678956, "frac_reward_zero_std": 0.2, "grad_norm": 0.09588342905044556, "kl": 0.012260648363735527, "learning_rate": 4.911921931693317e-06, "loss": 0.0011, "num_tokens": 18508560.0, "reward": -0.10625, "reward_std": 0.6253880485892296, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6253880605101585, "sampling/importance_sampling_ratio/max": 1.271811604499817, "sampling/importance_sampling_ratio/mean": 1.0007880091667176, "sampling/importance_sampling_ratio/min": 0.7302525192499161, "sampling/sampling_logp_difference/max": 0.3509999752044678, "sampling/sampling_logp_difference/mean": 0.021550890430808066, "step": 4260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 320.9, "completions/max_terminated_length": 298.4, "completions/mean_length": 221.375, "completions/mean_terminated_length": 217.76964416503907, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.1645951725542545, "epoch": 0.39257148110692286, "frac_reward_zero_std": 0.7, "grad_norm": 0.13313908874988556, "kl": 0.01209413178730756, "learning_rate": 4.91125409856445e-06, "loss": 0.0005, "num_tokens": 18552814.0, "reward": -0.13125, "reward_std": 0.18097035735845565, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.18097035735845565, "sampling/importance_sampling_ratio/max": 1.244187021255493, "sampling/importance_sampling_ratio/mean": 0.9997282445430755, "sampling/importance_sampling_ratio/min": 0.800002908706665, "sampling/sampling_logp_difference/max": 0.24869282245635987, "sampling/sampling_logp_difference/mean": 0.02173096816986799, "step": 4270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 199.4, "completions/mean_terminated_length": 199.4, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.1521642990410328, "epoch": 0.39349085225705616, "frac_reward_zero_std": 0.7, "grad_norm": 0.1121278926730156, "kl": 0.013033400708809494, "learning_rate": 4.910583788884336e-06, "loss": 0.0005, "num_tokens": 18594254.0, "reward": 0.025, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2389193534851075, "sampling/importance_sampling_ratio/mean": 0.9995200157165527, "sampling/importance_sampling_ratio/min": 0.7771966993808747, "sampling/sampling_logp_difference/max": 0.2548762083053589, "sampling/sampling_logp_difference/mean": 0.022655235975980757, "step": 4280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 344.8, "completions/max_terminated_length": 344.8, "completions/mean_length": 225.85, "completions/mean_terminated_length": 225.85, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.027185170352459, "epoch": 0.3944102234071895, "frac_reward_zero_std": 0.5, "grad_norm": 7.017943859100342, "kl": 0.010884489247109741, "learning_rate": 4.90991100334144e-06, "loss": 0.0006, "num_tokens": 18638922.0, "reward": -0.25, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2473251104354859, "sampling/importance_sampling_ratio/mean": 0.9995446383953095, "sampling/importance_sampling_ratio/min": 0.7802755117416382, "sampling/sampling_logp_difference/max": 0.2680867791175842, "sampling/sampling_logp_difference/mean": 0.020302304811775685, "step": 4290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.5, "completions/max_terminated_length": 304.5, "completions/mean_length": 212.7125, "completions/mean_terminated_length": 212.7125, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.1158136904239655, "epoch": 0.3953295945573228, "frac_reward_zero_std": 0.5, "grad_norm": 0.13745172321796417, "kl": 0.011755738058127463, "learning_rate": 4.909235742626765e-06, "loss": 0.0008, "num_tokens": 18685315.0, "reward": -0.125, "reward_std": 0.48816035985946654, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4881603717803955, "sampling/importance_sampling_ratio/max": 1.3109572529792786, "sampling/importance_sampling_ratio/mean": 1.000343143939972, "sampling/importance_sampling_ratio/min": 0.7889728903770447, "sampling/sampling_logp_difference/max": 0.28206329941749575, "sampling/sampling_logp_difference/mean": 0.021576874516904355, "step": 4300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.3, "completions/max_terminated_length": 256.3, "completions/mean_length": 186.525, "completions/mean_terminated_length": 186.525, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.103030838817358, "epoch": 0.3962489657074561, "frac_reward_zero_std": 0.5, "grad_norm": 7.077025413513184, "kl": 0.011198869231157005, "learning_rate": 4.908558007433859e-06, "loss": 0.0005, "num_tokens": 18727085.0, "reward": -0.175, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.239169979095459, "sampling/importance_sampling_ratio/mean": 1.0002214729785919, "sampling/importance_sampling_ratio/min": 0.7796527326107026, "sampling/sampling_logp_difference/max": 0.2538472890853882, "sampling/sampling_logp_difference/mean": 0.021943285316228866, "step": 4310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.3, "completions/max_terminated_length": 292.3, "completions/mean_length": 205.2125, "completions/mean_terminated_length": 205.2125, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.0839948415756226, "epoch": 0.3971683368575894, "frac_reward_zero_std": 0.5, "grad_norm": 4.776386260986328, "kl": 0.011089162767166271, "learning_rate": 4.907877798458809e-06, "loss": 0.0007, "num_tokens": 18771526.0, "reward": -0.025, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2324298739433288, "sampling/importance_sampling_ratio/mean": 0.9993818163871765, "sampling/importance_sampling_ratio/min": 0.7728717029094696, "sampling/sampling_logp_difference/max": 0.2745970368385315, "sampling/sampling_logp_difference/mean": 0.021205469593405725, "step": 4320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 293.4, "completions/max_terminated_length": 290.9, "completions/mean_length": 214.8625, "completions/mean_terminated_length": 201.55, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0057868413627147, "epoch": 0.3980877080077227, "frac_reward_zero_std": 0.5, "grad_norm": 0.09839917719364166, "kl": 0.010004157363437117, "learning_rate": 4.907195116400245e-06, "loss": 0.0006, "num_tokens": 18814211.0, "reward": 0.1125, "reward_std": 0.3825300335884094, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.3825300455093384, "sampling/importance_sampling_ratio/max": 1.29884512424469, "sampling/importance_sampling_ratio/mean": 0.9997987508773803, "sampling/importance_sampling_ratio/min": 0.7863820374011994, "sampling/sampling_logp_difference/max": 0.28178919553756715, "sampling/sampling_logp_difference/mean": 0.020872960239648818, "step": 4330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 211.3125, "completions/mean_terminated_length": 211.3125, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.0354847960174083, "epoch": 0.39900707915785605, "frac_reward_zero_std": 0.3, "grad_norm": 5.566727638244629, "kl": 0.011985427490435541, "learning_rate": 4.906509961959337e-06, "loss": 0.0001, "num_tokens": 18858660.0, "reward": -0.10625, "reward_std": 0.6341523706912995, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6341524124145508, "sampling/importance_sampling_ratio/max": 1.2798049569129943, "sampling/importance_sampling_ratio/mean": 1.0003681540489198, "sampling/importance_sampling_ratio/min": 0.7635370373725892, "sampling/sampling_logp_difference/max": 0.2934330224990845, "sampling/sampling_logp_difference/mean": 0.020594934560358526, "step": 4340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 301.4, "completions/max_terminated_length": 271.9, "completions/mean_length": 206.1375, "completions/mean_terminated_length": 195.92559814453125, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.1165982738137246, "epoch": 0.39992645030798935, "frac_reward_zero_std": 0.2, "grad_norm": 6.254741668701172, "kl": 0.01411072441842407, "learning_rate": 4.9058223358397924e-06, "loss": 0.001, "num_tokens": 18902871.0, "reward": -0.5875, "reward_std": 0.6555967152118682, "rewards/reward_fn/mean": -0.5875, "rewards/reward_fn/std": 0.6555967271327973, "sampling/importance_sampling_ratio/max": 1.2442841172218322, "sampling/importance_sampling_ratio/mean": 1.0000369668006897, "sampling/importance_sampling_ratio/min": 0.7693053424358368, "sampling/sampling_logp_difference/max": 0.2733058571815491, "sampling/sampling_logp_difference/mean": 0.021927562542259693, "step": 4350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 204.5875, "completions/mean_terminated_length": 204.5875, "completions/min_length": 122.2, "completions/min_terminated_length": 122.2, "entropy": 1.1031477630138398, "epoch": 0.40084582145812264, "frac_reward_zero_std": 0.6, "grad_norm": 0.1592874377965927, "kl": 0.01218166805920191, "learning_rate": 4.905132238747858e-06, "loss": 0.0006, "num_tokens": 18945118.0, "reward": -0.275, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.264967679977417, "sampling/importance_sampling_ratio/mean": 0.9999442279338837, "sampling/importance_sampling_ratio/min": 0.7812796175479889, "sampling/sampling_logp_difference/max": 0.26165342330932617, "sampling/sampling_logp_difference/mean": 0.02051820568740368, "step": 4360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.2, "completions/max_terminated_length": 267.2, "completions/mean_length": 199.2, "completions/mean_terminated_length": 199.2, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.083102073520422, "epoch": 0.40176519260825594, "frac_reward_zero_std": 0.5, "grad_norm": 6.6612749099731445, "kl": 0.011797474417835473, "learning_rate": 4.904439671392321e-06, "loss": 0.0004, "num_tokens": 18986974.0, "reward": -0.175, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2790374755859375, "sampling/importance_sampling_ratio/mean": 0.9998512446880341, "sampling/importance_sampling_ratio/min": 0.7073204189538955, "sampling/sampling_logp_difference/max": 0.385969877243042, "sampling/sampling_logp_difference/mean": 0.02137984074652195, "step": 4370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.4, "completions/max_terminated_length": 338.4, "completions/mean_length": 220.2875, "completions/mean_terminated_length": 220.2875, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.0785880103707313, "epoch": 0.40268456375838924, "frac_reward_zero_std": 0.5, "grad_norm": 5.293278217315674, "kl": 0.012535612931242213, "learning_rate": 4.9037446344845e-06, "loss": 0.0005, "num_tokens": 19033005.0, "reward": 0.025, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.308669126033783, "sampling/importance_sampling_ratio/mean": 0.9993424236774444, "sampling/importance_sampling_ratio/min": 0.7581714451313019, "sampling/sampling_logp_difference/max": 0.36804680824279784, "sampling/sampling_logp_difference/mean": 0.02064927350729704, "step": 4380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.6, "completions/max_terminated_length": 323.6, "completions/mean_length": 230.075, "completions/mean_terminated_length": 230.075, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.1193446546792984, "epoch": 0.4036039349085226, "frac_reward_zero_std": 0.3, "grad_norm": 5.467469215393066, "kl": 0.010905437293695286, "learning_rate": 4.903047128738257e-06, "loss": 0.0002, "num_tokens": 19080259.0, "reward": -0.25625, "reward_std": 0.6686686933040619, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.6686687171459198, "sampling/importance_sampling_ratio/max": 1.2514415383338928, "sampling/importance_sampling_ratio/mean": 0.9999465703964233, "sampling/importance_sampling_ratio/min": 0.7366574466228485, "sampling/sampling_logp_difference/max": 0.3229207545518875, "sampling/sampling_logp_difference/mean": 0.021195818856358527, "step": 4390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.9, "completions/max_terminated_length": 328.9, "completions/mean_length": 224.525, "completions/mean_terminated_length": 224.525, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.087701115012169, "epoch": 0.4045233060586559, "frac_reward_zero_std": 0.4, "grad_norm": 6.165444850921631, "kl": 0.013511087035294623, "learning_rate": 4.902347154869984e-06, "loss": 0.0008, "num_tokens": 19125453.0, "reward": -0.53125, "reward_std": 0.4620953604578972, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.46209537237882614, "sampling/importance_sampling_ratio/max": 1.2724752068519591, "sampling/importance_sampling_ratio/mean": 1.000162410736084, "sampling/importance_sampling_ratio/min": 0.8019136488437653, "sampling/sampling_logp_difference/max": 0.26145162582397463, "sampling/sampling_logp_difference/mean": 0.02060892414301634, "step": 4400 }, { "epoch": 0.4045233060586559, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 307.8935427574171, "eval_completions/max_terminated_length": 304.95986038394415, "eval_completions/mean_length": 210.87914485165794, "eval_completions/mean_terminated_length": 210.17566293446805, "eval_completions/min_length": 132.8324607329843, "eval_completions/min_terminated_length": 132.8324607329843, "eval_entropy": 1.0735425944415686, "eval_frac_reward_zero_std": 0.4467713787085515, "eval_kl": 0.01207038810327129, "eval_loss": 0.0006343135610222816, "eval_num_tokens": 19125453.0, "eval_reward": -0.13099912739965097, "eval_reward_std": 0.4808651658773006, "eval_rewards/reward_fn/mean": -0.13099912739965097, "eval_rewards/reward_fn/std": 0.4808651810645225, "eval_runtime": 1784.0936, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.2770484371751183, "eval_sampling/importance_sampling_ratio/mean": 0.9999512127140637, "eval_sampling/importance_sampling_ratio/min": 0.765700128664105, "eval_sampling/sampling_logp_difference/max": 0.3012712300343871, "eval_sampling/sampling_logp_difference/mean": 0.021039598369499566, "eval_steps_per_second": 0.04, "step": 4400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 201.65, "completions/mean_terminated_length": 201.65, "completions/min_length": 107.2, "completions/min_terminated_length": 107.2, "entropy": 1.054637898504734, "epoch": 0.4054426772087892, "frac_reward_zero_std": 0.4, "grad_norm": 21.120046615600586, "kl": 0.013947097142226994, "learning_rate": 4.901644713598612e-06, "loss": 0.0002, "num_tokens": 19165265.0, "reward": -0.20625, "reward_std": 0.4869155645370483, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.48691557049751283, "sampling/importance_sampling_ratio/max": 1.2910923957824707, "sampling/importance_sampling_ratio/mean": 1.0002331376075744, "sampling/importance_sampling_ratio/min": 0.782300341129303, "sampling/sampling_logp_difference/max": 0.2835899591445923, "sampling/sampling_logp_difference/mean": 0.02086752038449049, "step": 4410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.5, "completions/max_terminated_length": 265.5, "completions/mean_length": 188.6125, "completions/mean_terminated_length": 188.6125, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 0.9634748078882694, "epoch": 0.4063620483589225, "frac_reward_zero_std": 0.3, "grad_norm": 6.261925220489502, "kl": 0.01143052204279229, "learning_rate": 4.9009398056456045e-06, "loss": 0.0007, "num_tokens": 19205330.0, "reward": -0.28125, "reward_std": 0.5546773716807365, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.5546773836016655, "sampling/importance_sampling_ratio/max": 1.2544040203094482, "sampling/importance_sampling_ratio/mean": 0.999969232082367, "sampling/importance_sampling_ratio/min": 0.7617864191532135, "sampling/sampling_logp_difference/max": 0.2847472667694092, "sampling/sampling_logp_difference/mean": 0.02064214423298836, "step": 4420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.9, "completions/max_terminated_length": 284.9, "completions/mean_length": 204.1125, "completions/mean_terminated_length": 204.1125, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.1532613269984722, "epoch": 0.40728141950905583, "frac_reward_zero_std": 0.3, "grad_norm": 8.555257797241211, "kl": 0.01233347415109165, "learning_rate": 4.900232431734957e-06, "loss": 0.0008, "num_tokens": 19248731.0, "reward": -0.20625, "reward_std": 0.5659772038459778, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.5659772098064423, "sampling/importance_sampling_ratio/max": 1.2804153561592102, "sampling/importance_sampling_ratio/mean": 1.0002202689647675, "sampling/importance_sampling_ratio/min": 0.7930121779441833, "sampling/sampling_logp_difference/max": 0.26058908700942995, "sampling/sampling_logp_difference/mean": 0.022867058403789997, "step": 4430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 343.8, "completions/max_terminated_length": 341.7, "completions/mean_length": 234.525, "completions/mean_terminated_length": 232.5875030517578, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.0572208173573017, "epoch": 0.4082007906591891, "frac_reward_zero_std": 0.4, "grad_norm": 7.996652126312256, "kl": 0.015180602233158424, "learning_rate": 4.899522592593199e-06, "loss": 0.0007, "num_tokens": 19291701.0, "reward": -0.40625, "reward_std": 0.4402240261435509, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.44022403806447985, "sampling/importance_sampling_ratio/max": 1.3572595238685607, "sampling/importance_sampling_ratio/mean": 1.000078672170639, "sampling/importance_sampling_ratio/min": 0.7825940072536468, "sampling/sampling_logp_difference/max": 0.31523709297180175, "sampling/sampling_logp_difference/mean": 0.020729813538491727, "step": 4440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.0, "completions/max_terminated_length": 327.0, "completions/mean_length": 219.225, "completions/mean_terminated_length": 219.225, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1080649442970754, "epoch": 0.4091201618093224, "frac_reward_zero_std": 0.3, "grad_norm": 7.530383110046387, "kl": 0.012809316487982869, "learning_rate": 4.898810288949394e-06, "loss": 0.0006, "num_tokens": 19336471.0, "reward": -0.2, "reward_std": 0.6371305644512176, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6371305763721467, "sampling/importance_sampling_ratio/max": 1.2586910605430603, "sampling/importance_sampling_ratio/mean": 1.0003853559494018, "sampling/importance_sampling_ratio/min": 0.7825205624103546, "sampling/sampling_logp_difference/max": 0.2588387131690979, "sampling/sampling_logp_difference/mean": 0.021876754984259606, "step": 4450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.6, "completions/max_terminated_length": 319.6, "completions/mean_length": 217.55, "completions/mean_terminated_length": 217.55, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.0458416778594255, "epoch": 0.4100395329594557, "frac_reward_zero_std": 0.7, "grad_norm": 0.11887912452220917, "kl": 0.011899195978185163, "learning_rate": 4.898095521535134e-06, "loss": 0.0006, "num_tokens": 19382275.0, "reward": -0.275, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2498707532882691, "sampling/importance_sampling_ratio/mean": 0.9993833482265473, "sampling/importance_sampling_ratio/min": 0.7394495218992233, "sampling/sampling_logp_difference/max": 0.34876590967178345, "sampling/sampling_logp_difference/mean": 0.019987204112112522, "step": 4460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 345.3, "completions/max_terminated_length": 345.3, "completions/mean_length": 230.4125, "completions/mean_terminated_length": 230.4125, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.1458613954484462, "epoch": 0.410958904109589, "frac_reward_zero_std": 0.1, "grad_norm": 5.5895819664001465, "kl": 0.016070033609867095, "learning_rate": 4.897378291084543e-06, "loss": 0.0008, "num_tokens": 19431756.0, "reward": 0.16875, "reward_std": 0.7541638776659966, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.7541638895869255, "sampling/importance_sampling_ratio/max": 1.2776301145553588, "sampling/importance_sampling_ratio/mean": 0.9998882591724396, "sampling/importance_sampling_ratio/min": 0.7529747188091278, "sampling/sampling_logp_difference/max": 0.3097389698028564, "sampling/sampling_logp_difference/mean": 0.021813632734119893, "step": 4470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.8, "completions/max_terminated_length": 321.8, "completions/mean_length": 229.7375, "completions/mean_terminated_length": 229.7375, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.1282578885555268, "epoch": 0.41187827525972237, "frac_reward_zero_std": 0.3, "grad_norm": 5.520795822143555, "kl": 0.01391836158290971, "learning_rate": 4.896658598334273e-06, "loss": 0.0006, "num_tokens": 19479423.0, "reward": 0.175, "reward_std": 0.6514373540878295, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6514373898506165, "sampling/importance_sampling_ratio/max": 1.3530410051345825, "sampling/importance_sampling_ratio/mean": 1.0000315248966216, "sampling/importance_sampling_ratio/min": 0.7563826680183411, "sampling/sampling_logp_difference/max": 0.3790202647447586, "sampling/sampling_logp_difference/mean": 0.021841386146843434, "step": 4480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.1, "completions/max_terminated_length": 287.9, "completions/mean_length": 218.1875, "completions/mean_terminated_length": 215.2339294433594, "completions/min_length": 141.1, "completions/min_terminated_length": 141.1, "entropy": 1.079378540813923, "epoch": 0.41279764640985567, "frac_reward_zero_std": 0.7, "grad_norm": 6.569296360015869, "kl": 0.013282178697409108, "learning_rate": 4.8959364440235065e-06, "loss": 0.0007, "num_tokens": 19522582.0, "reward": 0.36875, "reward_std": 0.24951753616333008, "rewards/reward_fn/mean": 0.36875, "rewards/reward_fn/std": 0.2495175540447235, "sampling/importance_sampling_ratio/max": 1.293501353263855, "sampling/importance_sampling_ratio/mean": 1.0002484261989593, "sampling/importance_sampling_ratio/min": 0.7993585228919983, "sampling/sampling_logp_difference/max": 0.27202072739601135, "sampling/sampling_logp_difference/mean": 0.021076484024524687, "step": 4490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 190.525, "completions/mean_terminated_length": 190.525, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.1258207619190217, "epoch": 0.41371701755998896, "frac_reward_zero_std": 0.4, "grad_norm": 0.2600265443325043, "kl": 0.01348825006862171, "learning_rate": 4.8952118288939555e-06, "loss": 0.0008, "num_tokens": 19565536.0, "reward": -0.4, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.278635609149933, "sampling/importance_sampling_ratio/mean": 0.9990839183330535, "sampling/importance_sampling_ratio/min": 0.7550171673297882, "sampling/sampling_logp_difference/max": 0.3142871856689453, "sampling/sampling_logp_difference/mean": 0.021862667240202426, "step": 4500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 214.55, "completions/mean_terminated_length": 214.55, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0995950452983378, "epoch": 0.41463638871012226, "frac_reward_zero_std": 0.4, "grad_norm": 5.134932041168213, "kl": 0.014175607077777385, "learning_rate": 4.894484753689855e-06, "loss": 0.0008, "num_tokens": 19606668.0, "reward": -0.05, "reward_std": 0.5588710367679596, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5588710486888886, "sampling/importance_sampling_ratio/max": 1.2458376169204712, "sampling/importance_sampling_ratio/mean": 1.000057464838028, "sampling/importance_sampling_ratio/min": 0.7607628047466278, "sampling/sampling_logp_difference/max": 0.2874214231967926, "sampling/sampling_logp_difference/mean": 0.021476989053189753, "step": 4510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 213.8875, "completions/mean_terminated_length": 213.8875, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0883387744426727, "epoch": 0.41555575986025556, "frac_reward_zero_std": 0.4, "grad_norm": 0.12910139560699463, "kl": 0.012850678397808225, "learning_rate": 4.8937552191579716e-06, "loss": 0.0009, "num_tokens": 19651059.0, "reward": -0.25, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2649874448776246, "sampling/importance_sampling_ratio/mean": 0.9998495399951934, "sampling/importance_sampling_ratio/min": 0.7777285695075988, "sampling/sampling_logp_difference/max": 0.28540799021720886, "sampling/sampling_logp_difference/mean": 0.021311002783477306, "step": 4520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 358.5, "completions/max_terminated_length": 330.6, "completions/mean_length": 215.5875, "completions/mean_terminated_length": 211.5232147216797, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1005850806832314, "epoch": 0.4164751310103889, "frac_reward_zero_std": 0.5, "grad_norm": 0.1224328801035881, "kl": 0.013844084832817316, "learning_rate": 4.893023226047595e-06, "loss": 0.0005, "num_tokens": 19696522.0, "reward": -0.20625, "reward_std": 0.4573550045490265, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.4573550224304199, "sampling/importance_sampling_ratio/max": 1.2724144816398621, "sampling/importance_sampling_ratio/mean": 0.999883908033371, "sampling/importance_sampling_ratio/min": 0.7878267645835877, "sampling/sampling_logp_difference/max": 0.26369296908378603, "sampling/sampling_logp_difference/mean": 0.021296542882919312, "step": 4530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.6, "completions/max_terminated_length": 286.6, "completions/mean_length": 205.8625, "completions/mean_terminated_length": 205.8625, "completions/min_length": 117.7, "completions/min_terminated_length": 117.7, "entropy": 1.0530663207173347, "epoch": 0.4173945021605222, "frac_reward_zero_std": 0.4, "grad_norm": 0.21902485191822052, "kl": 0.015151386911747977, "learning_rate": 4.892288775110542e-06, "loss": 0.0011, "num_tokens": 19737071.0, "reward": -0.225, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.290149176120758, "sampling/importance_sampling_ratio/mean": 1.0002037644386292, "sampling/importance_sampling_ratio/min": 0.8047481656074524, "sampling/sampling_logp_difference/max": 0.2618436098098755, "sampling/sampling_logp_difference/mean": 0.020923238433897495, "step": 4540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 208.325, "completions/mean_terminated_length": 208.325, "completions/min_length": 109.6, "completions/min_terminated_length": 109.6, "entropy": 1.1066098541021347, "epoch": 0.4183138733106555, "frac_reward_zero_std": 0.3, "grad_norm": 5.796539306640625, "kl": 0.013413000758737326, "learning_rate": 4.891551867101151e-06, "loss": 0.0009, "num_tokens": 19780953.0, "reward": 0.05, "reward_std": 0.6589862048625946, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6589862406253815, "sampling/importance_sampling_ratio/max": 1.264914608001709, "sampling/importance_sampling_ratio/mean": 1.0002421140670776, "sampling/importance_sampling_ratio/min": 0.8037301421165466, "sampling/sampling_logp_difference/max": 0.24767435789108277, "sampling/sampling_logp_difference/mean": 0.020867038145661355, "step": 4550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 197.6625, "completions/mean_terminated_length": 197.6625, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 1.0919398814439774, "epoch": 0.4192332444607888, "frac_reward_zero_std": 0.7, "grad_norm": 0.13972947001457214, "kl": 0.015366761246696115, "learning_rate": 4.890812502776285e-06, "loss": 0.0009, "num_tokens": 19822518.0, "reward": -0.48125, "reward_std": 0.2878748536109924, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.2878748595714569, "sampling/importance_sampling_ratio/max": 1.269607388973236, "sampling/importance_sampling_ratio/mean": 1.0002316594123841, "sampling/importance_sampling_ratio/min": 0.7342676132917404, "sampling/sampling_logp_difference/max": 0.3371651113033295, "sampling/sampling_logp_difference/mean": 0.02149934694170952, "step": 4560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 342.2, "completions/max_terminated_length": 326.9, "completions/mean_length": 227.6, "completions/mean_terminated_length": 224.1482147216797, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 0.97963199429214, "epoch": 0.42015261561092215, "frac_reward_zero_std": 0.3, "grad_norm": 6.947952747344971, "kl": 0.014828102843603119, "learning_rate": 4.8900706828953335e-06, "loss": 0.001, "num_tokens": 19871614.0, "reward": -0.48125, "reward_std": 0.6363158702850342, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.6363159000873566, "sampling/importance_sampling_ratio/max": 1.2639047861099244, "sampling/importance_sampling_ratio/mean": 0.9997377753257751, "sampling/importance_sampling_ratio/min": 0.7579329788684845, "sampling/sampling_logp_difference/max": 0.2929945349693298, "sampling/sampling_logp_difference/mean": 0.019842839054763316, "step": 4570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 369.6, "completions/max_terminated_length": 349.6, "completions/mean_length": 212.575, "completions/mean_terminated_length": 205.22916717529296, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0896080985665322, "epoch": 0.42107198676105545, "frac_reward_zero_std": 0.7, "grad_norm": 0.11824537068605423, "kl": 0.016871709801489488, "learning_rate": 4.8893264082202015e-06, "loss": 0.0007, "num_tokens": 19915388.0, "reward": -0.5625, "reward_std": 0.24234963059425355, "rewards/reward_fn/mean": -0.5625, "rewards/reward_fn/std": 0.24234963059425355, "sampling/importance_sampling_ratio/max": 1.2721159219741822, "sampling/importance_sampling_ratio/mean": 0.9999995350837707, "sampling/importance_sampling_ratio/min": 0.7228146642446518, "sampling/sampling_logp_difference/max": 0.35654266476631163, "sampling/sampling_logp_difference/mean": 0.021434010192751884, "step": 4580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.1, "completions/max_terminated_length": 260.1, "completions/mean_length": 183.475, "completions/mean_terminated_length": 183.475, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.0653619751334191, "epoch": 0.42199135791118875, "frac_reward_zero_std": 0.4, "grad_norm": 7.110461711883545, "kl": 0.017837170459097253, "learning_rate": 4.888579679515321e-06, "loss": 0.0009, "num_tokens": 19957258.0, "reward": 0.125, "reward_std": 0.5950648546218872, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5950648665428162, "sampling/importance_sampling_ratio/max": 1.2726582169532776, "sampling/importance_sampling_ratio/mean": 1.0005559921264648, "sampling/importance_sampling_ratio/min": 0.7743561208248139, "sampling/sampling_logp_difference/max": 0.27725082635879517, "sampling/sampling_logp_difference/mean": 0.021580676548182964, "step": 4590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 194.0875, "completions/mean_terminated_length": 194.0875, "completions/min_length": 99.5, "completions/min_terminated_length": 99.5, "entropy": 1.0905418604612351, "epoch": 0.42291072906132204, "frac_reward_zero_std": 0.2, "grad_norm": 7.04024076461792, "kl": 0.014116451551672071, "learning_rate": 4.887830497547641e-06, "loss": 0.0007, "num_tokens": 20000777.0, "reward": -0.25, "reward_std": 0.6674932360649108, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6674932360649108, "sampling/importance_sampling_ratio/max": 1.2799574851989746, "sampling/importance_sampling_ratio/mean": 1.0004965603351592, "sampling/importance_sampling_ratio/min": 0.7736479461193084, "sampling/sampling_logp_difference/max": 0.2815155863761902, "sampling/sampling_logp_difference/mean": 0.021052085794508457, "step": 4600 }, { "epoch": 0.42291072906132204, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 294.5933682373473, "eval_completions/max_terminated_length": 291.5130890052356, "eval_completions/mean_length": 200.28970331588133, "eval_completions/mean_terminated_length": 199.50255572733454, "eval_completions/min_length": 122.1692844677138, "eval_completions/min_terminated_length": 122.1692844677138, "eval_entropy": 1.073634681615322, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.016119542439024487, "eval_loss": 0.0008329131524078548, "eval_num_tokens": 20000777.0, "eval_reward": -0.1087478184991274, "eval_reward_std": 0.4491245419463146, "eval_rewards/reward_fn/mean": -0.1087478184991274, "eval_rewards/reward_fn/std": 0.4491245561973379, "eval_runtime": 1716.9548, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2629049802117738, "eval_sampling/importance_sampling_ratio/mean": 0.9999536777250013, "eval_sampling/importance_sampling_ratio/min": 0.7728750485452266, "eval_sampling/sampling_logp_difference/max": 0.28925298687467194, "eval_sampling/sampling_logp_difference/mean": 0.020649282561605357, "eval_steps_per_second": 0.042, "step": 4600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.4, "completions/max_terminated_length": 279.4, "completions/mean_length": 191.2125, "completions/mean_terminated_length": 191.2125, "completions/min_length": 116.4, "completions/min_terminated_length": 116.4, "entropy": 1.058019071817398, "epoch": 0.42383010021145534, "frac_reward_zero_std": 0.5, "grad_norm": 0.13840192556381226, "kl": 0.013447888457449153, "learning_rate": 4.887078863086633e-06, "loss": 0.0001, "num_tokens": 20042018.0, "reward": -0.2375, "reward_std": 0.40024718046188357, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.40024718046188357, "sampling/importance_sampling_ratio/max": 1.2645349979400635, "sampling/importance_sampling_ratio/mean": 1.0004799664020538, "sampling/importance_sampling_ratio/min": 0.7441199511289597, "sampling/sampling_logp_difference/max": 0.3435834288597107, "sampling/sampling_logp_difference/mean": 0.02063285019248724, "step": 4610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.7, "completions/max_terminated_length": 306.7, "completions/mean_length": 209.05, "completions/mean_terminated_length": 209.05, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.0958145529031753, "epoch": 0.4247494713615887, "frac_reward_zero_std": 0.2, "grad_norm": 0.12313573062419891, "kl": 0.013900290458695962, "learning_rate": 4.886324776904284e-06, "loss": 0.0007, "num_tokens": 20086486.0, "reward": -0.325, "reward_std": 0.6531550586223602, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6531550824642182, "sampling/importance_sampling_ratio/max": 1.2916406750679017, "sampling/importance_sampling_ratio/mean": 1.000442111492157, "sampling/importance_sampling_ratio/min": 0.7678153693675995, "sampling/sampling_logp_difference/max": 0.28975670337677, "sampling/sampling_logp_difference/mean": 0.021878555603325366, "step": 4620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.8, "completions/max_terminated_length": 325.0, "completions/mean_length": 230.9875, "completions/mean_terminated_length": 227.97678833007814, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.0626677080988884, "epoch": 0.425668842511722, "frac_reward_zero_std": 0.6, "grad_norm": 0.1591137945652008, "kl": 0.014360571332508698, "learning_rate": 4.885568239775101e-06, "loss": 0.0008, "num_tokens": 20131301.0, "reward": -0.50625, "reward_std": 0.3383911535143852, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.3383911535143852, "sampling/importance_sampling_ratio/max": 1.305801558494568, "sampling/importance_sampling_ratio/mean": 1.00055330991745, "sampling/importance_sampling_ratio/min": 0.7459650337696075, "sampling/sampling_logp_difference/max": 0.3720982074737549, "sampling/sampling_logp_difference/mean": 0.020341503992676736, "step": 4630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 339.6, "completions/max_terminated_length": 328.7, "completions/mean_length": 222.325, "completions/mean_terminated_length": 218.5464294433594, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.1183530248701572, "epoch": 0.4265882136618553, "frac_reward_zero_std": 0.5, "grad_norm": 6.160386085510254, "kl": 0.012493924197042362, "learning_rate": 4.884809252476111e-06, "loss": 0.0007, "num_tokens": 20176359.0, "reward": -0.55625, "reward_std": 0.36611868590116503, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.3661187097430229, "sampling/importance_sampling_ratio/max": 1.3167296648025513, "sampling/importance_sampling_ratio/mean": 0.9998847723007203, "sampling/importance_sampling_ratio/min": 0.7657059848308563, "sampling/sampling_logp_difference/max": 0.3294498920440674, "sampling/sampling_logp_difference/mean": 0.02166303265839815, "step": 4640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.3, "completions/max_terminated_length": 322.3, "completions/mean_length": 225.9, "completions/mean_terminated_length": 225.9, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.1389670006930828, "epoch": 0.4275075848119886, "frac_reward_zero_std": 0.3, "grad_norm": 0.20558995008468628, "kl": 0.012585883995052427, "learning_rate": 4.884047815786852e-06, "loss": 0.0005, "num_tokens": 20224543.0, "reward": -0.2, "reward_std": 0.6295660197734833, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6295660555362701, "sampling/importance_sampling_ratio/max": 1.2524933338165283, "sampling/importance_sampling_ratio/mean": 1.0001934230327607, "sampling/importance_sampling_ratio/min": 0.8010773420333862, "sampling/sampling_logp_difference/max": 0.24442936182022096, "sampling/sampling_logp_difference/mean": 0.021541650779545306, "step": 4650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.9, "completions/max_terminated_length": 319.9, "completions/mean_length": 216.35, "completions/mean_terminated_length": 216.35, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.1525611989200115, "epoch": 0.42842695596212194, "frac_reward_zero_std": 0.3, "grad_norm": 5.454732418060303, "kl": 0.013552858115872368, "learning_rate": 4.883283930489381e-06, "loss": 0.0005, "num_tokens": 20269355.0, "reward": -0.4, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2638418912887572, "sampling/importance_sampling_ratio/mean": 0.9999869465827942, "sampling/importance_sampling_ratio/min": 0.7309197425842285, "sampling/sampling_logp_difference/max": 0.33148626089096067, "sampling/sampling_logp_difference/mean": 0.02147073596715927, "step": 4660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 198.4625, "completions/mean_terminated_length": 198.4625, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.044000618904829, "epoch": 0.42934632711225523, "frac_reward_zero_std": 0.6, "grad_norm": 6.170800685882568, "kl": 0.013513732864521445, "learning_rate": 4.88251759736827e-06, "loss": 0.0012, "num_tokens": 20313464.0, "reward": 0.075, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2862401723861694, "sampling/importance_sampling_ratio/mean": 1.0000767290592194, "sampling/importance_sampling_ratio/min": 0.7335525751113892, "sampling/sampling_logp_difference/max": 0.3565931499004364, "sampling/sampling_logp_difference/mean": 0.021290486305952074, "step": 4670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.0, "completions/max_terminated_length": 315.1, "completions/mean_length": 225.2625, "completions/mean_terminated_length": 222.07321472167968, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.0733529657125473, "epoch": 0.43026569826238853, "frac_reward_zero_std": 0.2, "grad_norm": 5.1028337478637695, "kl": 0.01281284416327253, "learning_rate": 4.881748817210604e-06, "loss": 0.0002, "num_tokens": 20359749.0, "reward": 0.11875, "reward_std": 0.7561971545219421, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.7561971724033356, "sampling/importance_sampling_ratio/max": 1.3137512922286987, "sampling/importance_sampling_ratio/mean": 0.9996890127658844, "sampling/importance_sampling_ratio/min": 0.7593605756759644, "sampling/sampling_logp_difference/max": 0.37316752076148985, "sampling/sampling_logp_difference/mean": 0.021093844063580038, "step": 4680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.8, "completions/max_terminated_length": 326.8, "completions/mean_length": 217.9875, "completions/mean_terminated_length": 217.9875, "completions/min_length": 141.7, "completions/min_terminated_length": 141.7, "entropy": 1.0378294579684735, "epoch": 0.4311850694125218, "frac_reward_zero_std": 0.7, "grad_norm": 0.6816912293434143, "kl": 0.011601912090554833, "learning_rate": 4.880977590805983e-06, "loss": 0.0009, "num_tokens": 20404052.0, "reward": 0.1, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.244780945777893, "sampling/importance_sampling_ratio/mean": 1.000049775838852, "sampling/importance_sampling_ratio/min": 0.7934948384761811, "sampling/sampling_logp_difference/max": 0.24497156143188475, "sampling/sampling_logp_difference/mean": 0.02016082089394331, "step": 4690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 316.4, "completions/max_terminated_length": 298.7, "completions/mean_length": 210.675, "completions/mean_terminated_length": 205.15833435058593, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0291843362152577, "epoch": 0.4321044405626551, "frac_reward_zero_std": 0.3, "grad_norm": 7.148164749145508, "kl": 0.013736267795320601, "learning_rate": 4.8802039189465164e-06, "loss": 0.0011, "num_tokens": 20447970.0, "reward": -0.2375, "reward_std": 0.5601452052593231, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.5601452171802521, "sampling/importance_sampling_ratio/max": 1.3124152421951294, "sampling/importance_sampling_ratio/mean": 1.0000545382499695, "sampling/importance_sampling_ratio/min": 0.7233867347240448, "sampling/sampling_logp_difference/max": 0.42136754989624026, "sampling/sampling_logp_difference/mean": 0.020755196176469325, "step": 4700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 174.675, "completions/mean_terminated_length": 174.675, "completions/min_length": 113.4, "completions/min_terminated_length": 113.4, "entropy": 1.0144341237843038, "epoch": 0.4330238117127885, "frac_reward_zero_std": 0.6, "grad_norm": 6.556826114654541, "kl": 0.013409603590844199, "learning_rate": 4.879427802426829e-06, "loss": 0.0006, "num_tokens": 20486456.0, "reward": 0.2, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2166276574134827, "sampling/importance_sampling_ratio/mean": 0.9996283113956451, "sampling/importance_sampling_ratio/min": 0.7340899288654328, "sampling/sampling_logp_difference/max": 0.3382696986198425, "sampling/sampling_logp_difference/mean": 0.021395320259034633, "step": 4710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 203.175, "completions/mean_terminated_length": 203.175, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.0600299961864947, "epoch": 0.4339431828629218, "frac_reward_zero_std": 0.5, "grad_norm": 7.896398544311523, "kl": 0.019391509180422872, "learning_rate": 4.878649242044055e-06, "loss": 0.001, "num_tokens": 20528126.0, "reward": 0.175, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.2403450846672057, "sampling/importance_sampling_ratio/mean": 1.0000899672508239, "sampling/importance_sampling_ratio/min": 0.75414337515831, "sampling/sampling_logp_difference/max": 0.2916059076786041, "sampling/sampling_logp_difference/mean": 0.021245722845196723, "step": 4720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.5, "completions/max_terminated_length": 292.5, "completions/mean_length": 179.8, "completions/mean_terminated_length": 179.8, "completions/min_length": 102.6, "completions/min_terminated_length": 102.6, "entropy": 0.9453475534915924, "epoch": 0.43486255401305507, "frac_reward_zero_std": 0.3, "grad_norm": 6.276083946228027, "kl": 0.01652462894562632, "learning_rate": 4.877868238597836e-06, "loss": 0.0005, "num_tokens": 20568710.0, "reward": 0.125, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.3042797803878785, "sampling/importance_sampling_ratio/mean": 1.0003132224082947, "sampling/importance_sampling_ratio/min": 0.7891328990459442, "sampling/sampling_logp_difference/max": 0.2934560298919678, "sampling/sampling_logp_difference/mean": 0.019461540784686805, "step": 4730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.9, "completions/max_terminated_length": 333.9, "completions/mean_length": 224.4125, "completions/mean_terminated_length": 224.4125, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.158565316349268, "epoch": 0.43578192516318837, "frac_reward_zero_std": 0.6, "grad_norm": 0.30162084102630615, "kl": 0.014682354108663276, "learning_rate": 4.877084792890327e-06, "loss": 0.0008, "num_tokens": 20618727.0, "reward": -0.55, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.247249472141266, "sampling/importance_sampling_ratio/mean": 0.9996816635131835, "sampling/importance_sampling_ratio/min": 0.7679404616355896, "sampling/sampling_logp_difference/max": 0.2703660011291504, "sampling/sampling_logp_difference/mean": 0.021610178984701633, "step": 4740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.0, "completions/max_terminated_length": 312.0, "completions/mean_length": 207.85, "completions/mean_terminated_length": 207.85, "completions/min_length": 113.2, "completions/min_terminated_length": 113.2, "entropy": 1.0875607680529356, "epoch": 0.43670129631332166, "frac_reward_zero_std": 0.6, "grad_norm": 5.107227802276611, "kl": 0.01393312358413823, "learning_rate": 4.876298905726189e-06, "loss": 0.0008, "num_tokens": 20662283.0, "reward": 0.05, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2714995265007019, "sampling/importance_sampling_ratio/mean": 0.9995714545249939, "sampling/importance_sampling_ratio/min": 0.7659084022045135, "sampling/sampling_logp_difference/max": 0.30353370904922483, "sampling/sampling_logp_difference/mean": 0.021760884299874307, "step": 4750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.8, "completions/max_terminated_length": 294.1, "completions/mean_length": 218.0625, "completions/mean_terminated_length": 215.5642883300781, "completions/min_length": 149.5, "completions/min_terminated_length": 149.5, "entropy": 1.1350074790418148, "epoch": 0.437620667463455, "frac_reward_zero_std": 0.6, "grad_norm": 0.12930865585803986, "kl": 0.012158301007002593, "learning_rate": 4.8755105779125915e-06, "loss": 0.0007, "num_tokens": 20707536.0, "reward": -0.45625, "reward_std": 0.3661186873912811, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.3661187171936035, "sampling/importance_sampling_ratio/max": 1.2699486851692199, "sampling/importance_sampling_ratio/mean": 0.9998308658599854, "sampling/importance_sampling_ratio/min": 0.8112403631210328, "sampling/sampling_logp_difference/max": 0.2468789577484131, "sampling/sampling_logp_difference/mean": 0.021919613517820836, "step": 4760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.4, "completions/max_terminated_length": 308.4, "completions/mean_length": 211.5625, "completions/mean_terminated_length": 211.5625, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0776331283152103, "epoch": 0.4385400386135883, "frac_reward_zero_std": 0.3, "grad_norm": 5.117267608642578, "kl": 0.011468908668030053, "learning_rate": 4.874719810259209e-06, "loss": 0.0004, "num_tokens": 20750837.0, "reward": 0.125, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.2525375604629516, "sampling/importance_sampling_ratio/mean": 0.9997254490852356, "sampling/importance_sampling_ratio/min": 0.7752107322216034, "sampling/sampling_logp_difference/max": 0.26935775876045226, "sampling/sampling_logp_difference/mean": 0.020601251162588596, "step": 4770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.9, "completions/max_terminated_length": 317.9, "completions/mean_length": 213.55, "completions/mean_terminated_length": 213.55, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.1095091745257377, "epoch": 0.4394594097637216, "frac_reward_zero_std": 0.3, "grad_norm": 5.852184295654297, "kl": 0.012273654201999306, "learning_rate": 4.873926603578225e-06, "loss": 0.0005, "num_tokens": 20795449.0, "reward": -0.425, "reward_std": 0.643904197216034, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.6439042091369629, "sampling/importance_sampling_ratio/max": 1.2325357675552369, "sampling/importance_sampling_ratio/mean": 0.9998161792755127, "sampling/importance_sampling_ratio/min": 0.7862249195575715, "sampling/sampling_logp_difference/max": 0.2582098484039307, "sampling/sampling_logp_difference/mean": 0.02143720369786024, "step": 4780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.1, "completions/max_terminated_length": 312.1, "completions/mean_length": 210.6, "completions/mean_terminated_length": 210.6, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0342254243791102, "epoch": 0.4403787809138549, "frac_reward_zero_std": 0.4, "grad_norm": 4.9674859046936035, "kl": 0.011637635406805203, "learning_rate": 4.873130958684325e-06, "loss": 0.0003, "num_tokens": 20835025.0, "reward": 0.1, "reward_std": 0.59920334815979, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5992033958435059, "sampling/importance_sampling_ratio/max": 1.238566279411316, "sampling/importance_sampling_ratio/mean": 0.9993701934814453, "sampling/importance_sampling_ratio/min": 0.7384608924388886, "sampling/sampling_logp_difference/max": 0.3222620725631714, "sampling/sampling_logp_difference/mean": 0.02032490149140358, "step": 4790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.8, "completions/max_terminated_length": 310.8, "completions/mean_length": 223.7, "completions/mean_terminated_length": 223.7, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.0530895218253136, "epoch": 0.44129815206398826, "frac_reward_zero_std": 0.5, "grad_norm": 0.1618771255016327, "kl": 0.01219914474640973, "learning_rate": 4.872332876394701e-06, "loss": 0.0003, "num_tokens": 20878529.0, "reward": -0.65, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2584924221038818, "sampling/importance_sampling_ratio/mean": 1.0000975251197814, "sampling/importance_sampling_ratio/min": 0.7757354617118836, "sampling/sampling_logp_difference/max": 0.25943453311920167, "sampling/sampling_logp_difference/mean": 0.02106231786310673, "step": 4800 }, { "epoch": 0.44129815206398826, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 308.03839441535774, "eval_completions/max_terminated_length": 305.7294938917976, "eval_completions/mean_length": 211.63939790575915, "eval_completions/mean_terminated_length": 210.99864974363103, "eval_completions/min_length": 132.56719022687608, "eval_completions/min_terminated_length": 132.56719022687608, "eval_entropy": 1.0893014871221147, "eval_frac_reward_zero_std": 0.44851657940663175, "eval_kl": 0.012215024210597156, "eval_loss": 0.0006033211247995496, "eval_num_tokens": 20878529.0, "eval_reward": -0.10700261780104713, "eval_reward_std": 0.4925643031405737, "eval_rewards/reward_fn/mean": -0.10700261780104713, "eval_rewards/reward_fn/std": 0.4925643192639942, "eval_runtime": 1785.7015, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.263032968963836, "eval_sampling/importance_sampling_ratio/mean": 0.999916067194148, "eval_sampling/importance_sampling_ratio/min": 0.7779097356825391, "eval_sampling/sampling_logp_difference/max": 0.2771551268367035, "eval_sampling/sampling_logp_difference/mean": 0.020644422019157317, "eval_steps_per_second": 0.04, "step": 4800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.1, "completions/max_terminated_length": 254.1, "completions/mean_length": 187.55, "completions/mean_terminated_length": 187.55, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0934534311294555, "epoch": 0.44221752321412156, "frac_reward_zero_std": 0.3, "grad_norm": 5.765825271606445, "kl": 0.0121953209862113, "learning_rate": 4.871532357529048e-06, "loss": 0.001, "num_tokens": 20919189.0, "reward": 0.44375, "reward_std": 0.6857351839542389, "rewards/reward_fn/mean": 0.44375, "rewards/reward_fn/std": 0.6857352197170258, "sampling/importance_sampling_ratio/max": 1.3648139595985413, "sampling/importance_sampling_ratio/mean": 0.9996397495269775, "sampling/importance_sampling_ratio/min": 0.7797386586666107, "sampling/sampling_logp_difference/max": 0.3292244911193848, "sampling/sampling_logp_difference/mean": 0.022401729598641396, "step": 4810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.2, "completions/max_terminated_length": 285.2, "completions/mean_length": 206.7875, "completions/mean_terminated_length": 206.7875, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0525523386895657, "epoch": 0.44313689436425485, "frac_reward_zero_std": 0.5, "grad_norm": 7.2952961921691895, "kl": 0.012152428802801296, "learning_rate": 4.870729402909561e-06, "loss": 0.0009, "num_tokens": 20962100.0, "reward": -0.425, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2741205096244812, "sampling/importance_sampling_ratio/mean": 1.0004018306732179, "sampling/importance_sampling_ratio/min": 0.7414525628089905, "sampling/sampling_logp_difference/max": 0.3257617950439453, "sampling/sampling_logp_difference/mean": 0.020136436074972154, "step": 4820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.9, "completions/max_terminated_length": 295.6, "completions/mean_length": 206.5375, "completions/mean_terminated_length": 202.67857208251954, "completions/min_length": 142.8, "completions/min_terminated_length": 142.8, "entropy": 1.0562504015862941, "epoch": 0.44405626551438815, "frac_reward_zero_std": 0.5, "grad_norm": 0.12112026661634445, "kl": 0.01279599685803987, "learning_rate": 4.869924013360941e-06, "loss": 0.0007, "num_tokens": 21005167.0, "reward": -0.35625, "reward_std": 0.4708753764629364, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.47087539434432985, "sampling/importance_sampling_ratio/max": 1.2224945545196533, "sampling/importance_sampling_ratio/mean": 0.99969442486763, "sampling/importance_sampling_ratio/min": 0.7142782390117646, "sampling/sampling_logp_difference/max": 0.42227036952972413, "sampling/sampling_logp_difference/mean": 0.020879371836781502, "step": 4830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.9, "completions/max_terminated_length": 257.9, "completions/mean_length": 195.125, "completions/mean_terminated_length": 195.125, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.0979996412992477, "epoch": 0.44497563666452145, "frac_reward_zero_std": 0.7, "grad_norm": 0.11827445030212402, "kl": 0.012160108017269522, "learning_rate": 4.869116189710386e-06, "loss": 0.0006, "num_tokens": 21051369.0, "reward": -0.125, "reward_std": 0.30299633741378784, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3029963493347168, "sampling/importance_sampling_ratio/max": 1.231055235862732, "sampling/importance_sampling_ratio/mean": 1.0000176429748535, "sampling/importance_sampling_ratio/min": 0.8015611112117768, "sampling/sampling_logp_difference/max": 0.23228275775909424, "sampling/sampling_logp_difference/mean": 0.02121412120759487, "step": 4840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.4, "completions/max_terminated_length": 291.4, "completions/mean_length": 206.8625, "completions/mean_terminated_length": 206.8625, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.067870057374239, "epoch": 0.4458950078146548, "frac_reward_zero_std": 0.4, "grad_norm": 0.15995070338249207, "kl": 0.010761679644929245, "learning_rate": 4.868305932787596e-06, "loss": 0.0008, "num_tokens": 21095726.0, "reward": -0.025, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2436874985694886, "sampling/importance_sampling_ratio/mean": 1.0001485109329225, "sampling/importance_sampling_ratio/min": 0.7686391711235047, "sampling/sampling_logp_difference/max": 0.28036280274391173, "sampling/sampling_logp_difference/mean": 0.020391837134957312, "step": 4850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.9, "completions/max_terminated_length": 312.9, "completions/mean_length": 227.3625, "completions/mean_terminated_length": 227.3625, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.1223405756056308, "epoch": 0.4468143789647881, "frac_reward_zero_std": 0.3, "grad_norm": 4.9405035972595215, "kl": 0.012966929574031383, "learning_rate": 4.8674932434247715e-06, "loss": 0.0006, "num_tokens": 21144523.0, "reward": -0.475, "reward_std": 0.6186381995677948, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.6186382234096527, "sampling/importance_sampling_ratio/max": 1.2528371214866638, "sampling/importance_sampling_ratio/mean": 0.999438351392746, "sampling/importance_sampling_ratio/min": 0.7988452196121216, "sampling/sampling_logp_difference/max": 0.23834426403045655, "sampling/sampling_logp_difference/mean": 0.021005039848387242, "step": 4860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.7, "completions/max_terminated_length": 286.7, "completions/mean_length": 195.275, "completions/mean_terminated_length": 195.275, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.0523985885083675, "epoch": 0.4477337501149214, "frac_reward_zero_std": 0.6, "grad_norm": 0.12126820534467697, "kl": 0.011263123745447957, "learning_rate": 4.866678122456607e-06, "loss": 0.0006, "num_tokens": 21183161.0, "reward": -0.075, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.257142961025238, "sampling/importance_sampling_ratio/mean": 0.9996438503265381, "sampling/importance_sampling_ratio/min": 0.7817487061023712, "sampling/sampling_logp_difference/max": 0.26296728253364565, "sampling/sampling_logp_difference/mean": 0.020846252515912055, "step": 4870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.5, "completions/max_terminated_length": 280.5, "completions/mean_length": 207.0125, "completions/mean_terminated_length": 207.0125, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.0922847151756288, "epoch": 0.4486531212650547, "frac_reward_zero_std": 0.5, "grad_norm": 6.466794013977051, "kl": 0.011672895989613607, "learning_rate": 4.865860570720298e-06, "loss": 0.0004, "num_tokens": 21226170.0, "reward": 0.11875, "reward_std": 0.4365772604942322, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.4365772604942322, "sampling/importance_sampling_ratio/max": 1.2662173748016357, "sampling/importance_sampling_ratio/mean": 1.0002239108085633, "sampling/importance_sampling_ratio/min": 0.7920225620269775, "sampling/sampling_logp_difference/max": 0.2658173680305481, "sampling/sampling_logp_difference/mean": 0.021007049642503262, "step": 4880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.6, "completions/max_terminated_length": 302.6, "completions/mean_length": 219.2875, "completions/mean_terminated_length": 219.2875, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.1248816974461078, "epoch": 0.449572492415188, "frac_reward_zero_std": 0.2, "grad_norm": 6.042624473571777, "kl": 0.012633965106215329, "learning_rate": 4.865040589055537e-06, "loss": 0.0007, "num_tokens": 21269065.0, "reward": -0.38125, "reward_std": 0.7460288584232331, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.7460288882255555, "sampling/importance_sampling_ratio/max": 1.260311245918274, "sampling/importance_sampling_ratio/mean": 0.9994315922260284, "sampling/importance_sampling_ratio/min": 0.7625337779521942, "sampling/sampling_logp_difference/max": 0.2927546501159668, "sampling/sampling_logp_difference/mean": 0.02176728714257479, "step": 4890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 208.9875, "completions/mean_terminated_length": 208.9875, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 0.9320349708199501, "epoch": 0.45049186356532134, "frac_reward_zero_std": 0.5, "grad_norm": 0.19582264125347137, "kl": 0.012148969603003934, "learning_rate": 4.86421817830451e-06, "loss": 0.0006, "num_tokens": 21310952.0, "reward": -0.075, "reward_std": 0.4738378763198853, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4738378882408142, "sampling/importance_sampling_ratio/max": 1.3493529200553893, "sampling/importance_sampling_ratio/mean": 1.0000371634960175, "sampling/importance_sampling_ratio/min": 0.6732578501105309, "sampling/sampling_logp_difference/max": 0.4563485085964203, "sampling/sampling_logp_difference/mean": 0.01964430697262287, "step": 4900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 199.575, "completions/mean_terminated_length": 199.575, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0543641433119775, "epoch": 0.45141123471545463, "frac_reward_zero_std": 0.2, "grad_norm": 5.966313362121582, "kl": 0.011490006413077935, "learning_rate": 4.863393339311897e-06, "loss": 0.0001, "num_tokens": 21352990.0, "reward": 0.2, "reward_std": 0.714614874124527, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.714614886045456, "sampling/importance_sampling_ratio/max": 1.2306870937347412, "sampling/importance_sampling_ratio/mean": 1.0002095341682433, "sampling/importance_sampling_ratio/min": 0.7812187850475312, "sampling/sampling_logp_difference/max": 0.25911293029785154, "sampling/sampling_logp_difference/mean": 0.020708657428622247, "step": 4910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.5, "completions/max_terminated_length": 317.5, "completions/mean_length": 222.6, "completions/mean_terminated_length": 222.6, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.0897082172334194, "epoch": 0.45233060586558793, "frac_reward_zero_std": 0.6, "grad_norm": 0.09881825000047684, "kl": 0.012726651708362624, "learning_rate": 4.8625660729248765e-06, "loss": 0.0008, "num_tokens": 21398310.0, "reward": -0.45, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2935795187950134, "sampling/importance_sampling_ratio/mean": 0.999754112958908, "sampling/importance_sampling_ratio/min": 0.7486090362071991, "sampling/sampling_logp_difference/max": 0.33290576934814453, "sampling/sampling_logp_difference/mean": 0.020490791648626328, "step": 4920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 346.4, "completions/max_terminated_length": 332.3, "completions/mean_length": 231.875, "completions/mean_terminated_length": 228.64642944335938, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.026636384986341, "epoch": 0.45324997701572123, "frac_reward_zero_std": 0.2, "grad_norm": 4.706917762756348, "kl": 0.012498455416061916, "learning_rate": 4.861736379993116e-06, "loss": 0.0002, "num_tokens": 21446476.0, "reward": -0.18125, "reward_std": 0.7789678514003754, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.7789678633213043, "sampling/importance_sampling_ratio/max": 1.355564033985138, "sampling/importance_sampling_ratio/mean": 1.0005542993545533, "sampling/importance_sampling_ratio/min": 0.7876303851604461, "sampling/sampling_logp_difference/max": 0.30710543394088746, "sampling/sampling_logp_difference/mean": 0.020168581698089837, "step": 4930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.9, "completions/max_terminated_length": 301.9, "completions/mean_length": 191.4125, "completions/mean_terminated_length": 191.4125, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0440019190311431, "epoch": 0.4541693481658546, "frac_reward_zero_std": 0.4, "grad_norm": 6.30952787399292, "kl": 0.01180479567265138, "learning_rate": 4.8609042613687766e-06, "loss": 0.0004, "num_tokens": 21487877.0, "reward": -0.125, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.3237103939056396, "sampling/importance_sampling_ratio/mean": 1.0006455659866333, "sampling/importance_sampling_ratio/min": 0.7975074112415313, "sampling/sampling_logp_difference/max": 0.29482307434082033, "sampling/sampling_logp_difference/mean": 0.020875980705022813, "step": 4940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 213.125, "completions/mean_terminated_length": 213.125, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 0.9899908728897572, "epoch": 0.4550887193159879, "frac_reward_zero_std": 0.9, "grad_norm": 0.16978298127651215, "kl": 0.012368996627628803, "learning_rate": 4.860069717906511e-06, "loss": 0.0008, "num_tokens": 21525975.0, "reward": -0.175, "reward_std": 0.07071067690849304, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.07071067690849304, "sampling/importance_sampling_ratio/max": 1.2810198307037353, "sampling/importance_sampling_ratio/mean": 1.0000928878784179, "sampling/importance_sampling_ratio/min": 0.7794631063938141, "sampling/sampling_logp_difference/max": 0.26877981424331665, "sampling/sampling_logp_difference/mean": 0.020637849159538747, "step": 4950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 208.0, "completions/mean_terminated_length": 208.0, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.1040277555584908, "epoch": 0.4560080904661212, "frac_reward_zero_std": 0.5, "grad_norm": 0.11605235189199448, "kl": 0.012773231737082825, "learning_rate": 4.859232750463463e-06, "loss": 0.0003, "num_tokens": 21567583.0, "reward": -0.53125, "reward_std": 0.43548367023468015, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.43548368811607363, "sampling/importance_sampling_ratio/max": 1.2831959843635559, "sampling/importance_sampling_ratio/mean": 0.9999805092811584, "sampling/importance_sampling_ratio/min": 0.774621206521988, "sampling/sampling_logp_difference/max": 0.291933012008667, "sampling/sampling_logp_difference/mean": 0.02218358814716339, "step": 4960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.4, "completions/max_terminated_length": 297.0, "completions/mean_length": 209.675, "completions/mean_terminated_length": 206.75535888671874, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.1103802293539047, "epoch": 0.45692746161625447, "frac_reward_zero_std": 0.1, "grad_norm": 6.231795310974121, "kl": 0.012155479658395052, "learning_rate": 4.858393359899265e-06, "loss": 0.0008, "num_tokens": 21610685.0, "reward": -0.38125, "reward_std": 0.7306155860424042, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.7306156039237977, "sampling/importance_sampling_ratio/max": 1.2585838198661805, "sampling/importance_sampling_ratio/mean": 0.99977006316185, "sampling/importance_sampling_ratio/min": 0.7899548828601837, "sampling/sampling_logp_difference/max": 0.26174386739730837, "sampling/sampling_logp_difference/mean": 0.021948704868555068, "step": 4970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 213.25, "completions/mean_terminated_length": 213.25, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0332473024725914, "epoch": 0.45784683276638777, "frac_reward_zero_std": 0.6, "grad_norm": 0.0974714607000351, "kl": 0.012056757276877762, "learning_rate": 4.857551547076037e-06, "loss": 0.0005, "num_tokens": 21653225.0, "reward": -0.15, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2655020475387573, "sampling/importance_sampling_ratio/mean": 0.999963229894638, "sampling/importance_sampling_ratio/min": 0.742551851272583, "sampling/sampling_logp_difference/max": 0.3178479075431824, "sampling/sampling_logp_difference/mean": 0.02047889158129692, "step": 4980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 356.5, "completions/max_terminated_length": 356.5, "completions/mean_length": 242.1, "completions/mean_terminated_length": 242.1, "completions/min_length": 157.2, "completions/min_terminated_length": 157.2, "entropy": 1.0672308139503002, "epoch": 0.4587662039165211, "frac_reward_zero_std": 0.4, "grad_norm": 5.450389862060547, "kl": 0.012334697460755707, "learning_rate": 4.85670731285839e-06, "loss": 0.0008, "num_tokens": 21700729.0, "reward": -0.3, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.31872056722641, "sampling/importance_sampling_ratio/mean": 1.000405740737915, "sampling/importance_sampling_ratio/min": 0.7880564153194427, "sampling/sampling_logp_difference/max": 0.2898052096366882, "sampling/sampling_logp_difference/mean": 0.020652634277939796, "step": 4990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.9, "completions/max_terminated_length": 274.9, "completions/mean_length": 210.8875, "completions/mean_terminated_length": 210.8875, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.0517433062195778, "epoch": 0.4596855750666544, "frac_reward_zero_std": 0.4, "grad_norm": 0.1399354189634323, "kl": 0.013411807076772675, "learning_rate": 4.855860658113419e-06, "loss": 0.0005, "num_tokens": 21743104.0, "reward": -0.4, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.2755627036094666, "sampling/importance_sampling_ratio/mean": 1.0005995273590087, "sampling/importance_sampling_ratio/min": 0.8038640320301056, "sampling/sampling_logp_difference/max": 0.2522912502288818, "sampling/sampling_logp_difference/mean": 0.02094597201794386, "step": 5000 }, { "epoch": 0.4596855750666544, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 293.5828970331588, "eval_completions/max_terminated_length": 292.17102966841185, "eval_completions/mean_length": 206.13219895287958, "eval_completions/mean_terminated_length": 205.87621545708407, "eval_completions/min_length": 133.1588132635253, "eval_completions/min_terminated_length": 133.1588132635253, "eval_entropy": 1.0466184384639767, "eval_frac_reward_zero_std": 0.462478184991274, "eval_kl": 0.013056977617881061, "eval_loss": 0.0006702912505716085, "eval_num_tokens": 21743104.0, "eval_reward": -0.11474694589877836, "eval_reward_std": 0.47708693110297906, "eval_rewards/reward_fn/mean": -0.11474694589877836, "eval_rewards/reward_fn/std": 0.47708694618617886, "eval_runtime": 1712.969, "eval_samples_per_second": 0.335, "eval_sampling/importance_sampling_ratio/max": 1.2635597955910116, "eval_sampling/importance_sampling_ratio/mean": 1.0000054195497257, "eval_sampling/importance_sampling_ratio/min": 0.7718561526264404, "eval_sampling/sampling_logp_difference/max": 0.28848949267186, "eval_sampling/sampling_logp_difference/mean": 0.020932348675856832, "eval_steps_per_second": 0.042, "step": 5000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.6, "completions/max_terminated_length": 303.6, "completions/mean_length": 211.75, "completions/mean_terminated_length": 211.75, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.004664459079504, "epoch": 0.4606049462167877, "frac_reward_zero_std": 0.4, "grad_norm": 0.12875671684741974, "kl": 0.012643802084494383, "learning_rate": 4.855011583710707e-06, "loss": 0.0003, "num_tokens": 21786036.0, "reward": -0.225, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.325749695301056, "sampling/importance_sampling_ratio/mean": 1.0002267777919769, "sampling/importance_sampling_ratio/min": 0.7620134770870208, "sampling/sampling_logp_difference/max": 0.3156653165817261, "sampling/sampling_logp_difference/mean": 0.02011100947856903, "step": 5010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.0, "completions/max_terminated_length": 301.0, "completions/mean_length": 204.425, "completions/mean_terminated_length": 204.425, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.02359873726964, "epoch": 0.461524317366921, "frac_reward_zero_std": 0.8, "grad_norm": 0.10781216621398926, "kl": 0.01259893910610117, "learning_rate": 4.854160090522321e-06, "loss": 0.0008, "num_tokens": 21826654.0, "reward": 0.0, "reward_std": 0.1414213538169861, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.1414213538169861, "sampling/importance_sampling_ratio/max": 1.2725447416305542, "sampling/importance_sampling_ratio/mean": 1.0005612015724181, "sampling/importance_sampling_ratio/min": 0.7444216728210449, "sampling/sampling_logp_difference/max": 0.3248041868209839, "sampling/sampling_logp_difference/mean": 0.021000218950212, "step": 5020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.4, "completions/max_terminated_length": 274.4, "completions/mean_length": 189.9625, "completions/mean_terminated_length": 189.9625, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.1054276548326016, "epoch": 0.4624436885170543, "frac_reward_zero_std": 0.4, "grad_norm": 7.347875118255615, "kl": 0.012611873995047063, "learning_rate": 4.853306179422814e-06, "loss": 0.0008, "num_tokens": 21868907.0, "reward": -0.2, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.32891126871109, "sampling/importance_sampling_ratio/mean": 0.9996663928031921, "sampling/importance_sampling_ratio/min": 0.7947661936283111, "sampling/sampling_logp_difference/max": 0.31573607921600344, "sampling/sampling_logp_difference/mean": 0.02219123635441065, "step": 5030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 329.9, "completions/max_terminated_length": 313.3, "completions/mean_length": 220.275, "completions/mean_terminated_length": 213.19583435058593, "completions/min_length": 111.3, "completions/min_terminated_length": 111.3, "entropy": 0.9670088920742274, "epoch": 0.46336305966718766, "frac_reward_zero_std": 0.5, "grad_norm": 0.12278541922569275, "kl": 0.011872620135545731, "learning_rate": 4.852449851289219e-06, "loss": 0.0005, "num_tokens": 21914529.0, "reward": -0.2625, "reward_std": 0.4165701389312744, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.4165701508522034, "sampling/importance_sampling_ratio/max": 1.2344521760940552, "sampling/importance_sampling_ratio/mean": 0.999990725517273, "sampling/importance_sampling_ratio/min": 0.8007081985473633, "sampling/sampling_logp_difference/max": 0.23186758756637574, "sampling/sampling_logp_difference/mean": 0.01942192791029811, "step": 5040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.5, "completions/max_terminated_length": 316.5, "completions/mean_length": 210.8625, "completions/mean_terminated_length": 210.8625, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0730721548199653, "epoch": 0.46428243081732096, "frac_reward_zero_std": 0.5, "grad_norm": 0.08940266817808151, "kl": 0.013814101717434824, "learning_rate": 4.851591107001055e-06, "loss": 0.0009, "num_tokens": 21957926.0, "reward": -0.20625, "reward_std": 0.5069427490234375, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.5069427728652954, "sampling/importance_sampling_ratio/max": 1.2931722402572632, "sampling/importance_sampling_ratio/mean": 1.0002142012119293, "sampling/importance_sampling_ratio/min": 0.7605769127607346, "sampling/sampling_logp_difference/max": 0.30998225808143615, "sampling/sampling_logp_difference/mean": 0.021756075508892535, "step": 5050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 208.9375, "completions/mean_terminated_length": 208.9375, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 0.9753674462437629, "epoch": 0.46520180196745425, "frac_reward_zero_std": 0.4, "grad_norm": 0.13599872589111328, "kl": 0.013190596457570791, "learning_rate": 4.850729947440321e-06, "loss": 0.0007, "num_tokens": 22000113.0, "reward": 0.175, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2407723903656005, "sampling/importance_sampling_ratio/mean": 1.0002837121486663, "sampling/importance_sampling_ratio/min": 0.7755236923694611, "sampling/sampling_logp_difference/max": 0.2569628179073334, "sampling/sampling_logp_difference/mean": 0.02001712303608656, "step": 5060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 357.1, "completions/max_terminated_length": 334.1, "completions/mean_length": 242.0125, "completions/mean_terminated_length": 234.94464416503905, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.0204179756343366, "epoch": 0.46612117311758755, "frac_reward_zero_std": 0.2, "grad_norm": 3.7121593952178955, "kl": 0.014476958243176342, "learning_rate": 4.8498663734914975e-06, "loss": 0.0004, "num_tokens": 22044858.0, "reward": -0.11875, "reward_std": 0.621119749546051, "rewards/reward_fn/mean": -0.11875, "rewards/reward_fn/std": 0.6211197674274445, "sampling/importance_sampling_ratio/max": 1.2354776024818421, "sampling/importance_sampling_ratio/mean": 1.0004219233989715, "sampling/importance_sampling_ratio/min": 0.738431715965271, "sampling/sampling_logp_difference/max": 0.3177373051643372, "sampling/sampling_logp_difference/mean": 0.020383267477154732, "step": 5070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 194.7375, "completions/mean_terminated_length": 194.7375, "completions/min_length": 109.4, "completions/min_terminated_length": 109.4, "entropy": 1.0184748739004135, "epoch": 0.4670405442677209, "frac_reward_zero_std": 0.4, "grad_norm": 6.030892372131348, "kl": 0.013309187081176788, "learning_rate": 4.8490003860415436e-06, "loss": 0.0009, "num_tokens": 22085045.0, "reward": -0.55, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2869211554527282, "sampling/importance_sampling_ratio/mean": 0.999978768825531, "sampling/importance_sampling_ratio/min": 0.7760288417339325, "sampling/sampling_logp_difference/max": 0.299783992767334, "sampling/sampling_logp_difference/mean": 0.020753191970288753, "step": 5080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 205.25, "completions/mean_terminated_length": 205.25, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0138204660266639, "epoch": 0.4679599154178542, "frac_reward_zero_std": 0.2, "grad_norm": 8.643546104431152, "kl": 0.013913884636713192, "learning_rate": 4.848131985979897e-06, "loss": 0.0005, "num_tokens": 22129969.0, "reward": -0.025, "reward_std": 0.685954213142395, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6859542489051819, "sampling/importance_sampling_ratio/max": 1.2861674189567567, "sampling/importance_sampling_ratio/mean": 1.0004273116588593, "sampling/importance_sampling_ratio/min": 0.7895570755004883, "sampling/sampling_logp_difference/max": 0.27133007645606994, "sampling/sampling_logp_difference/mean": 0.020665212906897067, "step": 5090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.5, "completions/max_terminated_length": 274.5, "completions/mean_length": 197.6, "completions/mean_terminated_length": 197.6, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 0.984489668160677, "epoch": 0.4688792865679875, "frac_reward_zero_std": 0.6, "grad_norm": 5.831820011138916, "kl": 0.011460717511363327, "learning_rate": 4.847261174198475e-06, "loss": 0.0008, "num_tokens": 22176345.0, "reward": 0.2, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.277844989299774, "sampling/importance_sampling_ratio/mean": 1.0000254809856415, "sampling/importance_sampling_ratio/min": 0.7839222311973572, "sampling/sampling_logp_difference/max": 0.2715777516365051, "sampling/sampling_logp_difference/mean": 0.02057779226452112, "step": 5100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.3, "completions/max_terminated_length": 308.3, "completions/mean_length": 218.7, "completions/mean_terminated_length": 218.7, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0339699640870095, "epoch": 0.4697986577181208, "frac_reward_zero_std": 0.2, "grad_norm": 5.567153453826904, "kl": 0.01321974562597461, "learning_rate": 4.846387951591672e-06, "loss": 0.0012, "num_tokens": 22220977.0, "reward": -0.3, "reward_std": 0.7002923905849456, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.7002924025058747, "sampling/importance_sampling_ratio/max": 1.2923652410507203, "sampling/importance_sampling_ratio/mean": 1.000276553630829, "sampling/importance_sampling_ratio/min": 0.7423801749944687, "sampling/sampling_logp_difference/max": 0.3424058198928833, "sampling/sampling_logp_difference/mean": 0.020734795741736887, "step": 5110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.0, "completions/max_terminated_length": 321.0, "completions/mean_length": 219.8875, "completions/mean_terminated_length": 219.8875, "completions/min_length": 154.7, "completions/min_terminated_length": 154.7, "entropy": 1.0280052192509175, "epoch": 0.4707180288682541, "frac_reward_zero_std": 0.5, "grad_norm": 0.16461864113807678, "kl": 0.012833667959785088, "learning_rate": 4.8455123190563556e-06, "loss": 0.0009, "num_tokens": 22263672.0, "reward": -0.075, "reward_std": 0.5100160002708435, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5100160360336303, "sampling/importance_sampling_ratio/max": 1.2459577560424804, "sampling/importance_sampling_ratio/mean": 1.0003187775611877, "sampling/importance_sampling_ratio/min": 0.7826995074748992, "sampling/sampling_logp_difference/max": 0.25807485580444334, "sampling/sampling_logp_difference/mean": 0.019612261466681956, "step": 5120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.4, "completions/max_terminated_length": 301.6, "completions/mean_length": 226.2625, "completions/mean_terminated_length": 222.97142944335937, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.1289741687476635, "epoch": 0.47163740001838744, "frac_reward_zero_std": 0.5, "grad_norm": 0.13918569684028625, "kl": 0.017835539893712848, "learning_rate": 4.844634277491873e-06, "loss": 0.0004, "num_tokens": 22308925.0, "reward": -0.33125, "reward_std": 0.35519086569547653, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.3551908776164055, "sampling/importance_sampling_ratio/max": 1.2611980319023133, "sampling/importance_sampling_ratio/mean": 0.999942809343338, "sampling/importance_sampling_ratio/min": 0.7617108404636384, "sampling/sampling_logp_difference/max": 0.2905771017074585, "sampling/sampling_logp_difference/mean": 0.021452205441892146, "step": 5130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 291.1, "completions/max_terminated_length": 270.6, "completions/mean_length": 202.45, "completions/mean_terminated_length": 199.16785888671876, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.0771476402878761, "epoch": 0.47255677116852074, "frac_reward_zero_std": 0.5, "grad_norm": 0.16941936314105988, "kl": 0.012774487939896061, "learning_rate": 4.843753827800041e-06, "loss": 0.0009, "num_tokens": 22352001.0, "reward": 0.09375, "reward_std": 0.4741435945034027, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.4741436064243317, "sampling/importance_sampling_ratio/max": 1.276336407661438, "sampling/importance_sampling_ratio/mean": 1.0002790808677673, "sampling/importance_sampling_ratio/min": 0.7801722824573517, "sampling/sampling_logp_difference/max": 0.2577356457710266, "sampling/sampling_logp_difference/mean": 0.020846420712769032, "step": 5140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.9, "completions/max_terminated_length": 281.9, "completions/mean_length": 211.2125, "completions/mean_terminated_length": 211.2125, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0596356257796287, "epoch": 0.47347614231865404, "frac_reward_zero_std": 0.4, "grad_norm": 0.0991073027253151, "kl": 0.013162382337031885, "learning_rate": 4.842870970885153e-06, "loss": 0.0009, "num_tokens": 22396730.0, "reward": -0.425, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.239285910129547, "sampling/importance_sampling_ratio/mean": 1.0001122295856475, "sampling/importance_sampling_ratio/min": 0.7840543866157532, "sampling/sampling_logp_difference/max": 0.2531113624572754, "sampling/sampling_logp_difference/mean": 0.020239243656396864, "step": 5150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 194.4625, "completions/mean_terminated_length": 194.4625, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 1.0481193415820598, "epoch": 0.47439551346878733, "frac_reward_zero_std": 0.4, "grad_norm": 0.24073366820812225, "kl": 0.013721312425332144, "learning_rate": 4.841985707653974e-06, "loss": 0.0005, "num_tokens": 22437151.0, "reward": -0.1, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.2427141547203064, "sampling/importance_sampling_ratio/mean": 0.9997084021568299, "sampling/importance_sampling_ratio/min": 0.7442298412322998, "sampling/sampling_logp_difference/max": 0.3952692985534668, "sampling/sampling_logp_difference/mean": 0.02077340092509985, "step": 5160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 201.5625, "completions/mean_terminated_length": 201.5625, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1000211976468564, "epoch": 0.4753148846189207, "frac_reward_zero_std": 0.6, "grad_norm": 0.12990355491638184, "kl": 0.013226689549628644, "learning_rate": 4.84109803901574e-06, "loss": 0.0008, "num_tokens": 22480204.0, "reward": -0.05, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.235254442691803, "sampling/importance_sampling_ratio/mean": 1.0000951766967774, "sampling/importance_sampling_ratio/min": 0.769539886713028, "sampling/sampling_logp_difference/max": 0.27873663902282714, "sampling/sampling_logp_difference/mean": 0.021351318433880807, "step": 5170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.3, "completions/max_terminated_length": 279.3, "completions/mean_length": 197.425, "completions/mean_terminated_length": 197.425, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.103804711252451, "epoch": 0.476234255769054, "frac_reward_zero_std": 0.5, "grad_norm": 0.21689869463443756, "kl": 0.012398879486136138, "learning_rate": 4.840207965882158e-06, "loss": 0.0008, "num_tokens": 22521198.0, "reward": -0.4, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3165428042411804, "sampling/importance_sampling_ratio/mean": 0.999915075302124, "sampling/importance_sampling_ratio/min": 0.7427598148584366, "sampling/sampling_logp_difference/max": 0.3726984322071075, "sampling/sampling_logp_difference/mean": 0.021696367114782334, "step": 5180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.3, "completions/max_terminated_length": 307.3, "completions/mean_length": 217.0, "completions/mean_terminated_length": 217.0, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.0621801525354386, "epoch": 0.4771536269191873, "frac_reward_zero_std": 0.4, "grad_norm": 5.629326343536377, "kl": 0.012731557642109693, "learning_rate": 4.839315489167405e-06, "loss": 0.0006, "num_tokens": 22563302.0, "reward": -0.175, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.2860819101333618, "sampling/importance_sampling_ratio/mean": 1.0001328468322754, "sampling/importance_sampling_ratio/min": 0.7698484539985657, "sampling/sampling_logp_difference/max": 0.3105497121810913, "sampling/sampling_logp_difference/mean": 0.02129819057881832, "step": 5190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.9, "completions/max_terminated_length": 276.9, "completions/mean_length": 208.4375, "completions/mean_terminated_length": 208.4375, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.0589510843157768, "epoch": 0.4780729980693206, "frac_reward_zero_std": 0.3, "grad_norm": 5.961905002593994, "kl": 0.012688736827112735, "learning_rate": 4.838420609788125e-06, "loss": 0.0004, "num_tokens": 22607649.0, "reward": -0.075, "reward_std": 0.6480583846569061, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6480584084987641, "sampling/importance_sampling_ratio/max": 1.2556210517883302, "sampling/importance_sampling_ratio/mean": 0.9999720692634583, "sampling/importance_sampling_ratio/min": 0.7877919733524322, "sampling/sampling_logp_difference/max": 0.2512882471084595, "sampling/sampling_logp_difference/mean": 0.020585130248218775, "step": 5200 }, { "epoch": 0.4780729980693206, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 295.5567190226876, "eval_completions/max_terminated_length": 293.3333333333333, "eval_completions/mean_length": 209.9410994764398, "eval_completions/mean_terminated_length": 209.45178899049344, "eval_completions/min_length": 140.14659685863873, "eval_completions/min_terminated_length": 140.14659685863873, "eval_entropy": 1.0875552312241799, "eval_frac_reward_zero_std": 0.450261780104712, "eval_kl": 0.012317749543235772, "eval_loss": 0.0005853655748069286, "eval_num_tokens": 22607649.0, "eval_reward": -0.1143106457242583, "eval_reward_std": 0.4807692399251732, "eval_rewards/reward_fn/mean": -0.1143106457242583, "eval_rewards/reward_fn/std": 0.48076925480032884, "eval_runtime": 1721.8915, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.275219314385459, "eval_sampling/importance_sampling_ratio/mean": 0.9999404185521457, "eval_sampling/importance_sampling_ratio/min": 0.783109163582637, "eval_sampling/sampling_logp_difference/max": 0.27737603590126436, "eval_sampling/sampling_logp_difference/mean": 0.020780360784518156, "eval_steps_per_second": 0.042, "step": 5200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 203.825, "completions/mean_terminated_length": 203.825, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.093987337499857, "epoch": 0.4789923692194539, "frac_reward_zero_std": 0.2, "grad_norm": 5.2634429931640625, "kl": 0.0120485079678474, "learning_rate": 4.8375233286634315e-06, "loss": 0.0007, "num_tokens": 22650459.0, "reward": -0.15625, "reward_std": 0.7210969269275666, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.7210969269275666, "sampling/importance_sampling_ratio/max": 1.2296167373657227, "sampling/importance_sampling_ratio/mean": 0.9994602620601654, "sampling/importance_sampling_ratio/min": 0.7535184800624848, "sampling/sampling_logp_difference/max": 0.29173082709312437, "sampling/sampling_logp_difference/mean": 0.02120869429782033, "step": 5210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.0, "completions/max_terminated_length": 315.0, "completions/mean_length": 214.5875, "completions/mean_terminated_length": 214.5875, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.1204146943986415, "epoch": 0.4799117403695872, "frac_reward_zero_std": 0.5, "grad_norm": 0.13234396278858185, "kl": 0.013961851893691346, "learning_rate": 4.836623646714904e-06, "loss": 0.001, "num_tokens": 22694634.0, "reward": -0.5, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2545308828353883, "sampling/importance_sampling_ratio/mean": 0.9997150659561157, "sampling/importance_sampling_ratio/min": 0.7964275062084198, "sampling/sampling_logp_difference/max": 0.2499106764793396, "sampling/sampling_logp_difference/mean": 0.0210031870752573, "step": 5220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.7, "completions/max_terminated_length": 307.7, "completions/mean_length": 218.875, "completions/mean_terminated_length": 218.875, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.108737948536873, "epoch": 0.4808311115197205, "frac_reward_zero_std": 0.6, "grad_norm": 5.66285514831543, "kl": 0.01192550907144323, "learning_rate": 4.8357215648665876e-06, "loss": 0.0006, "num_tokens": 22739504.0, "reward": 0.025, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.299254822731018, "sampling/importance_sampling_ratio/mean": 0.9999650478363037, "sampling/importance_sampling_ratio/min": 0.7595689117908477, "sampling/sampling_logp_difference/max": 0.3158516943454742, "sampling/sampling_logp_difference/mean": 0.020709323324263097, "step": 5230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.0, "completions/max_terminated_length": 334.0, "completions/mean_length": 222.3375, "completions/mean_terminated_length": 222.3375, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.0698212169110775, "epoch": 0.4817504826698538, "frac_reward_zero_std": 0.4, "grad_norm": 4.216022968292236, "kl": 0.013177189300768078, "learning_rate": 4.834817084044994e-06, "loss": 0.0008, "num_tokens": 22781843.0, "reward": -0.175, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2821684002876281, "sampling/importance_sampling_ratio/mean": 0.9999168455600739, "sampling/importance_sampling_ratio/min": 0.7492824882268906, "sampling/sampling_logp_difference/max": 0.3382062792778015, "sampling/sampling_logp_difference/mean": 0.02103486992418766, "step": 5240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.5, "completions/max_terminated_length": 271.5, "completions/mean_length": 196.05, "completions/mean_terminated_length": 196.05, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.0608417205512524, "epoch": 0.4826698538199871, "frac_reward_zero_std": 0.2, "grad_norm": 0.16319362819194794, "kl": 0.017803014413220807, "learning_rate": 4.833910205179096e-06, "loss": 0.0008, "num_tokens": 22823007.0, "reward": -0.175, "reward_std": 0.685954213142395, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6859542489051819, "sampling/importance_sampling_ratio/max": 1.2465123414993287, "sampling/importance_sampling_ratio/mean": 1.0000401854515075, "sampling/importance_sampling_ratio/min": 0.700940090417862, "sampling/sampling_logp_difference/max": 0.41120549440383913, "sampling/sampling_logp_difference/mean": 0.020922703854739665, "step": 5250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 211.675, "completions/mean_terminated_length": 211.675, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.04888329282403, "epoch": 0.4835892249701204, "frac_reward_zero_std": 0.5, "grad_norm": 4.985872745513916, "kl": 0.012868320860434324, "learning_rate": 4.833000929200331e-06, "loss": 0.0004, "num_tokens": 22864629.0, "reward": 0.15, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.3576210379600524, "sampling/importance_sampling_ratio/mean": 1.0000470399856567, "sampling/importance_sampling_ratio/min": 0.7867831528186798, "sampling/sampling_logp_difference/max": 0.4357964277267456, "sampling/sampling_logp_difference/mean": 0.0211361488327384, "step": 5260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 325.4, "completions/max_terminated_length": 318.2, "completions/mean_length": 226.9875, "completions/mean_terminated_length": 224.0125, "completions/min_length": 151.2, "completions/min_terminated_length": 151.2, "entropy": 1.0101955156773328, "epoch": 0.48450859612025376, "frac_reward_zero_std": 0.4, "grad_norm": 6.560452461242676, "kl": 0.012427160501829349, "learning_rate": 4.8320892570426e-06, "loss": 0.0008, "num_tokens": 22908812.0, "reward": -0.45625, "reward_std": 0.5432875275611877, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.5432875394821167, "sampling/importance_sampling_ratio/max": 1.2859188199043274, "sampling/importance_sampling_ratio/mean": 1.0004424512386323, "sampling/importance_sampling_ratio/min": 0.7952850818634033, "sampling/sampling_logp_difference/max": 0.25561401844024656, "sampling/sampling_logp_difference/mean": 0.020336034707725048, "step": 5270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 182.5125, "completions/mean_terminated_length": 182.5125, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 0.9894833117723465, "epoch": 0.48542796727038706, "frac_reward_zero_std": 0.8, "grad_norm": 0.21025653183460236, "kl": 0.018722178181633353, "learning_rate": 4.831175189642261e-06, "loss": 0.0013, "num_tokens": 22946261.0, "reward": -0.225, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.3089984059333801, "sampling/importance_sampling_ratio/mean": 1.0001187384128571, "sampling/importance_sampling_ratio/min": 0.7574678122997284, "sampling/sampling_logp_difference/max": 0.324986058473587, "sampling/sampling_logp_difference/mean": 0.02072713617235422, "step": 5280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 194.0, "completions/mean_terminated_length": 194.0, "completions/min_length": 114.2, "completions/min_terminated_length": 114.2, "entropy": 1.0401216186583042, "epoch": 0.48634733842052036, "frac_reward_zero_std": 0.3, "grad_norm": 6.7388811111450195, "kl": 0.012403536378405989, "learning_rate": 4.8302587279381365e-06, "loss": 0.0007, "num_tokens": 22985285.0, "reward": -0.025, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.237746274471283, "sampling/importance_sampling_ratio/mean": 0.9997860252857208, "sampling/importance_sampling_ratio/min": 0.7513849020004273, "sampling/sampling_logp_difference/max": 0.29494929909706114, "sampling/sampling_logp_difference/mean": 0.02115449998527765, "step": 5290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 200.5125, "completions/mean_terminated_length": 200.5125, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 0.9838359773159027, "epoch": 0.48726670957065366, "frac_reward_zero_std": 0.6, "grad_norm": 0.13338373601436615, "kl": 0.01173378505045548, "learning_rate": 4.829339872871505e-06, "loss": 0.0006, "num_tokens": 23026806.0, "reward": -0.025, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2565070271492005, "sampling/importance_sampling_ratio/mean": 1.0003445267677307, "sampling/importance_sampling_ratio/min": 0.7860248208045959, "sampling/sampling_logp_difference/max": 0.25869364738464357, "sampling/sampling_logp_difference/mean": 0.020369666256010534, "step": 5300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.0, "completions/max_terminated_length": 329.0, "completions/mean_length": 214.8125, "completions/mean_terminated_length": 214.8125, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.0209974773228168, "epoch": 0.488186080720787, "frac_reward_zero_std": 0.3, "grad_norm": 6.444166660308838, "kl": 0.01167583295609802, "learning_rate": 4.828418625386105e-06, "loss": 0.0009, "num_tokens": 23068879.0, "reward": -0.20625, "reward_std": 0.6386135458946228, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.6386135697364808, "sampling/importance_sampling_ratio/max": 1.2453895926475524, "sampling/importance_sampling_ratio/mean": 1.0000482261180879, "sampling/importance_sampling_ratio/min": 0.7932053208351135, "sampling/sampling_logp_difference/max": 0.24476518630981445, "sampling/sampling_logp_difference/mean": 0.019888460636138916, "step": 5310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.9, "completions/max_terminated_length": 267.9, "completions/mean_length": 189.95, "completions/mean_terminated_length": 189.95, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.026459702849388, "epoch": 0.4891054518709203, "frac_reward_zero_std": 0.6, "grad_norm": 0.1305590122938156, "kl": 0.012846912146778777, "learning_rate": 4.8274949864281304e-06, "loss": 0.0004, "num_tokens": 23109467.0, "reward": -0.03125, "reward_std": 0.3583177328109741, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.3583177328109741, "sampling/importance_sampling_ratio/max": 1.258798098564148, "sampling/importance_sampling_ratio/mean": 1.0007733166217805, "sampling/importance_sampling_ratio/min": 0.7556799590587616, "sampling/sampling_logp_difference/max": 0.3109339475631714, "sampling/sampling_logp_difference/mean": 0.0204321151599288, "step": 5320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 210.0375, "completions/mean_terminated_length": 210.0375, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0879841901361942, "epoch": 0.4900248230210536, "frac_reward_zero_std": 0.3, "grad_norm": 7.157545566558838, "kl": 0.013833353016525506, "learning_rate": 4.826568956946233e-06, "loss": 0.001, "num_tokens": 23153070.0, "reward": -0.25, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2474465489387512, "sampling/importance_sampling_ratio/mean": 0.9998497128486633, "sampling/importance_sampling_ratio/min": 0.787189781665802, "sampling/sampling_logp_difference/max": 0.25491111278533934, "sampling/sampling_logp_difference/mean": 0.021296424418687822, "step": 5330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.1, "completions/max_terminated_length": 295.1, "completions/mean_length": 206.5625, "completions/mean_terminated_length": 206.5625, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0538564622402191, "epoch": 0.4909441941711869, "frac_reward_zero_std": 0.5, "grad_norm": 6.509616851806641, "kl": 0.012304365122690796, "learning_rate": 4.825640537891521e-06, "loss": 0.0009, "num_tokens": 23196083.0, "reward": -0.075, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2588162541389465, "sampling/importance_sampling_ratio/mean": 0.9994840145111084, "sampling/importance_sampling_ratio/min": 0.7935552477836609, "sampling/sampling_logp_difference/max": 0.25456039905548095, "sampling/sampling_logp_difference/mean": 0.020683585666120052, "step": 5340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.8, "completions/max_terminated_length": 267.8, "completions/mean_length": 187.975, "completions/mean_terminated_length": 187.975, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.076381579786539, "epoch": 0.4918635653213202, "frac_reward_zero_std": 0.6, "grad_norm": 5.538012981414795, "kl": 0.014075051323743537, "learning_rate": 4.824709730217553e-06, "loss": 0.0007, "num_tokens": 23235857.0, "reward": 0.19375, "reward_std": 0.3718110263347626, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.3718110382556915, "sampling/importance_sampling_ratio/max": 1.2591856479644776, "sampling/importance_sampling_ratio/mean": 1.0005742013454437, "sampling/importance_sampling_ratio/min": 0.7635951817035675, "sampling/sampling_logp_difference/max": 0.2940999746322632, "sampling/sampling_logp_difference/mean": 0.0213538846001029, "step": 5350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 209.225, "completions/mean_terminated_length": 209.225, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.1188813909888267, "epoch": 0.49278293647145355, "frac_reward_zero_std": 0.4, "grad_norm": 6.364650249481201, "kl": 0.013982200878672302, "learning_rate": 4.8237765348803435e-06, "loss": 0.0002, "num_tokens": 23282579.0, "reward": -0.575, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2591118574142457, "sampling/importance_sampling_ratio/mean": 0.9996820330619812, "sampling/importance_sampling_ratio/min": 0.7434278100728988, "sampling/sampling_logp_difference/max": 0.34676127433776854, "sampling/sampling_logp_difference/mean": 0.02129863128066063, "step": 5360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.1, "completions/max_terminated_length": 306.1, "completions/mean_length": 215.5875, "completions/mean_terminated_length": 215.5875, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.0398186855018139, "epoch": 0.49370230762158684, "frac_reward_zero_std": 0.3, "grad_norm": 0.11289431154727936, "kl": 0.013851236569462345, "learning_rate": 4.8228409528383595e-06, "loss": 0.0007, "num_tokens": 23326042.0, "reward": -0.25, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.3056758880615233, "sampling/importance_sampling_ratio/mean": 1.0002526104450227, "sampling/importance_sampling_ratio/min": 0.7420894891023636, "sampling/sampling_logp_difference/max": 0.366279137134552, "sampling/sampling_logp_difference/mean": 0.020458483323454856, "step": 5370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 206.4125, "completions/mean_terminated_length": 206.4125, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0489323753863573, "epoch": 0.49462167877172014, "frac_reward_zero_std": 0.4, "grad_norm": 6.084827899932861, "kl": 0.011888790974626317, "learning_rate": 4.821902985052518e-06, "loss": 0.0006, "num_tokens": 23367331.0, "reward": -0.075, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.245952308177948, "sampling/importance_sampling_ratio/mean": 0.9998699486255646, "sampling/importance_sampling_ratio/min": 0.789851975440979, "sampling/sampling_logp_difference/max": 0.24973161220550538, "sampling/sampling_logp_difference/mean": 0.020778874307870863, "step": 5380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.5, "completions/max_terminated_length": 307.5, "completions/mean_length": 225.4125, "completions/mean_terminated_length": 225.4125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.1045550968497992, "epoch": 0.49554104992185344, "frac_reward_zero_std": 0.3, "grad_norm": 5.143423080444336, "kl": 0.013347972341580316, "learning_rate": 4.820962632486187e-06, "loss": 0.0005, "num_tokens": 23412612.0, "reward": 0.04375, "reward_std": 0.5878328442573547, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.5878328740596771, "sampling/importance_sampling_ratio/max": 1.263407003879547, "sampling/importance_sampling_ratio/mean": 0.9997262120246887, "sampling/importance_sampling_ratio/min": 0.7919655740261078, "sampling/sampling_logp_difference/max": 0.25642826557159426, "sampling/sampling_logp_difference/mean": 0.020921301655471326, "step": 5390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.3, "completions/max_terminated_length": 296.3, "completions/mean_length": 209.475, "completions/mean_terminated_length": 209.475, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.0908569373190402, "epoch": 0.49646042107198673, "frac_reward_zero_std": 0.6, "grad_norm": 6.546214580535889, "kl": 0.011947426141705363, "learning_rate": 4.820019896105186e-06, "loss": 0.0006, "num_tokens": 23455906.0, "reward": 0.05, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2964376091957093, "sampling/importance_sampling_ratio/mean": 1.0003369092941283, "sampling/importance_sampling_ratio/min": 0.7877450942993164, "sampling/sampling_logp_difference/max": 0.2662349581718445, "sampling/sampling_logp_difference/mean": 0.020823153108358382, "step": 5400 }, { "epoch": 0.49646042107198673, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 302.20767888307154, "eval_completions/max_terminated_length": 298.9458987783595, "eval_completions/mean_length": 211.0098167539267, "eval_completions/mean_terminated_length": 210.34835477233142, "eval_completions/min_length": 139.00698080279233, "eval_completions/min_terminated_length": 139.00698080279233, "eval_entropy": 1.1314031309066226, "eval_frac_reward_zero_std": 0.42757417102966844, "eval_kl": 0.013241767370504495, "eval_loss": 0.0007309522479772568, "eval_num_tokens": 23455906.0, "eval_reward": -0.1174738219895288, "eval_reward_std": 0.5060937461844706, "eval_rewards/reward_fn/mean": -0.1174738219895288, "eval_rewards/reward_fn/std": 0.5060937644923544, "eval_runtime": 1757.3638, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.264326042321756, "eval_sampling/importance_sampling_ratio/mean": 1.0000120811229394, "eval_sampling/importance_sampling_ratio/min": 0.7821921814902707, "eval_sampling/sampling_logp_difference/max": 0.27573814574648575, "eval_sampling/sampling_logp_difference/mean": 0.020720751463625755, "eval_steps_per_second": 0.041, "step": 5400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 209.0625, "completions/mean_terminated_length": 209.0625, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.1472887329757213, "epoch": 0.4973797922221201, "frac_reward_zero_std": 0.4, "grad_norm": 0.11002101004123688, "kl": 0.013965883711352944, "learning_rate": 4.819074776877778e-06, "loss": 0.0011, "num_tokens": 23504327.0, "reward": -0.00625, "reward_std": 0.4368450567126274, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.4368450567126274, "sampling/importance_sampling_ratio/max": 1.2640942811965943, "sampling/importance_sampling_ratio/mean": 0.9997585356235504, "sampling/importance_sampling_ratio/min": 0.7546513915061951, "sampling/sampling_logp_difference/max": 0.30907121300697327, "sampling/sampling_logp_difference/mean": 0.021198511868715287, "step": 5410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.5, "completions/max_terminated_length": 302.9, "completions/mean_length": 206.375, "completions/mean_terminated_length": 203.42857360839844, "completions/min_length": 117.2, "completions/min_terminated_length": 117.2, "entropy": 1.0934120383113624, "epoch": 0.4982991633722534, "frac_reward_zero_std": 0.2, "grad_norm": 5.350105285644531, "kl": 0.012444861727999523, "learning_rate": 4.818127275774678e-06, "loss": 0.0008, "num_tokens": 23547805.0, "reward": -0.21875, "reward_std": 0.6630949661135673, "rewards/reward_fn/mean": -0.21875, "rewards/reward_fn/std": 0.6630949720740318, "sampling/importance_sampling_ratio/max": 1.2614344239234925, "sampling/importance_sampling_ratio/mean": 1.0001213490962981, "sampling/importance_sampling_ratio/min": 0.7881251037120819, "sampling/sampling_logp_difference/max": 0.25891963243484495, "sampling/sampling_logp_difference/mean": 0.02116831112653017, "step": 5420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.1, "completions/max_terminated_length": 333.1, "completions/mean_length": 228.15, "completions/mean_terminated_length": 228.15, "completions/min_length": 147.2, "completions/min_terminated_length": 147.2, "entropy": 1.209950478374958, "epoch": 0.4992185345223867, "frac_reward_zero_std": 0.4, "grad_norm": 0.15530167520046234, "kl": 0.014559066510992124, "learning_rate": 4.817177393769046e-06, "loss": 0.0009, "num_tokens": 23596289.0, "reward": -0.2, "reward_std": 0.4898623704910278, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.48986239433288575, "sampling/importance_sampling_ratio/max": 1.26696435213089, "sampling/importance_sampling_ratio/mean": 0.9997070014476777, "sampling/importance_sampling_ratio/min": 0.7783333122730255, "sampling/sampling_logp_difference/max": 0.26759564876556396, "sampling/sampling_logp_difference/mean": 0.022634914517402648, "step": 5430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.0, "completions/max_terminated_length": 307.0, "completions/mean_length": 199.15, "completions/mean_terminated_length": 199.15, "completions/min_length": 113.1, "completions/min_terminated_length": 113.1, "entropy": 1.061456163972616, "epoch": 0.50013790567252, "frac_reward_zero_std": 0.7, "grad_norm": 0.0997234359383583, "kl": 0.013758428615983576, "learning_rate": 4.816225131836485e-06, "loss": 0.0007, "num_tokens": 23639181.0, "reward": 0.125, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2573357582092286, "sampling/importance_sampling_ratio/mean": 1.0001139640808105, "sampling/importance_sampling_ratio/min": 0.772485613822937, "sampling/sampling_logp_difference/max": 0.26308544874191286, "sampling/sampling_logp_difference/mean": 0.02156846094876528, "step": 5440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 211.5125, "completions/mean_terminated_length": 211.5125, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.0206796541810035, "epoch": 0.5010572768226533, "frac_reward_zero_std": 0.4, "grad_norm": 7.525967121124268, "kl": 0.012035417385050096, "learning_rate": 4.8152704909550455e-06, "loss": 0.0004, "num_tokens": 23683462.0, "reward": -0.175, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2331228494644164, "sampling/importance_sampling_ratio/mean": 1.00036381483078, "sampling/importance_sampling_ratio/min": 0.7997820913791657, "sampling/sampling_logp_difference/max": 0.23312923908233643, "sampling/sampling_logp_difference/mean": 0.01993821132928133, "step": 5450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.9, "completions/max_terminated_length": 295.9, "completions/mean_length": 202.2, "completions/mean_terminated_length": 202.2, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.141084360331297, "epoch": 0.5019766479727866, "frac_reward_zero_std": 0.5, "grad_norm": 0.11676759272813797, "kl": 0.012936891103163362, "learning_rate": 4.8143134721052205e-06, "loss": 0.0006, "num_tokens": 23727934.0, "reward": -0.15, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.245789122581482, "sampling/importance_sampling_ratio/mean": 1.0004112422466278, "sampling/importance_sampling_ratio/min": 0.8063056528568268, "sampling/sampling_logp_difference/max": 0.23539336323738097, "sampling/sampling_logp_difference/mean": 0.02115023285150528, "step": 5460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.5, "completions/max_terminated_length": 287.5, "completions/mean_length": 187.4875, "completions/mean_terminated_length": 187.4875, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0204409040510654, "epoch": 0.5028960191229199, "frac_reward_zero_std": 0.5, "grad_norm": 6.3619561195373535, "kl": 0.012006750126602128, "learning_rate": 4.8133540762699445e-06, "loss": 0.0006, "num_tokens": 23770189.0, "reward": -0.225, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2648565173149109, "sampling/importance_sampling_ratio/mean": 1.0001740336418152, "sampling/importance_sampling_ratio/min": 0.7793761372566224, "sampling/sampling_logp_difference/max": 0.2675227761268616, "sampling/sampling_logp_difference/mean": 0.020210181549191474, "step": 5470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.4, "completions/max_terminated_length": 257.4, "completions/mean_length": 186.925, "completions/mean_terminated_length": 186.925, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0395287483930589, "epoch": 0.5038153902730532, "frac_reward_zero_std": 0.4, "grad_norm": 6.543190956115723, "kl": 0.01438865253003314, "learning_rate": 4.812392304434595e-06, "loss": 0.0004, "num_tokens": 23810319.0, "reward": -0.1, "reward_std": 0.5916701912879944, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5916702151298523, "sampling/importance_sampling_ratio/max": 1.24901340007782, "sampling/importance_sampling_ratio/mean": 1.0003915131092072, "sampling/importance_sampling_ratio/min": 0.7848486006259918, "sampling/sampling_logp_difference/max": 0.2524983763694763, "sampling/sampling_logp_difference/mean": 0.020734318159520627, "step": 5480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 233.2, "completions/max_terminated_length": 233.2, "completions/mean_length": 174.3625, "completions/mean_terminated_length": 174.3625, "completions/min_length": 115.4, "completions/min_terminated_length": 115.4, "entropy": 1.0270351819694041, "epoch": 0.5047347614231865, "frac_reward_zero_std": 0.9, "grad_norm": 0.22074264287948608, "kl": 0.012975584605010226, "learning_rate": 4.811428157586987e-06, "loss": 0.0006, "num_tokens": 23849132.0, "reward": 0.175, "reward_std": 0.07071067690849304, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.07071067690849304, "sampling/importance_sampling_ratio/max": 1.2753729701042176, "sampling/importance_sampling_ratio/mean": 0.9995200753211975, "sampling/importance_sampling_ratio/min": 0.7577791303396225, "sampling/sampling_logp_difference/max": 0.30765478014945985, "sampling/sampling_logp_difference/mean": 0.02118015196174383, "step": 5490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.0, "completions/max_terminated_length": 273.0, "completions/mean_length": 206.725, "completions/mean_terminated_length": 206.725, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0564965590834619, "epoch": 0.5056541325733198, "frac_reward_zero_std": 0.7, "grad_norm": 5.22079610824585, "kl": 0.012468088397872635, "learning_rate": 4.810461636717378e-06, "loss": 0.0004, "num_tokens": 23895278.0, "reward": 0.1, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.3001884341239929, "sampling/importance_sampling_ratio/mean": 0.9997402429580688, "sampling/importance_sampling_ratio/min": 0.7170394629240036, "sampling/sampling_logp_difference/max": 0.3888967037200928, "sampling/sampling_logp_difference/mean": 0.020326339080929755, "step": 5500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 216.7375, "completions/mean_terminated_length": 216.7375, "completions/min_length": 152.4, "completions/min_terminated_length": 152.4, "entropy": 1.0100433390587569, "epoch": 0.5065735037234531, "frac_reward_zero_std": 0.3, "grad_norm": 5.058108329772949, "kl": 0.01208488644915633, "learning_rate": 4.809492742818461e-06, "loss": 0.0006, "num_tokens": 23938777.0, "reward": -0.125, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.2595778822898864, "sampling/importance_sampling_ratio/mean": 0.999910318851471, "sampling/importance_sampling_ratio/min": 0.7698951959609985, "sampling/sampling_logp_difference/max": 0.28226454854011535, "sampling/sampling_logp_difference/mean": 0.02002736460417509, "step": 5510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.5, "completions/max_terminated_length": 308.5, "completions/mean_length": 216.0875, "completions/mean_terminated_length": 216.0875, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.102257677167654, "epoch": 0.5074928748735865, "frac_reward_zero_std": 0.3, "grad_norm": 4.261312484741211, "kl": 0.012206951103871688, "learning_rate": 4.808521476885369e-06, "loss": 0.0007, "num_tokens": 23983784.0, "reward": -0.225, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.2433959603309632, "sampling/importance_sampling_ratio/mean": 0.9998415231704711, "sampling/importance_sampling_ratio/min": 0.7964862942695617, "sampling/sampling_logp_difference/max": 0.2423648416996002, "sampling/sampling_logp_difference/mean": 0.020834720879793166, "step": 5520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 217.55, "completions/mean_terminated_length": 217.55, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.0657261833548546, "epoch": 0.5084122460237198, "frac_reward_zero_std": 0.6, "grad_norm": 0.12047350406646729, "kl": 0.011912653601029888, "learning_rate": 4.80754783991567e-06, "loss": 0.0005, "num_tokens": 24030204.0, "reward": 0.375, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2634344339370727, "sampling/importance_sampling_ratio/mean": 0.999928230047226, "sampling/importance_sampling_ratio/min": 0.7900159120559692, "sampling/sampling_logp_difference/max": 0.2548913836479187, "sampling/sampling_logp_difference/mean": 0.020677041634917258, "step": 5530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.1, "completions/max_terminated_length": 239.1, "completions/mean_length": 169.35, "completions/mean_terminated_length": 169.35, "completions/min_length": 107.0, "completions/min_terminated_length": 107.0, "entropy": 0.9958102196455002, "epoch": 0.5093316171738531, "frac_reward_zero_std": 0.5, "grad_norm": 7.196700572967529, "kl": 0.011013422248652205, "learning_rate": 4.806571832909367e-06, "loss": 0.0003, "num_tokens": 24072480.0, "reward": -0.2, "reward_std": 0.3897472023963928, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3897472023963928, "sampling/importance_sampling_ratio/max": 1.2437776327133179, "sampling/importance_sampling_ratio/mean": 0.9998109757900238, "sampling/importance_sampling_ratio/min": 0.7684959948062897, "sampling/sampling_logp_difference/max": 0.27728047370910647, "sampling/sampling_logp_difference/mean": 0.020350584015250205, "step": 5540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.6, "completions/max_terminated_length": 304.6, "completions/mean_length": 202.05, "completions/mean_terminated_length": 202.05, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.061276477575302, "epoch": 0.5102509883239864, "frac_reward_zero_std": 0.6, "grad_norm": 0.12790052592754364, "kl": 0.012859805760672315, "learning_rate": 4.805593456868897e-06, "loss": 0.0007, "num_tokens": 24114164.0, "reward": 0.1, "reward_std": 0.38124017119407655, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3812402069568634, "sampling/importance_sampling_ratio/max": 1.2899776101112366, "sampling/importance_sampling_ratio/mean": 1.0002273142337799, "sampling/importance_sampling_ratio/min": 0.8043971598148346, "sampling/sampling_logp_difference/max": 0.27183390259742735, "sampling/sampling_logp_difference/mean": 0.02047591283917427, "step": 5550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 195.925, "completions/mean_terminated_length": 195.925, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.0310080267488957, "epoch": 0.5111703594741197, "frac_reward_zero_std": 0.4, "grad_norm": 0.10182178765535355, "kl": 0.01181789832189679, "learning_rate": 4.804612712799131e-06, "loss": 0.0007, "num_tokens": 24153390.0, "reward": -0.075, "reward_std": 0.5185230314731598, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5185230314731598, "sampling/importance_sampling_ratio/max": 1.2426900267601013, "sampling/importance_sampling_ratio/mean": 1.0001989662647248, "sampling/importance_sampling_ratio/min": 0.8045618116855622, "sampling/sampling_logp_difference/max": 0.2349104642868042, "sampling/sampling_logp_difference/mean": 0.0204125152900815, "step": 5560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 207.075, "completions/mean_terminated_length": 207.075, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.0188142098486423, "epoch": 0.512089730624253, "frac_reward_zero_std": 0.6, "grad_norm": 5.6374592781066895, "kl": 0.012737719033611938, "learning_rate": 4.803629601707371e-06, "loss": 0.0005, "num_tokens": 24195540.0, "reward": -0.375, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.298264217376709, "sampling/importance_sampling_ratio/mean": 1.0003949701786041, "sampling/importance_sampling_ratio/min": 0.7765696287155152, "sampling/sampling_logp_difference/max": 0.3073842763900757, "sampling/sampling_logp_difference/mean": 0.020265903696417807, "step": 5570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.2, "completions/max_terminated_length": 275.2, "completions/mean_length": 181.325, "completions/mean_terminated_length": 181.325, "completions/min_length": 104.0, "completions/min_terminated_length": 104.0, "entropy": 1.1231561124324798, "epoch": 0.5130091017743863, "frac_reward_zero_std": 0.4, "grad_norm": 6.192976951599121, "kl": 0.012620203191181645, "learning_rate": 4.802644124603352e-06, "loss": 0.0006, "num_tokens": 24236366.0, "reward": -0.15, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2735597133636474, "sampling/importance_sampling_ratio/mean": 0.9996258080005646, "sampling/importance_sampling_ratio/min": 0.7993391871452331, "sampling/sampling_logp_difference/max": 0.2664495587348938, "sampling/sampling_logp_difference/mean": 0.022292640060186386, "step": 5580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.4, "completions/max_terminated_length": 319.4, "completions/mean_length": 225.0375, "completions/mean_terminated_length": 225.0375, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0064160905778408, "epoch": 0.5139284729245196, "frac_reward_zero_std": 0.4, "grad_norm": 7.175397872924805, "kl": 0.011607095994986594, "learning_rate": 4.8016562824992375e-06, "loss": 0.0006, "num_tokens": 24279545.0, "reward": -0.475, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.3221856951713562, "sampling/importance_sampling_ratio/mean": 1.0001377701759337, "sampling/importance_sampling_ratio/min": 0.8023458480834961, "sampling/sampling_logp_difference/max": 0.284383225440979, "sampling/sampling_logp_difference/mean": 0.020287402160465717, "step": 5590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.8, "completions/max_terminated_length": 293.8, "completions/mean_length": 213.9, "completions/mean_terminated_length": 213.9, "completions/min_length": 159.2, "completions/min_terminated_length": 159.2, "entropy": 1.029401994496584, "epoch": 0.5148478440746529, "frac_reward_zero_std": 0.2, "grad_norm": 6.135865211486816, "kl": 0.010404325259150938, "learning_rate": 4.80066607640962e-06, "loss": 0.0009, "num_tokens": 24324425.0, "reward": -0.10625, "reward_std": 0.7070265471935272, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.7070265769958496, "sampling/importance_sampling_ratio/max": 1.2880445957183837, "sampling/importance_sampling_ratio/mean": 0.9996644854545593, "sampling/importance_sampling_ratio/min": 0.7876709222793579, "sampling/sampling_logp_difference/max": 0.28111970126628877, "sampling/sampling_logp_difference/mean": 0.020287426002323628, "step": 5600 }, { "epoch": 0.5148478440746529, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 309.10994764397907, "eval_completions/max_terminated_length": 307.56195462478183, "eval_completions/mean_length": 215.3619109947644, "eval_completions/mean_terminated_length": 214.859729706929, "eval_completions/min_length": 139.06806282722513, "eval_completions/min_terminated_length": 139.06806282722513, "eval_entropy": 1.077312634305388, "eval_frac_reward_zero_std": 0.45200698080279234, "eval_kl": 0.012931814144185077, "eval_loss": 0.0005989734781906009, "eval_num_tokens": 24324425.0, "eval_reward": -0.10580279232111693, "eval_reward_std": 0.4875448613994825, "eval_rewards/reward_fn/mean": -0.10580279232111693, "eval_rewards/reward_fn/std": 0.4875448740901747, "eval_runtime": 1790.5778, "eval_samples_per_second": 0.32, "eval_sampling/importance_sampling_ratio/max": 1.2633877197901409, "eval_sampling/importance_sampling_ratio/mean": 0.999897960696961, "eval_sampling/importance_sampling_ratio/min": 0.7762342506522284, "eval_sampling/sampling_logp_difference/max": 0.28112938812889976, "eval_sampling/sampling_logp_difference/mean": 0.020560182515616296, "eval_steps_per_second": 0.04, "step": 5600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.5, "completions/max_terminated_length": 335.5, "completions/mean_length": 232.95, "completions/mean_terminated_length": 232.95, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.024285451695323, "epoch": 0.5157672152247862, "frac_reward_zero_std": 0.3, "grad_norm": 0.1359027475118637, "kl": 0.012994409503880889, "learning_rate": 4.799673507351522e-06, "loss": 0.0005, "num_tokens": 24366837.0, "reward": -0.45, "reward_std": 0.6589862048625946, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.6589862406253815, "sampling/importance_sampling_ratio/max": 1.3210198163986206, "sampling/importance_sampling_ratio/mean": 1.0001651763916015, "sampling/importance_sampling_ratio/min": 0.7951807022094727, "sampling/sampling_logp_difference/max": 0.2995669722557068, "sampling/sampling_logp_difference/mean": 0.01991758532822132, "step": 5610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 209.45, "completions/mean_terminated_length": 209.45, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.0812566619366408, "epoch": 0.5166865863749196, "frac_reward_zero_std": 0.7, "grad_norm": 9.468076705932617, "kl": 0.012945756828412414, "learning_rate": 4.79867857634439e-06, "loss": 0.0006, "num_tokens": 24407689.0, "reward": 0.325, "reward_std": 0.31731882095336916, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.3173188328742981, "sampling/importance_sampling_ratio/max": 1.3243974924087525, "sampling/importance_sampling_ratio/mean": 1.000050288438797, "sampling/importance_sampling_ratio/min": 0.7775729596614838, "sampling/sampling_logp_difference/max": 0.31379027366638185, "sampling/sampling_logp_difference/mean": 0.02175551662221551, "step": 5620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.7, "completions/max_terminated_length": 283.7, "completions/mean_length": 208.0125, "completions/mean_terminated_length": 208.0125, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.0651330105960368, "epoch": 0.5176059575250529, "frac_reward_zero_std": 0.3, "grad_norm": 6.369870662689209, "kl": 0.013509562984108926, "learning_rate": 4.797681284410099e-06, "loss": 0.0003, "num_tokens": 24451914.0, "reward": -0.525, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.274602472782135, "sampling/importance_sampling_ratio/mean": 0.9999897718429566, "sampling/importance_sampling_ratio/min": 0.7938518404960633, "sampling/sampling_logp_difference/max": 0.2671345710754395, "sampling/sampling_logp_difference/mean": 0.020736329909414052, "step": 5630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.4, "completions/max_terminated_length": 307.9, "completions/mean_length": 225.4, "completions/mean_terminated_length": 222.07500152587892, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0472074504941702, "epoch": 0.5185253286751862, "frac_reward_zero_std": 0.6, "grad_norm": 0.1661979854106903, "kl": 0.012173630495090038, "learning_rate": 4.796681632572946e-06, "loss": 0.0007, "num_tokens": 24492674.0, "reward": -0.63125, "reward_std": 0.2878748521208763, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.2878748521208763, "sampling/importance_sampling_ratio/max": 1.2330743432044984, "sampling/importance_sampling_ratio/mean": 1.0003144919872284, "sampling/importance_sampling_ratio/min": 0.7894923031330109, "sampling/sampling_logp_difference/max": 0.2474573254585266, "sampling/sampling_logp_difference/mean": 0.020335477590560914, "step": 5640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 350.4, "completions/max_terminated_length": 335.5, "completions/mean_length": 237.0625, "completions/mean_terminated_length": 234.52142944335938, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.111094280332327, "epoch": 0.5194446998253195, "frac_reward_zero_std": 0.3, "grad_norm": 0.12364870309829712, "kl": 0.012772416666848585, "learning_rate": 4.795679621859656e-06, "loss": 0.0008, "num_tokens": 24539583.0, "reward": 0.01875, "reward_std": 0.6753024876117706, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.6753025412559509, "sampling/importance_sampling_ratio/max": 1.2698575854301453, "sampling/importance_sampling_ratio/mean": 1.0006968975067139, "sampling/importance_sampling_ratio/min": 0.7308303058147431, "sampling/sampling_logp_difference/max": 0.34610280990600584, "sampling/sampling_logp_difference/mean": 0.02118680104613304, "step": 5650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.1, "completions/max_terminated_length": 290.1, "completions/mean_length": 199.2, "completions/mean_terminated_length": 199.2, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0550979226827621, "epoch": 0.5203640709754528, "frac_reward_zero_std": 0.7, "grad_norm": 0.11829140037298203, "kl": 0.01127256482723169, "learning_rate": 4.794675253299373e-06, "loss": 0.0007, "num_tokens": 24583095.0, "reward": -0.4, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2564857363700868, "sampling/importance_sampling_ratio/mean": 1.0001864075660705, "sampling/importance_sampling_ratio/min": 0.7841481029987335, "sampling/sampling_logp_difference/max": 0.2681380271911621, "sampling/sampling_logp_difference/mean": 0.020303083769977092, "step": 5660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 213.975, "completions/mean_terminated_length": 213.975, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.019645094871521, "epoch": 0.5212834421255861, "frac_reward_zero_std": 0.4, "grad_norm": 0.158222496509552, "kl": 0.011743696633493528, "learning_rate": 4.793668527923664e-06, "loss": 0.0004, "num_tokens": 24630077.0, "reward": 0.15, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.2492385149002074, "sampling/importance_sampling_ratio/mean": 1.0000330805778503, "sampling/importance_sampling_ratio/min": 0.7678417325019836, "sampling/sampling_logp_difference/max": 0.2734883904457092, "sampling/sampling_logp_difference/mean": 0.019828339107334614, "step": 5670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 205.725, "completions/mean_terminated_length": 205.725, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.0771323665976524, "epoch": 0.5222028132757194, "frac_reward_zero_std": 0.5, "grad_norm": 0.1901135891675949, "kl": 0.013023847158183343, "learning_rate": 4.792659446766517e-06, "loss": 0.0005, "num_tokens": 24673839.0, "reward": 0.35, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.2885743975639343, "sampling/importance_sampling_ratio/mean": 1.0004958987236023, "sampling/importance_sampling_ratio/min": 0.7859337985515594, "sampling/sampling_logp_difference/max": 0.29449403285980225, "sampling/sampling_logp_difference/mean": 0.021139393746852874, "step": 5680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.0, "completions/max_terminated_length": 340.0, "completions/mean_length": 215.0, "completions/mean_terminated_length": 215.0, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0641091912984848, "epoch": 0.5231221844258527, "frac_reward_zero_std": 0.5, "grad_norm": 5.6918230056762695, "kl": 0.012756778084440156, "learning_rate": 4.791648010864338e-06, "loss": 0.0006, "num_tokens": 24717143.0, "reward": -0.35, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.3443294525146485, "sampling/importance_sampling_ratio/mean": 0.9997354388237, "sampling/importance_sampling_ratio/min": 0.71180160343647, "sampling/sampling_logp_difference/max": 0.39380708932876585, "sampling/sampling_logp_difference/mean": 0.021309465728700162, "step": 5690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 203.175, "completions/mean_terminated_length": 203.175, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.0442371241748334, "epoch": 0.524041555575986, "frac_reward_zero_std": 0.1, "grad_norm": 5.196913242340088, "kl": 0.016711584641598165, "learning_rate": 4.790634221255955e-06, "loss": 0.0005, "num_tokens": 24760781.0, "reward": 0.025, "reward_std": 0.7347935557365417, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.7347935914993287, "sampling/importance_sampling_ratio/max": 1.2870280981063842, "sampling/importance_sampling_ratio/mean": 0.9998199343681335, "sampling/importance_sampling_ratio/min": 0.7761616170406341, "sampling/sampling_logp_difference/max": 0.27709136009216306, "sampling/sampling_logp_difference/mean": 0.020994409918785095, "step": 5700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 200.425, "completions/mean_terminated_length": 200.425, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.1901666097342969, "epoch": 0.5249609267261194, "frac_reward_zero_std": 0.7, "grad_norm": 0.11156832426786423, "kl": 0.014623096626019105, "learning_rate": 4.789618078982609e-06, "loss": 0.0007, "num_tokens": 24806455.0, "reward": -0.05, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.2390235543251038, "sampling/importance_sampling_ratio/mean": 0.9997166275978089, "sampling/importance_sampling_ratio/min": 0.7965483963489532, "sampling/sampling_logp_difference/max": 0.24235846996307372, "sampling/sampling_logp_difference/mean": 0.022442874684929847, "step": 5710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.3, "completions/max_terminated_length": 300.3, "completions/mean_length": 210.1125, "completions/mean_terminated_length": 210.1125, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0483377762138844, "epoch": 0.5258802978762527, "frac_reward_zero_std": 0.4, "grad_norm": 6.434218883514404, "kl": 0.013688442422426306, "learning_rate": 4.788599585087961e-06, "loss": 0.0005, "num_tokens": 24849592.0, "reward": -0.025, "reward_std": 0.5807423710823059, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5807423830032349, "sampling/importance_sampling_ratio/max": 1.2479105591773987, "sampling/importance_sampling_ratio/mean": 0.999613630771637, "sampling/importance_sampling_ratio/min": 0.7721704721450806, "sampling/sampling_logp_difference/max": 0.2697384595870972, "sampling/sampling_logp_difference/mean": 0.02022982221096754, "step": 5720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.1, "completions/max_terminated_length": 321.1, "completions/mean_length": 222.95, "completions/mean_terminated_length": 222.95, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.1434067480266095, "epoch": 0.526799669026386, "frac_reward_zero_std": 0.5, "grad_norm": 0.09783146530389786, "kl": 0.013869909156346694, "learning_rate": 4.787578740618085e-06, "loss": 0.0007, "num_tokens": 24897388.0, "reward": -0.225, "reward_std": 0.49153932929039, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.49153935313224795, "sampling/importance_sampling_ratio/max": 1.22851322889328, "sampling/importance_sampling_ratio/mean": 0.9991890668869019, "sampling/importance_sampling_ratio/min": 0.7749674499034882, "sampling/sampling_logp_difference/max": 0.2589911460876465, "sampling/sampling_logp_difference/mean": 0.021487729530781506, "step": 5730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 343.5, "completions/max_terminated_length": 340.3, "completions/mean_length": 236.7125, "completions/mean_terminated_length": 230.725, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.045932705886662, "epoch": 0.5277190401765193, "frac_reward_zero_std": 0.6, "grad_norm": 0.12891311943531036, "kl": 0.013662449538242073, "learning_rate": 4.786555546621471e-06, "loss": 0.0007, "num_tokens": 24943989.0, "reward": -0.3125, "reward_std": 0.38376863598823546, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.38376864790916443, "sampling/importance_sampling_ratio/max": 1.2581586599349976, "sampling/importance_sampling_ratio/mean": 0.9993919730186462, "sampling/importance_sampling_ratio/min": 0.7921957612037659, "sampling/sampling_logp_difference/max": 0.26377347111701965, "sampling/sampling_logp_difference/mean": 0.020104332733899354, "step": 5740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.2, "completions/max_terminated_length": 323.2, "completions/mean_length": 226.925, "completions/mean_terminated_length": 226.925, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.1828702993690967, "epoch": 0.5286384113266526, "frac_reward_zero_std": 0.4, "grad_norm": 3.8999993801116943, "kl": 0.014874731795862317, "learning_rate": 4.78553000414902e-06, "loss": 0.0006, "num_tokens": 24991127.0, "reward": -0.28125, "reward_std": 0.40403020828962327, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.40403022021055224, "sampling/importance_sampling_ratio/max": 1.268424355983734, "sampling/importance_sampling_ratio/mean": 1.0000215470790863, "sampling/importance_sampling_ratio/min": 0.7695874631404876, "sampling/sampling_logp_difference/max": 0.2935630798339844, "sampling/sampling_logp_difference/mean": 0.022664897702634336, "step": 5750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 359.6, "completions/max_terminated_length": 352.7, "completions/mean_length": 227.9125, "completions/mean_terminated_length": 225.49285888671875, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.0773141857236623, "epoch": 0.5295577824767859, "frac_reward_zero_std": 0.3, "grad_norm": 4.6747822761535645, "kl": 0.012194506890955381, "learning_rate": 4.784502114254048e-06, "loss": 0.0006, "num_tokens": 25033264.0, "reward": -0.15625, "reward_std": 0.5184835538268089, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.5184835657477379, "sampling/importance_sampling_ratio/max": 1.2946914076805114, "sampling/importance_sampling_ratio/mean": 1.0000840425491333, "sampling/importance_sampling_ratio/min": 0.7902414619922637, "sampling/sampling_logp_difference/max": 0.2811056554317474, "sampling/sampling_logp_difference/mean": 0.020638069324195384, "step": 5760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.4, "completions/max_terminated_length": 325.4, "completions/mean_length": 225.85, "completions/mean_terminated_length": 225.85, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.0822862647473812, "epoch": 0.5304771536269192, "frac_reward_zero_std": 0.3, "grad_norm": 0.10757394134998322, "kl": 0.013958663714583963, "learning_rate": 4.783471877992279e-06, "loss": 0.0006, "num_tokens": 25076068.0, "reward": -0.20625, "reward_std": 0.711001181602478, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.711001205444336, "sampling/importance_sampling_ratio/max": 1.255208671092987, "sampling/importance_sampling_ratio/mean": 1.0004118382930756, "sampling/importance_sampling_ratio/min": 0.7655411481857299, "sampling/sampling_logp_difference/max": 0.29288363456726074, "sampling/sampling_logp_difference/mean": 0.0206966757774353, "step": 5770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.3, "completions/max_terminated_length": 339.3, "completions/mean_length": 217.675, "completions/mean_terminated_length": 217.675, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0765034407377243, "epoch": 0.5313965247770525, "frac_reward_zero_std": 0.5, "grad_norm": 0.2319401204586029, "kl": 0.013714697383693419, "learning_rate": 4.782439296421849e-06, "loss": 0.001, "num_tokens": 25119538.0, "reward": -0.2, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2829017162322998, "sampling/importance_sampling_ratio/mean": 0.9998002886772156, "sampling/importance_sampling_ratio/min": 0.7691384792327881, "sampling/sampling_logp_difference/max": 0.29528141021728516, "sampling/sampling_logp_difference/mean": 0.021307608857750893, "step": 5780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.2, "completions/max_terminated_length": 326.2, "completions/mean_length": 218.725, "completions/mean_terminated_length": 218.725, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.072419425100088, "epoch": 0.5323158959271858, "frac_reward_zero_std": 0.6, "grad_norm": 0.13599687814712524, "kl": 0.014312557526864111, "learning_rate": 4.781404370603301e-06, "loss": 0.0007, "num_tokens": 25163356.0, "reward": 0.075, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2698535799980164, "sampling/importance_sampling_ratio/mean": 1.0001352369785308, "sampling/importance_sampling_ratio/min": 0.7682908773422241, "sampling/sampling_logp_difference/max": 0.28666980266571046, "sampling/sampling_logp_difference/mean": 0.021728724241256714, "step": 5790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.7, "completions/max_terminated_length": 323.9, "completions/mean_length": 234.8875, "completions/mean_terminated_length": 232.40178833007812, "completions/min_length": 167.1, "completions/min_terminated_length": 167.1, "entropy": 1.0606774665415286, "epoch": 0.5332352670773192, "frac_reward_zero_std": 0.2, "grad_norm": 5.727106094360352, "kl": 0.013875321252271533, "learning_rate": 4.7803671015995875e-06, "loss": 0.0006, "num_tokens": 25207027.0, "reward": -0.10625, "reward_std": 0.6110655650496483, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6110655769705773, "sampling/importance_sampling_ratio/max": 1.2718957662582397, "sampling/importance_sampling_ratio/mean": 0.9996731340885162, "sampling/importance_sampling_ratio/min": 0.7336752355098725, "sampling/sampling_logp_difference/max": 0.3439004302024841, "sampling/sampling_logp_difference/mean": 0.020361351780593395, "step": 5800 }, { "epoch": 0.5332352670773192, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.005017452006980803, "eval_completions/max_length": 314.869109947644, "eval_completions/max_terminated_length": 311.04537521815007, "eval_completions/mean_length": 218.5479930191972, "eval_completions/mean_terminated_length": 217.44007149208696, "eval_completions/min_length": 139.78534031413614, "eval_completions/min_terminated_length": 139.78534031413614, "eval_entropy": 1.0996763176734952, "eval_frac_reward_zero_std": 0.4467713787085515, "eval_kl": 0.013363690886737528, "eval_loss": 0.0006485127960331738, "eval_num_tokens": 25207027.0, "eval_reward": -0.11900087260034904, "eval_reward_std": 0.4849372269751514, "eval_rewards/reward_fn/mean": -0.11900087260034904, "eval_rewards/reward_fn/std": 0.4849372424744395, "eval_runtime": 1820.298, "eval_samples_per_second": 0.315, "eval_sampling/importance_sampling_ratio/max": 1.2677449545519097, "eval_sampling/importance_sampling_ratio/mean": 0.999961303062672, "eval_sampling/importance_sampling_ratio/min": 0.7688543352625132, "eval_sampling/sampling_logp_difference/max": 0.2915854482646596, "eval_sampling/sampling_logp_difference/mean": 0.02110626250602968, "eval_steps_per_second": 0.04, "step": 5800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.6, "completions/max_terminated_length": 291.6, "completions/mean_length": 207.4875, "completions/mean_terminated_length": 207.4875, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0189701668918132, "epoch": 0.5341546382274525, "frac_reward_zero_std": 0.3, "grad_norm": 4.993710041046143, "kl": 0.01925445455708541, "learning_rate": 4.779327490476068e-06, "loss": 0.001, "num_tokens": 25249042.0, "reward": -0.2, "reward_std": 0.7019536554813385, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.7019536912441253, "sampling/importance_sampling_ratio/max": 1.2812476754188538, "sampling/importance_sampling_ratio/mean": 0.999680095911026, "sampling/importance_sampling_ratio/min": 0.7489375829696655, "sampling/sampling_logp_difference/max": 0.3147422671318054, "sampling/sampling_logp_difference/mean": 0.0202588165178895, "step": 5810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 362.3, "completions/max_terminated_length": 362.3, "completions/mean_length": 248.3875, "completions/mean_terminated_length": 248.3875, "completions/min_length": 163.7, "completions/min_terminated_length": 163.7, "entropy": 1.0884426210075617, "epoch": 0.5350740093775858, "frac_reward_zero_std": 0.2, "grad_norm": 5.027644157409668, "kl": 0.014840269880369305, "learning_rate": 4.778285538300505e-06, "loss": 0.0009, "num_tokens": 25297633.0, "reward": -0.25625, "reward_std": 0.6611360728740692, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.6611360907554626, "sampling/importance_sampling_ratio/max": 1.2799265503883361, "sampling/importance_sampling_ratio/mean": 1.0002203047275544, "sampling/importance_sampling_ratio/min": 0.7770942866802215, "sampling/sampling_logp_difference/max": 0.2828426122665405, "sampling/sampling_logp_difference/mean": 0.02104938505217433, "step": 5820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.6, "completions/max_terminated_length": 324.6, "completions/mean_length": 230.0375, "completions/mean_terminated_length": 230.0375, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.1549418196082115, "epoch": 0.535993380527719, "frac_reward_zero_std": 0.5, "grad_norm": 0.10164506733417511, "kl": 0.015998885239241645, "learning_rate": 4.777241246143066e-06, "loss": 0.0008, "num_tokens": 25344372.0, "reward": -0.275, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.377225661277771, "sampling/importance_sampling_ratio/mean": 0.9996744215488433, "sampling/importance_sampling_ratio/min": 0.7891199171543122, "sampling/sampling_logp_difference/max": 0.32260265946388245, "sampling/sampling_logp_difference/mean": 0.02140926234424114, "step": 5830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.7, "completions/max_terminated_length": 295.7, "completions/mean_length": 219.375, "completions/mean_terminated_length": 219.375, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0629051156342029, "epoch": 0.5369127516778524, "frac_reward_zero_std": 0.2, "grad_norm": 5.42703914642334, "kl": 0.013479963777353988, "learning_rate": 4.776194615076325e-06, "loss": 0.0009, "num_tokens": 25387106.0, "reward": 0.2, "reward_std": 0.6893488764762878, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.6893489003181458, "sampling/importance_sampling_ratio/max": 1.2341394543647766, "sampling/importance_sampling_ratio/mean": 1.000399100780487, "sampling/importance_sampling_ratio/min": 0.7702701151371002, "sampling/sampling_logp_difference/max": 0.2712538123130798, "sampling/sampling_logp_difference/mean": 0.020386701449751854, "step": 5840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.5, "completions/max_terminated_length": 348.5, "completions/mean_length": 243.2625, "completions/mean_terminated_length": 243.2625, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.11024968624115, "epoch": 0.5378321228279856, "frac_reward_zero_std": 0.0, "grad_norm": 5.673213958740234, "kl": 0.012892712600296364, "learning_rate": 4.7751456461752545e-06, "loss": 0.001, "num_tokens": 25432879.0, "reward": 0.4, "reward_std": 0.8164477467536926, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.8164477705955505, "sampling/importance_sampling_ratio/max": 1.2513674974441529, "sampling/importance_sampling_ratio/mean": 1.0002239048480988, "sampling/importance_sampling_ratio/min": 0.7718429505825043, "sampling/sampling_logp_difference/max": 0.2686744570732117, "sampling/sampling_logp_difference/mean": 0.020937025360763074, "step": 5850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.5, "completions/max_terminated_length": 322.5, "completions/mean_length": 229.7125, "completions/mean_terminated_length": 229.7125, "completions/min_length": 152.4, "completions/min_terminated_length": 152.4, "entropy": 1.0275617584586143, "epoch": 0.538751493978119, "frac_reward_zero_std": 0.2, "grad_norm": 8.690839767456055, "kl": 0.01709414614015259, "learning_rate": 4.7740943405172305e-06, "loss": 0.0008, "num_tokens": 25480048.0, "reward": -0.025, "reward_std": 0.7836078464984894, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.7836078703403473, "sampling/importance_sampling_ratio/max": 1.2612192273139953, "sampling/importance_sampling_ratio/mean": 0.9995320677757263, "sampling/importance_sampling_ratio/min": 0.7836325228214264, "sampling/sampling_logp_difference/max": 0.27062546014785765, "sampling/sampling_logp_difference/mean": 0.020193849690258503, "step": 5860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 217.1875, "completions/mean_terminated_length": 217.1875, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0490927681326867, "epoch": 0.5396708651282522, "frac_reward_zero_std": 0.6, "grad_norm": 4.724264621734619, "kl": 0.015775913919787853, "learning_rate": 4.773040699182028e-06, "loss": 0.0007, "num_tokens": 25523415.0, "reward": -0.35, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2881056308746337, "sampling/importance_sampling_ratio/mean": 0.9999593734741211, "sampling/importance_sampling_ratio/min": 0.7652535498142242, "sampling/sampling_logp_difference/max": 0.29609904289245603, "sampling/sampling_logp_difference/mean": 0.020765377953648568, "step": 5870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 199.65, "completions/mean_terminated_length": 199.65, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.0508727468550205, "epoch": 0.5405902362783855, "frac_reward_zero_std": 0.6, "grad_norm": 5.931421279907227, "kl": 0.015407769929151983, "learning_rate": 4.77198472325182e-06, "loss": 0.0008, "num_tokens": 25562731.0, "reward": 0.1, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2904655694961549, "sampling/importance_sampling_ratio/mean": 1.0003722012043, "sampling/importance_sampling_ratio/min": 0.7797191798686981, "sampling/sampling_logp_difference/max": 0.27476840019226073, "sampling/sampling_logp_difference/mean": 0.02166856750845909, "step": 5880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.5, "completions/max_terminated_length": 273.5, "completions/mean_length": 202.5375, "completions/mean_terminated_length": 202.5375, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "entropy": 1.1124643728137016, "epoch": 0.541509607428519, "frac_reward_zero_std": 0.7, "grad_norm": 0.1027638390660286, "kl": 0.01457720571779646, "learning_rate": 4.77092641381118e-06, "loss": 0.0008, "num_tokens": 25604798.0, "reward": 0.225, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2681536674499512, "sampling/importance_sampling_ratio/mean": 1.000376296043396, "sampling/importance_sampling_ratio/min": 0.7829257607460022, "sampling/sampling_logp_difference/max": 0.27098660469055175, "sampling/sampling_logp_difference/mean": 0.021837439201772214, "step": 5890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.5, "completions/max_terminated_length": 274.5, "completions/mean_length": 180.4, "completions/mean_terminated_length": 180.4, "completions/min_length": 116.4, "completions/min_terminated_length": 116.4, "entropy": 0.9994997691363097, "epoch": 0.5424289785786522, "frac_reward_zero_std": 0.4, "grad_norm": 0.40534183382987976, "kl": 0.012700824716011994, "learning_rate": 4.769865771947076e-06, "loss": 0.0002, "num_tokens": 25643046.0, "reward": -0.45, "reward_std": 0.5588710367679596, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.5588710486888886, "sampling/importance_sampling_ratio/max": 1.2677251815795898, "sampling/importance_sampling_ratio/mean": 1.000241792201996, "sampling/importance_sampling_ratio/min": 0.6961112469434738, "sampling/sampling_logp_difference/max": 0.4111971199512482, "sampling/sampling_logp_difference/mean": 0.020308008790016173, "step": 5900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 320.4, "completions/max_terminated_length": 301.8, "completions/mean_length": 216.35, "completions/mean_terminated_length": 212.67678680419922, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.1358140379190445, "epoch": 0.5433483497287855, "frac_reward_zero_std": 0.4, "grad_norm": 0.13390712440013885, "kl": 0.01422555887256749, "learning_rate": 4.768802798748871e-06, "loss": 0.0005, "num_tokens": 25687362.0, "reward": -0.23125, "reward_std": 0.4835209012031555, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.483520919084549, "sampling/importance_sampling_ratio/max": 1.3104843735694884, "sampling/importance_sampling_ratio/mean": 0.9999956786632538, "sampling/importance_sampling_ratio/min": 0.7803399562835693, "sampling/sampling_logp_difference/max": 0.2876611053943634, "sampling/sampling_logp_difference/mean": 0.022167828306555747, "step": 5910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 205.475, "completions/mean_terminated_length": 205.475, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.091882048547268, "epoch": 0.5442677208789188, "frac_reward_zero_std": 0.4, "grad_norm": 6.003787517547607, "kl": 0.013287719379877671, "learning_rate": 4.767737495308327e-06, "loss": 0.0004, "num_tokens": 25729912.0, "reward": -0.5, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.306846010684967, "sampling/importance_sampling_ratio/mean": 1.0002390265464782, "sampling/importance_sampling_ratio/min": 0.7832334339618683, "sampling/sampling_logp_difference/max": 0.2847090601921082, "sampling/sampling_logp_difference/mean": 0.02133383955806494, "step": 5920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 197.175, "completions/mean_terminated_length": 197.175, "completions/min_length": 120.2, "completions/min_terminated_length": 120.2, "entropy": 1.1237472899258136, "epoch": 0.5451870920290521, "frac_reward_zero_std": 0.2, "grad_norm": 5.389667987823486, "kl": 0.013647226570174097, "learning_rate": 4.766669862719593e-06, "loss": 0.001, "num_tokens": 25775118.0, "reward": -0.25, "reward_std": 0.6422272384166717, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6422272503376008, "sampling/importance_sampling_ratio/max": 1.2515487551689148, "sampling/importance_sampling_ratio/mean": 1.0008442640304565, "sampling/importance_sampling_ratio/min": 0.7912187159061432, "sampling/sampling_logp_difference/max": 0.25519187450408937, "sampling/sampling_logp_difference/mean": 0.02098984979093075, "step": 5930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.1, "completions/max_terminated_length": 330.1, "completions/mean_length": 220.075, "completions/mean_terminated_length": 220.075, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.0494517967104913, "epoch": 0.5461064631791854, "frac_reward_zero_std": 0.4, "grad_norm": 0.18867653608322144, "kl": 0.013372844218974934, "learning_rate": 4.765599902079214e-06, "loss": 0.0006, "num_tokens": 25820772.0, "reward": -0.225, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2860673546791077, "sampling/importance_sampling_ratio/mean": 0.9998741984367371, "sampling/importance_sampling_ratio/min": 0.7669319272041321, "sampling/sampling_logp_difference/max": 0.2987280011177063, "sampling/sampling_logp_difference/mean": 0.020499254018068312, "step": 5940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.8, "completions/max_terminated_length": 277.8, "completions/mean_length": 192.6625, "completions/mean_terminated_length": 192.6625, "completions/min_length": 118.9, "completions/min_terminated_length": 118.9, "entropy": 0.9449721217155457, "epoch": 0.5470258343293187, "frac_reward_zero_std": 0.3, "grad_norm": 0.1300814300775528, "kl": 0.014289415656821802, "learning_rate": 4.764527614486125e-06, "loss": 0.0008, "num_tokens": 25861497.0, "reward": 0.15, "reward_std": 0.6944204986095428, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6944205105304718, "sampling/importance_sampling_ratio/max": 1.3483479976654054, "sampling/importance_sampling_ratio/mean": 0.9999773383140564, "sampling/importance_sampling_ratio/min": 0.7817942678928376, "sampling/sampling_logp_difference/max": 0.3092191696166992, "sampling/sampling_logp_difference/mean": 0.02019168920814991, "step": 5950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.9, "completions/max_terminated_length": 304.9, "completions/mean_length": 210.4875, "completions/mean_terminated_length": 210.4875, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.0155700504779817, "epoch": 0.547945205479452, "frac_reward_zero_std": 0.7, "grad_norm": 5.0496039390563965, "kl": 0.012978146015666426, "learning_rate": 4.763453001041653e-06, "loss": 0.0006, "num_tokens": 25907120.0, "reward": -0.05, "reward_std": 0.30639100074768066, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.30639100074768066, "sampling/importance_sampling_ratio/max": 1.227819550037384, "sampling/importance_sampling_ratio/mean": 1.0003583610057831, "sampling/importance_sampling_ratio/min": 0.7716256856918335, "sampling/sampling_logp_difference/max": 0.26632544994354246, "sampling/sampling_logp_difference/mean": 0.01973624397069216, "step": 5960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 210.1625, "completions/mean_terminated_length": 210.1625, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0913101822137832, "epoch": 0.5488645766295853, "frac_reward_zero_std": 0.7, "grad_norm": 0.18440456688404083, "kl": 0.016375915450043976, "learning_rate": 4.762376062849511e-06, "loss": 0.0009, "num_tokens": 25952069.0, "reward": -0.1, "reward_std": 0.29206851720809934, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.29206851720809934, "sampling/importance_sampling_ratio/max": 1.2553067088127137, "sampling/importance_sampling_ratio/mean": 0.9998726785182953, "sampling/importance_sampling_ratio/min": 0.7213925540447235, "sampling/sampling_logp_difference/max": 0.34263637065887453, "sampling/sampling_logp_difference/mean": 0.020892103388905525, "step": 5970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 204.275, "completions/mean_terminated_length": 204.275, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.0725830480456353, "epoch": 0.5497839477797186, "frac_reward_zero_std": 0.4, "grad_norm": 4.833893299102783, "kl": 0.012766585976351053, "learning_rate": 4.7612968010158e-06, "loss": 0.0008, "num_tokens": 25995747.0, "reward": -0.075, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2505171179771424, "sampling/importance_sampling_ratio/mean": 1.000273770093918, "sampling/importance_sampling_ratio/min": 0.7403906971216202, "sampling/sampling_logp_difference/max": 0.3298409402370453, "sampling/sampling_logp_difference/mean": 0.020893578603863715, "step": 5980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 313.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 215.15, "completions/mean_terminated_length": 208.14285888671876, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.0407246857881547, "epoch": 0.550703318929852, "frac_reward_zero_std": 0.3, "grad_norm": 0.11607225984334946, "kl": 0.012513871706323699, "learning_rate": 4.760215216649009e-06, "loss": 0.0003, "num_tokens": 26039327.0, "reward": 0.3125, "reward_std": 0.5286123737692833, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.5286123916506767, "sampling/importance_sampling_ratio/max": 1.2866267323493958, "sampling/importance_sampling_ratio/mean": 0.9998406052589417, "sampling/importance_sampling_ratio/min": 0.7963234782218933, "sampling/sampling_logp_difference/max": 0.25969159603118896, "sampling/sampling_logp_difference/mean": 0.020698292925953866, "step": 5990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.4, "completions/max_terminated_length": 273.4, "completions/mean_length": 183.5125, "completions/mean_terminated_length": 183.5125, "completions/min_length": 110.1, "completions/min_terminated_length": 110.1, "entropy": 0.9952455185353756, "epoch": 0.5516226900799853, "frac_reward_zero_std": 0.7, "grad_norm": 0.11781564354896545, "kl": 0.014348141878144816, "learning_rate": 4.759131310860012e-06, "loss": 0.001, "num_tokens": 26082064.0, "reward": 0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.226545798778534, "sampling/importance_sampling_ratio/mean": 1.0000622689723968, "sampling/importance_sampling_ratio/min": 0.7907076597213745, "sampling/sampling_logp_difference/max": 0.24009737968444825, "sampling/sampling_logp_difference/mean": 0.02033825032413006, "step": 6000 }, { "epoch": 0.5516226900799853, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 284.10122164048863, "eval_completions/max_terminated_length": 282.5828970331588, "eval_completions/mean_length": 198.33202443280976, "eval_completions/mean_terminated_length": 198.04425341754268, "eval_completions/min_length": 127.59336823734729, "eval_completions/min_terminated_length": 127.59336823734729, "eval_entropy": 1.0781713886930888, "eval_frac_reward_zero_std": 0.462478184991274, "eval_kl": 0.013988926975839946, "eval_loss": 0.0006732423207722604, "eval_num_tokens": 26082064.0, "eval_reward": -0.10700261780104713, "eval_reward_std": 0.4609910309782827, "eval_rewards/reward_fn/mean": -0.10700261780104713, "eval_rewards/reward_fn/std": 0.4609910471017032, "eval_runtime": 1663.4407, "eval_samples_per_second": 0.344, "eval_sampling/importance_sampling_ratio/max": 1.2675279643107042, "eval_sampling/importance_sampling_ratio/mean": 1.0000538894643334, "eval_sampling/importance_sampling_ratio/min": 0.7759438596314785, "eval_sampling/sampling_logp_difference/max": 0.2840364550420751, "eval_sampling/sampling_logp_difference/mean": 0.021085529502019626, "eval_steps_per_second": 0.043, "step": 6000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.7, "completions/max_terminated_length": 286.7, "completions/mean_length": 210.6125, "completions/mean_terminated_length": 210.6125, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 1.145802552998066, "epoch": 0.5525420612301186, "frac_reward_zero_std": 0.5, "grad_norm": 5.350851535797119, "kl": 0.01428521730704233, "learning_rate": 4.758045084762068e-06, "loss": 0.0008, "num_tokens": 26128513.0, "reward": -0.225, "reward_std": 0.4956935167312622, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.49569355249404906, "sampling/importance_sampling_ratio/max": 1.2695760607719422, "sampling/importance_sampling_ratio/mean": 0.9997400522232056, "sampling/importance_sampling_ratio/min": 0.764477264881134, "sampling/sampling_logp_difference/max": 0.3005372494459152, "sampling/sampling_logp_difference/mean": 0.022365072928369045, "step": 6010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 192.325, "completions/mean_terminated_length": 192.325, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 0.9755276355892419, "epoch": 0.5534614323802519, "frac_reward_zero_std": 0.3, "grad_norm": 7.13377571105957, "kl": 0.012205420935060829, "learning_rate": 4.756956539470817e-06, "loss": 0.0002, "num_tokens": 26175331.0, "reward": 0.225, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2699368834495544, "sampling/importance_sampling_ratio/mean": 0.9998356521129608, "sampling/importance_sampling_ratio/min": 0.7712970018386841, "sampling/sampling_logp_difference/max": 0.2919417262077332, "sampling/sampling_logp_difference/mean": 0.019777134247124194, "step": 6020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.3, "completions/max_terminated_length": 287.9, "completions/mean_length": 202.7625, "completions/mean_terminated_length": 199.46428680419922, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "entropy": 1.0450338199734688, "epoch": 0.5543808035303852, "frac_reward_zero_std": 0.5, "grad_norm": 0.11768748611211777, "kl": 0.012688575475476683, "learning_rate": 4.755865676104283e-06, "loss": 0.0009, "num_tokens": 26215096.0, "reward": 0.09375, "reward_std": 0.3800110697746277, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.3800110757350922, "sampling/importance_sampling_ratio/max": 1.281259799003601, "sampling/importance_sampling_ratio/mean": 0.9998457193374634, "sampling/importance_sampling_ratio/min": 0.7697804272174835, "sampling/sampling_logp_difference/max": 0.28608894944190977, "sampling/sampling_logp_difference/mean": 0.020863662101328374, "step": 6030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 205.2375, "completions/mean_terminated_length": 205.2375, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.0006029993295669, "epoch": 0.5553001746805185, "frac_reward_zero_std": 0.7, "grad_norm": 10.60975170135498, "kl": 0.014375544717768206, "learning_rate": 4.75477249578287e-06, "loss": 0.0007, "num_tokens": 26258691.0, "reward": -0.38125, "reward_std": 0.2604610502719879, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.2604610562324524, "sampling/importance_sampling_ratio/max": 1.2375682950019837, "sampling/importance_sampling_ratio/mean": 0.999636310338974, "sampling/importance_sampling_ratio/min": 0.7460236370563507, "sampling/sampling_logp_difference/max": 0.3033482789993286, "sampling/sampling_logp_difference/mean": 0.020259457267820837, "step": 6040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.4, "completions/max_terminated_length": 293.4, "completions/mean_length": 213.925, "completions/mean_terminated_length": 213.925, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.0531173691153526, "epoch": 0.5562195458306518, "frac_reward_zero_std": 0.6, "grad_norm": 7.8633623123168945, "kl": 0.011853812844492495, "learning_rate": 4.753676999629362e-06, "loss": 0.0008, "num_tokens": 26300517.0, "reward": -0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2454741954803468, "sampling/importance_sampling_ratio/mean": 0.9994049191474914, "sampling/importance_sampling_ratio/min": 0.7766666352748871, "sampling/sampling_logp_difference/max": 0.25464640855789183, "sampling/sampling_logp_difference/mean": 0.02105821408331394, "step": 6050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 337.6, "completions/max_terminated_length": 336.5, "completions/mean_length": 227.0, "completions/mean_terminated_length": 223.52142944335938, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.0954403568059206, "epoch": 0.5571389169807851, "frac_reward_zero_std": 0.4, "grad_norm": 0.17314232885837555, "kl": 0.012353494763374329, "learning_rate": 4.752579188768921e-06, "loss": 0.0007, "num_tokens": 26344653.0, "reward": -0.65625, "reward_std": 0.42590154260396956, "rewards/reward_fn/mean": -0.65625, "rewards/reward_fn/std": 0.42590155452489853, "sampling/importance_sampling_ratio/max": 1.299543070793152, "sampling/importance_sampling_ratio/mean": 0.9997314810752869, "sampling/importance_sampling_ratio/min": 0.7510980039834976, "sampling/sampling_logp_difference/max": 0.3439400792121887, "sampling/sampling_logp_difference/mean": 0.02037389623001218, "step": 6060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.1, "completions/max_terminated_length": 257.1, "completions/mean_length": 187.1125, "completions/mean_terminated_length": 187.1125, "completions/min_length": 113.6, "completions/min_terminated_length": 113.6, "entropy": 1.0706636257469655, "epoch": 0.5580582881309184, "frac_reward_zero_std": 0.3, "grad_norm": 8.519725799560547, "kl": 0.014121870504459367, "learning_rate": 4.7514790643290855e-06, "loss": 0.0007, "num_tokens": 26385942.0, "reward": -0.33125, "reward_std": 0.610797768831253, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.610797780752182, "sampling/importance_sampling_ratio/max": 1.2562537789344788, "sampling/importance_sampling_ratio/mean": 1.0003083884716033, "sampling/importance_sampling_ratio/min": 0.783939516544342, "sampling/sampling_logp_difference/max": 0.26503087878227233, "sampling/sampling_logp_difference/mean": 0.02097295578569174, "step": 6070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 208.4625, "completions/mean_terminated_length": 208.4625, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0669391669332982, "epoch": 0.5589776592810518, "frac_reward_zero_std": 0.3, "grad_norm": 5.628911972045898, "kl": 0.012988036865135656, "learning_rate": 4.750376627439774e-06, "loss": 0.0006, "num_tokens": 26428267.0, "reward": -0.15, "reward_std": 0.6917853593826294, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6917854070663452, "sampling/importance_sampling_ratio/max": 1.275505828857422, "sampling/importance_sampling_ratio/mean": 1.000207394361496, "sampling/importance_sampling_ratio/min": 0.7785300374031067, "sampling/sampling_logp_difference/max": 0.2704038739204407, "sampling/sampling_logp_difference/mean": 0.02142903208732605, "step": 6080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.7, "completions/max_terminated_length": 317.7, "completions/mean_length": 220.175, "completions/mean_terminated_length": 220.175, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.0709145329892635, "epoch": 0.5598970304311851, "frac_reward_zero_std": 0.6, "grad_norm": 7.0751824378967285, "kl": 0.011707639560336247, "learning_rate": 4.749271879233276e-06, "loss": 0.0007, "num_tokens": 26475113.0, "reward": 0.01875, "reward_std": 0.4080048441886902, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.4080048561096191, "sampling/importance_sampling_ratio/max": 1.2402038812637328, "sampling/importance_sampling_ratio/mean": 0.999874621629715, "sampling/importance_sampling_ratio/min": 0.7830210030078888, "sampling/sampling_logp_difference/max": 0.26271848678588866, "sampling/sampling_logp_difference/mean": 0.0209973132237792, "step": 6090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.9, "completions/max_terminated_length": 338.9, "completions/mean_length": 231.8125, "completions/mean_terminated_length": 231.8125, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.0719462908804416, "epoch": 0.5608164015813184, "frac_reward_zero_std": 0.4, "grad_norm": 6.44711971282959, "kl": 0.013748869393020868, "learning_rate": 4.748164820844257e-06, "loss": 0.0008, "num_tokens": 26521314.0, "reward": -0.15, "reward_std": 0.5588710367679596, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5588710486888886, "sampling/importance_sampling_ratio/max": 1.3650095224380494, "sampling/importance_sampling_ratio/mean": 1.0002765715122224, "sampling/importance_sampling_ratio/min": 0.7377782940864563, "sampling/sampling_logp_difference/max": 0.36016035079956055, "sampling/sampling_logp_difference/mean": 0.020824669674038888, "step": 6100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.6, "completions/max_terminated_length": 266.6, "completions/mean_length": 198.4, "completions/mean_terminated_length": 198.4, "completions/min_length": 135.4, "completions/min_terminated_length": 135.4, "entropy": 1.092569798603654, "epoch": 0.5617357727314517, "frac_reward_zero_std": 0.6, "grad_norm": 16.743928909301758, "kl": 0.03460606187873054, "learning_rate": 4.747055453409755e-06, "loss": 0.0018, "num_tokens": 26567810.0, "reward": -0.225, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.297599196434021, "sampling/importance_sampling_ratio/mean": 0.9997401416301728, "sampling/importance_sampling_ratio/min": 0.7412247687578202, "sampling/sampling_logp_difference/max": 0.36523712873458863, "sampling/sampling_logp_difference/mean": 0.02096012532711029, "step": 6110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.4, "completions/max_terminated_length": 321.4, "completions/mean_length": 223.2125, "completions/mean_terminated_length": 223.2125, "completions/min_length": 151.8, "completions/min_terminated_length": 151.8, "entropy": 1.087101937830448, "epoch": 0.562655143881585, "frac_reward_zero_std": 0.5, "grad_norm": 5.13347864151001, "kl": 0.013604020827915519, "learning_rate": 4.745943778069177e-06, "loss": 0.0007, "num_tokens": 26616171.0, "reward": -0.225, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2631612777709962, "sampling/importance_sampling_ratio/mean": 0.9999999403953552, "sampling/importance_sampling_ratio/min": 0.7774055778980256, "sampling/sampling_logp_difference/max": 0.25639774799346926, "sampling/sampling_logp_difference/mean": 0.020335411280393602, "step": 6120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 221.0375, "completions/mean_terminated_length": 221.0375, "completions/min_length": 149.3, "completions/min_terminated_length": 149.3, "entropy": 1.1116154670715332, "epoch": 0.5635745150317183, "frac_reward_zero_std": 0.5, "grad_norm": 5.518352031707764, "kl": 0.012224446336040274, "learning_rate": 4.744829795964306e-06, "loss": 0.0009, "num_tokens": 26662446.0, "reward": 0.325, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2892455816268922, "sampling/importance_sampling_ratio/mean": 0.9999022781848907, "sampling/importance_sampling_ratio/min": 0.7929666399955749, "sampling/sampling_logp_difference/max": 0.2572880506515503, "sampling/sampling_logp_difference/mean": 0.02141437344253063, "step": 6130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.9, "completions/max_terminated_length": 254.9, "completions/mean_length": 184.3125, "completions/mean_terminated_length": 184.3125, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.0860144257545472, "epoch": 0.5644938861818516, "frac_reward_zero_std": 0.7, "grad_norm": 5.984280109405518, "kl": 0.015151782153407112, "learning_rate": 4.74371350823929e-06, "loss": 0.0007, "num_tokens": 26704039.0, "reward": -0.23125, "reward_std": 0.2780517518520355, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.2780517518520355, "sampling/importance_sampling_ratio/max": 1.2657331109046936, "sampling/importance_sampling_ratio/mean": 1.0001564383506776, "sampling/importance_sampling_ratio/min": 0.7686228632926941, "sampling/sampling_logp_difference/max": 0.30079888105392455, "sampling/sampling_logp_difference/mean": 0.021511434018611907, "step": 6140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 219.55, "completions/mean_terminated_length": 219.55, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1129062809050083, "epoch": 0.5654132573319849, "frac_reward_zero_std": 0.5, "grad_norm": 5.62891149520874, "kl": 0.013266347238095477, "learning_rate": 4.742594916040644e-06, "loss": 0.0008, "num_tokens": 26748451.0, "reward": 0.16875, "reward_std": 0.5005868554115296, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.5005868673324585, "sampling/importance_sampling_ratio/max": 1.2672033309936523, "sampling/importance_sampling_ratio/mean": 0.9998232185840606, "sampling/importance_sampling_ratio/min": 0.7399705946445465, "sampling/sampling_logp_difference/max": 0.3327832341194153, "sampling/sampling_logp_difference/mean": 0.02168082110583782, "step": 6150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.4, "completions/max_terminated_length": 269.4, "completions/mean_length": 196.1375, "completions/mean_terminated_length": 196.1375, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0365056186914443, "epoch": 0.5663326284821182, "frac_reward_zero_std": 1.0, "grad_norm": 0.12099802494049072, "kl": 0.011643069260753692, "learning_rate": 4.741474020517254e-06, "loss": 0.0006, "num_tokens": 26791742.0, "reward": -0.2, "reward_std": 0.0, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.0, "sampling/importance_sampling_ratio/max": 1.2462936520576477, "sampling/importance_sampling_ratio/mean": 0.9991954565048218, "sampling/importance_sampling_ratio/min": 0.7881840288639068, "sampling/sampling_logp_difference/max": 0.2628297686576843, "sampling/sampling_logp_difference/mean": 0.02041773349046707, "step": 6160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.3, "completions/max_terminated_length": 333.3, "completions/mean_length": 225.0, "completions/mean_terminated_length": 225.0, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0873339101672173, "epoch": 0.5672519996322516, "frac_reward_zero_std": 0.3, "grad_norm": 6.22245454788208, "kl": 0.014593799423892051, "learning_rate": 4.7403508228203684e-06, "loss": 0.0007, "num_tokens": 26837814.0, "reward": -0.15, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2504542946815491, "sampling/importance_sampling_ratio/mean": 0.9996072053909302, "sampling/importance_sampling_ratio/min": 0.7558193385601044, "sampling/sampling_logp_difference/max": 0.30862083435058596, "sampling/sampling_logp_difference/mean": 0.02167534176260233, "step": 6170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 219.8375, "completions/mean_terminated_length": 219.8375, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 1.0787760958075523, "epoch": 0.5681713707823849, "frac_reward_zero_std": 0.4, "grad_norm": 5.813712120056152, "kl": 0.01339965156512335, "learning_rate": 4.739225324103601e-06, "loss": 0.0006, "num_tokens": 26883913.0, "reward": -0.425, "reward_std": 0.6025980114936829, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.6025980472564697, "sampling/importance_sampling_ratio/max": 1.2495638966560363, "sampling/importance_sampling_ratio/mean": 1.0000077545642854, "sampling/importance_sampling_ratio/min": 0.7432852864265442, "sampling/sampling_logp_difference/max": 0.31896655559539794, "sampling/sampling_logp_difference/mean": 0.021264359168708325, "step": 6180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 354.0, "completions/max_terminated_length": 344.6, "completions/mean_length": 249.5375, "completions/mean_terminated_length": 247.16428833007814, "completions/min_length": 157.3, "completions/min_terminated_length": 157.3, "entropy": 1.0797791186720134, "epoch": 0.5690907419325182, "frac_reward_zero_std": 0.4, "grad_norm": 4.969552993774414, "kl": 0.012200876319548115, "learning_rate": 4.738097525522931e-06, "loss": 0.0007, "num_tokens": 26931812.0, "reward": -0.53125, "reward_std": 0.4477728769183159, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.4477728888392448, "sampling/importance_sampling_ratio/max": 1.285599958896637, "sampling/importance_sampling_ratio/mean": 0.9998096227645874, "sampling/importance_sampling_ratio/min": 0.787180095911026, "sampling/sampling_logp_difference/max": 0.2690299987792969, "sampling/sampling_logp_difference/mean": 0.020653602294623853, "step": 6190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 191.825, "completions/mean_terminated_length": 191.825, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.0649948954582213, "epoch": 0.5700101130826515, "frac_reward_zero_std": 0.2, "grad_norm": 0.1113407164812088, "kl": 0.013267139077652246, "learning_rate": 4.736967428236697e-06, "loss": 0.0005, "num_tokens": 26974286.0, "reward": -0.2, "reward_std": 0.7112359046936035, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.7112359046936035, "sampling/importance_sampling_ratio/max": 1.2338434338569642, "sampling/importance_sampling_ratio/mean": 1.0001287460327148, "sampling/importance_sampling_ratio/min": 0.7861011981964111, "sampling/sampling_logp_difference/max": 0.2540988206863403, "sampling/sampling_logp_difference/mean": 0.021224045567214488, "step": 6200 }, { "epoch": 0.5700101130826515, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 301.95986038394415, "eval_completions/max_terminated_length": 299.8586387434555, "eval_completions/mean_length": 213.0250872600349, "eval_completions/mean_terminated_length": 212.5709716637097, "eval_completions/min_length": 139.62478184991275, "eval_completions/min_terminated_length": 139.62478184991275, "eval_entropy": 1.0759982226406717, "eval_frac_reward_zero_std": 0.4572425828970332, "eval_kl": 0.012807807657562105, "eval_loss": 0.0006234689499251544, "eval_num_tokens": 26974286.0, "eval_reward": -0.10263961605584643, "eval_reward_std": 0.4754487438091641, "eval_rewards/reward_fn/mean": -0.10263961605584643, "eval_rewards/reward_fn/std": 0.4754487597245404, "eval_runtime": 1755.2076, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.2768185987522465, "eval_sampling/importance_sampling_ratio/mean": 0.9999196845615513, "eval_sampling/importance_sampling_ratio/min": 0.7795928744745504, "eval_sampling/sampling_logp_difference/max": 0.2847368848261409, "eval_sampling/sampling_logp_difference/mean": 0.02060733341740324, "eval_steps_per_second": 0.041, "step": 6200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.6, "completions/max_terminated_length": 281.6, "completions/mean_length": 207.4, "completions/mean_terminated_length": 207.4, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "entropy": 0.9752660192549228, "epoch": 0.5709294842327848, "frac_reward_zero_std": 0.8, "grad_norm": 0.1490468680858612, "kl": 0.012376546818995849, "learning_rate": 4.735835033405598e-06, "loss": 0.0006, "num_tokens": 27016302.0, "reward": 0.05, "reward_std": 0.19948650598526002, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.19948650598526002, "sampling/importance_sampling_ratio/max": 1.2647347331047059, "sampling/importance_sampling_ratio/mean": 0.9999642133712768, "sampling/importance_sampling_ratio/min": 0.7495433270931244, "sampling/sampling_logp_difference/max": 0.2980692028999329, "sampling/sampling_logp_difference/mean": 0.020495995599776506, "step": 6210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 209.4875, "completions/mean_terminated_length": 209.4875, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.0670496337115765, "epoch": 0.5718488553829181, "frac_reward_zero_std": 0.4, "grad_norm": 0.2594447135925293, "kl": 0.014218291529687122, "learning_rate": 4.734700342192697e-06, "loss": 0.0007, "num_tokens": 27059821.0, "reward": -0.35, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.257755947113037, "sampling/importance_sampling_ratio/mean": 0.999796849489212, "sampling/importance_sampling_ratio/min": 0.7780355989933014, "sampling/sampling_logp_difference/max": 0.2589974343776703, "sampling/sampling_logp_difference/mean": 0.021400661021471024, "step": 6220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.8, "completions/max_terminated_length": 280.8, "completions/mean_length": 205.4875, "completions/mean_terminated_length": 205.4875, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0190522782504559, "epoch": 0.5727682265330514, "frac_reward_zero_std": 0.4, "grad_norm": 4.864781379699707, "kl": 0.013676727458368986, "learning_rate": 4.7335633557634105e-06, "loss": 0.0008, "num_tokens": 27102332.0, "reward": -0.025, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.266554307937622, "sampling/importance_sampling_ratio/mean": 0.9997530519962311, "sampling/importance_sampling_ratio/min": 0.8021456301212311, "sampling/sampling_logp_difference/max": 0.24996281862258912, "sampling/sampling_logp_difference/mean": 0.020350201055407524, "step": 6230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.3, "completions/max_terminated_length": 321.7, "completions/mean_length": 233.9, "completions/mean_terminated_length": 231.59821472167968, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0335706017911435, "epoch": 0.5736875976831847, "frac_reward_zero_std": 0.4, "grad_norm": 0.14328792691230774, "kl": 0.013568574114469812, "learning_rate": 4.732424075285517e-06, "loss": 0.0008, "num_tokens": 27150692.0, "reward": -0.43125, "reward_std": 0.4292962059378624, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.4292962059378624, "sampling/importance_sampling_ratio/max": 1.251185941696167, "sampling/importance_sampling_ratio/mean": 1.0001368284225465, "sampling/importance_sampling_ratio/min": 0.7692109525203705, "sampling/sampling_logp_difference/max": 0.28567986488342284, "sampling/sampling_logp_difference/mean": 0.020070490799844264, "step": 6240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 206.325, "completions/mean_terminated_length": 206.325, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.099171593785286, "epoch": 0.574606968833318, "frac_reward_zero_std": 0.7, "grad_norm": 0.12875135242938995, "kl": 0.013321186078246682, "learning_rate": 4.731282501929145e-06, "loss": 0.0004, "num_tokens": 27195238.0, "reward": -0.025, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2307451486587524, "sampling/importance_sampling_ratio/mean": 0.9998651266098022, "sampling/importance_sampling_ratio/min": 0.8002692222595215, "sampling/sampling_logp_difference/max": 0.22857242822647095, "sampling/sampling_logp_difference/mean": 0.021006323210895062, "step": 6250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.4, "completions/max_terminated_length": 322.4, "completions/mean_length": 209.025, "completions/mean_terminated_length": 209.025, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.067379754036665, "epoch": 0.5755263399834514, "frac_reward_zero_std": 0.5, "grad_norm": 0.10576607286930084, "kl": 0.013519534998340531, "learning_rate": 4.730138636866786e-06, "loss": 0.001, "num_tokens": 27240376.0, "reward": -0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2511059999465943, "sampling/importance_sampling_ratio/mean": 0.9998291313648224, "sampling/importance_sampling_ratio/min": 0.7633257687091828, "sampling/sampling_logp_difference/max": 0.2899088740348816, "sampling/sampling_logp_difference/mean": 0.020453711040318012, "step": 6260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.5, "completions/max_terminated_length": 266.5, "completions/mean_length": 187.275, "completions/mean_terminated_length": 187.275, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.0315898753702641, "epoch": 0.5764457111335847, "frac_reward_zero_std": 0.2, "grad_norm": 6.487916946411133, "kl": 0.017939853749703617, "learning_rate": 4.728992481273279e-06, "loss": 0.001, "num_tokens": 27280990.0, "reward": 0.325, "reward_std": 0.7406403958797455, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.7406404197216034, "sampling/importance_sampling_ratio/max": 1.2645603179931642, "sampling/importance_sampling_ratio/mean": 0.9993134796619415, "sampling/importance_sampling_ratio/min": 0.7434527426958084, "sampling/sampling_logp_difference/max": 0.3324572205543518, "sampling/sampling_logp_difference/mean": 0.020902800559997558, "step": 6270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.4, "completions/max_terminated_length": 263.4, "completions/mean_length": 189.4375, "completions/mean_terminated_length": 189.4375, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0485113240778445, "epoch": 0.577365082283718, "frac_reward_zero_std": 0.6, "grad_norm": 0.11597459763288498, "kl": 0.013616604160051792, "learning_rate": 4.727844036325819e-06, "loss": 0.0007, "num_tokens": 27323929.0, "reward": -0.25, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.3095606923103333, "sampling/importance_sampling_ratio/mean": 0.9994057655334473, "sampling/importance_sampling_ratio/min": 0.7686529636383057, "sampling/sampling_logp_difference/max": 0.30002942085266116, "sampling/sampling_logp_difference/mean": 0.021048137545585634, "step": 6280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.0, "completions/max_terminated_length": 257.0, "completions/mean_length": 190.275, "completions/mean_terminated_length": 190.275, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 0.999631866812706, "epoch": 0.5782844534338513, "frac_reward_zero_std": 0.4, "grad_norm": 5.9664998054504395, "kl": 0.011961379554122686, "learning_rate": 4.726693303203949e-06, "loss": 0.0006, "num_tokens": 27367495.0, "reward": 0.21875, "reward_std": 0.5494261980056763, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.5494262099266052, "sampling/importance_sampling_ratio/max": 1.2128410339355469, "sampling/importance_sampling_ratio/mean": 1.0001188516616821, "sampling/importance_sampling_ratio/min": 0.8135342180728913, "sampling/sampling_logp_difference/max": 0.22069182395935058, "sampling/sampling_logp_difference/mean": 0.019666276499629022, "step": 6290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 203.1625, "completions/mean_terminated_length": 203.1625, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0476365834474564, "epoch": 0.5792038245839846, "frac_reward_zero_std": 0.3, "grad_norm": 6.394033908843994, "kl": 0.011933853017399087, "learning_rate": 4.725540283089566e-06, "loss": 0.0008, "num_tokens": 27410900.0, "reward": -0.475, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.3273104071617126, "sampling/importance_sampling_ratio/mean": 1.0001704454421998, "sampling/importance_sampling_ratio/min": 0.7836531937122345, "sampling/sampling_logp_difference/max": 0.3203702181577682, "sampling/sampling_logp_difference/mean": 0.020655503310263158, "step": 6300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.7, "completions/max_terminated_length": 271.7, "completions/mean_length": 195.725, "completions/mean_terminated_length": 195.725, "completions/min_length": 124.7, "completions/min_terminated_length": 124.7, "entropy": 1.0768679663538934, "epoch": 0.5801231957341179, "frac_reward_zero_std": 0.3, "grad_norm": 5.922355651855469, "kl": 0.014028220309410245, "learning_rate": 4.724384977166913e-06, "loss": 0.0005, "num_tokens": 27452262.0, "reward": -0.5, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2505314350128174, "sampling/importance_sampling_ratio/mean": 1.000163918733597, "sampling/importance_sampling_ratio/min": 0.7561722517013549, "sampling/sampling_logp_difference/max": 0.2942532539367676, "sampling/sampling_logp_difference/mean": 0.021712548285722732, "step": 6310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 318.8, "completions/max_terminated_length": 302.2, "completions/mean_length": 224.85, "completions/mean_terminated_length": 221.79107360839845, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.0209189388900994, "epoch": 0.5810425668842512, "frac_reward_zero_std": 0.5, "grad_norm": 6.378838539123535, "kl": 0.012919382023392245, "learning_rate": 4.723227386622583e-06, "loss": 0.0005, "num_tokens": 27495490.0, "reward": -0.63125, "reward_std": 0.43548367023468015, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.43548368811607363, "sampling/importance_sampling_ratio/max": 1.2522066593170167, "sampling/importance_sampling_ratio/mean": 0.9997277975082397, "sampling/importance_sampling_ratio/min": 0.7446709871292114, "sampling/sampling_logp_difference/max": 0.3016711473464966, "sampling/sampling_logp_difference/mean": 0.021341763250529767, "step": 6320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 339.6, "completions/max_terminated_length": 339.5, "completions/mean_length": 241.35, "completions/mean_terminated_length": 238.69500122070312, "completions/min_length": 156.2, "completions/min_terminated_length": 156.2, "entropy": 1.0614330761134625, "epoch": 0.5819619380343845, "frac_reward_zero_std": 0.3, "grad_norm": 5.535905838012695, "kl": 0.014321396482409909, "learning_rate": 4.722067512645512e-06, "loss": 0.0011, "num_tokens": 27545686.0, "reward": -0.09375, "reward_std": 0.5519336462020874, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.5519336730241775, "sampling/importance_sampling_ratio/max": 1.2937144994735719, "sampling/importance_sampling_ratio/mean": 0.9994823932647705, "sampling/importance_sampling_ratio/min": 0.739529812335968, "sampling/sampling_logp_difference/max": 0.34213059544563296, "sampling/sampling_logp_difference/mean": 0.02106685917824507, "step": 6330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.2, "completions/max_terminated_length": 246.2, "completions/mean_length": 173.5625, "completions/mean_terminated_length": 173.5625, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 0.9129418171942234, "epoch": 0.5828813091845177, "frac_reward_zero_std": 0.7, "grad_norm": 0.1432679444551468, "kl": 0.012959863618016243, "learning_rate": 4.7209053564269854e-06, "loss": 0.0007, "num_tokens": 27583195.0, "reward": 0.075, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2310020446777343, "sampling/importance_sampling_ratio/mean": 1.0000525951385497, "sampling/importance_sampling_ratio/min": 0.7666615337133408, "sampling/sampling_logp_difference/max": 0.2809038758277893, "sampling/sampling_logp_difference/mean": 0.020106946490705012, "step": 6340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 198.0125, "completions/mean_terminated_length": 198.0125, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.0320655167102815, "epoch": 0.583800680334651, "frac_reward_zero_std": 0.6, "grad_norm": 0.1340881884098053, "kl": 0.01368690537638031, "learning_rate": 4.719740919160631e-06, "loss": 0.0009, "num_tokens": 27625580.0, "reward": 0.35, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2503974437713623, "sampling/importance_sampling_ratio/mean": 0.9998067438602447, "sampling/importance_sampling_ratio/min": 0.7532598793506622, "sampling/sampling_logp_difference/max": 0.3051076471805573, "sampling/sampling_logp_difference/mean": 0.02063188385218382, "step": 6350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.0, "completions/max_terminated_length": 277.0, "completions/mean_length": 203.7, "completions/mean_terminated_length": 203.7, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.0450155422091485, "epoch": 0.5847200514847845, "frac_reward_zero_std": 0.5, "grad_norm": 0.12277191877365112, "kl": 0.012631364539265633, "learning_rate": 4.718574202042419e-06, "loss": 0.0003, "num_tokens": 27667964.0, "reward": 0.05, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2553475856781007, "sampling/importance_sampling_ratio/mean": 1.0001072704792022, "sampling/importance_sampling_ratio/min": 0.7322216361761094, "sampling/sampling_logp_difference/max": 0.34834927320480347, "sampling/sampling_logp_difference/mean": 0.020814315788447856, "step": 6360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.5, "completions/max_terminated_length": 316.5, "completions/mean_length": 218.3125, "completions/mean_terminated_length": 218.3125, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.048986452817917, "epoch": 0.5856394226349178, "frac_reward_zero_std": 0.3, "grad_norm": 5.408479690551758, "kl": 0.013034357270225883, "learning_rate": 4.71740520627066e-06, "loss": 0.0003, "num_tokens": 27711557.0, "reward": 0.1, "reward_std": 0.6657755315303803, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6657755434513092, "sampling/importance_sampling_ratio/max": 1.259723138809204, "sampling/importance_sampling_ratio/mean": 0.9999006807804107, "sampling/importance_sampling_ratio/min": 0.7668256640434266, "sampling/sampling_logp_difference/max": 0.27193193435668944, "sampling/sampling_logp_difference/mean": 0.020833071321249008, "step": 6370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 335.2, "completions/max_terminated_length": 327.8, "completions/mean_length": 226.8375, "completions/mean_terminated_length": 224.69464416503905, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 1.1383411563932895, "epoch": 0.586558793785051, "frac_reward_zero_std": 0.1, "grad_norm": 5.942285060882568, "kl": 0.01633014978724532, "learning_rate": 4.716233933046008e-06, "loss": 0.0005, "num_tokens": 27755784.0, "reward": -0.20625, "reward_std": 0.8310620188713074, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.8310620486736298, "sampling/importance_sampling_ratio/max": 1.3355380773544312, "sampling/importance_sampling_ratio/mean": 1.0004649758338928, "sampling/importance_sampling_ratio/min": 0.7542935192584992, "sampling/sampling_logp_difference/max": 0.3407460033893585, "sampling/sampling_logp_difference/mean": 0.022789489664137363, "step": 6380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.6, "completions/max_terminated_length": 303.6, "completions/mean_length": 220.7625, "completions/mean_terminated_length": 220.7625, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 1.0821843557059765, "epoch": 0.5874781649351843, "frac_reward_zero_std": 0.7, "grad_norm": 4.954213619232178, "kl": 0.013172792067052796, "learning_rate": 4.715060383571454e-06, "loss": 0.0006, "num_tokens": 27803013.0, "reward": -0.375, "reward_std": 0.2886738538742065, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.2886738657951355, "sampling/importance_sampling_ratio/max": 1.2685502767562866, "sampling/importance_sampling_ratio/mean": 1.0005382120609283, "sampling/importance_sampling_ratio/min": 0.7727256774902344, "sampling/sampling_logp_difference/max": 0.2805886805057526, "sampling/sampling_logp_difference/mean": 0.021008232422173024, "step": 6390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.8, "completions/max_terminated_length": 307.8, "completions/mean_length": 217.2, "completions/mean_terminated_length": 217.2, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.1172985821962356, "epoch": 0.5883975360853176, "frac_reward_zero_std": 0.6, "grad_norm": 6.367059230804443, "kl": 0.013408011390129104, "learning_rate": 4.713884559052327e-06, "loss": 0.0009, "num_tokens": 27848581.0, "reward": -0.45, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.242226207256317, "sampling/importance_sampling_ratio/mean": 0.9999369084835052, "sampling/importance_sampling_ratio/min": 0.760951915383339, "sampling/sampling_logp_difference/max": 0.2946118116378784, "sampling/sampling_logp_difference/mean": 0.02155954670161009, "step": 6400 }, { "epoch": 0.5883975360853176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 304.61431064572423, "eval_completions/max_terminated_length": 301.4956369982548, "eval_completions/mean_length": 213.25196335078533, "eval_completions/mean_terminated_length": 212.50767911208655, "eval_completions/min_length": 139.97905759162305, "eval_completions/min_terminated_length": 139.97905759162305, "eval_entropy": 1.1086045119774903, "eval_frac_reward_zero_std": 0.4537521815008726, "eval_kl": 0.01325919231754219, "eval_loss": 0.0007249863119795918, "eval_num_tokens": 27848581.0, "eval_reward": -0.10427574171029669, "eval_reward_std": 0.4751225365602117, "eval_rewards/reward_fn/mean": -0.10427574171029669, "eval_rewards/reward_fn/std": 0.47512255309972046, "eval_runtime": 1768.4327, "eval_samples_per_second": 0.324, "eval_sampling/importance_sampling_ratio/max": 1.2693612571370123, "eval_sampling/importance_sampling_ratio/mean": 0.9999691462225523, "eval_sampling/importance_sampling_ratio/min": 0.7802590814144408, "eval_sampling/sampling_logp_difference/max": 0.2814073105445083, "eval_sampling/sampling_logp_difference/mean": 0.020932650656795313, "eval_steps_per_second": 0.041, "step": 6400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.1, "completions/max_terminated_length": 261.1, "completions/mean_length": 188.0625, "completions/mean_terminated_length": 188.0625, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.055342423915863, "epoch": 0.5893169072354509, "frac_reward_zero_std": 0.6, "grad_norm": 6.894700050354004, "kl": 0.012635192216839642, "learning_rate": 4.712706460696292e-06, "loss": 0.0007, "num_tokens": 27888994.0, "reward": -0.05, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2673742651939393, "sampling/importance_sampling_ratio/mean": 0.999931812286377, "sampling/importance_sampling_ratio/min": 0.8028109967708588, "sampling/sampling_logp_difference/max": 0.2512237071990967, "sampling/sampling_logp_difference/mean": 0.021498346142470835, "step": 6410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 376.3, "completions/max_terminated_length": 361.9, "completions/mean_length": 256.65, "completions/mean_terminated_length": 251.62143249511718, "completions/min_length": 164.1, "completions/min_terminated_length": 164.1, "entropy": 1.0800878889858723, "epoch": 0.5902362783855842, "frac_reward_zero_std": 0.1, "grad_norm": 0.10889072716236115, "kl": 0.012714111502282322, "learning_rate": 4.711526089713353e-06, "loss": 0.0006, "num_tokens": 27937310.0, "reward": 0.0875, "reward_std": 0.8812166571617126, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.8812166810035705, "sampling/importance_sampling_ratio/max": 1.2635182976722716, "sampling/importance_sampling_ratio/mean": 0.9995003044605255, "sampling/importance_sampling_ratio/min": 0.7741999745368957, "sampling/sampling_logp_difference/max": 0.2732981085777283, "sampling/sampling_logp_difference/mean": 0.021076329983770847, "step": 6420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 216.6125, "completions/mean_terminated_length": 216.6125, "completions/min_length": 152.6, "completions/min_terminated_length": 152.6, "entropy": 0.9873565539717675, "epoch": 0.5911556495357175, "frac_reward_zero_std": 0.5, "grad_norm": 4.299776077270508, "kl": 0.012576891458593309, "learning_rate": 4.710343447315845e-06, "loss": 0.001, "num_tokens": 27980391.0, "reward": 0.025, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2632948517799378, "sampling/importance_sampling_ratio/mean": 1.000193452835083, "sampling/importance_sampling_ratio/min": 0.780275309085846, "sampling/sampling_logp_difference/max": 0.27843213379383086, "sampling/sampling_logp_difference/mean": 0.01945577785372734, "step": 6430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.8, "completions/max_terminated_length": 293.8, "completions/mean_length": 210.9375, "completions/mean_terminated_length": 210.9375, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.085482657700777, "epoch": 0.5920750206858508, "frac_reward_zero_std": 0.3, "grad_norm": 0.1303771585226059, "kl": 0.01225585961365141, "learning_rate": 4.709158534718435e-06, "loss": 0.0008, "num_tokens": 28020378.0, "reward": 0.1, "reward_std": 0.5967668652534485, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5967668890953064, "sampling/importance_sampling_ratio/max": 1.2569820880889893, "sampling/importance_sampling_ratio/mean": 0.9999720931053162, "sampling/importance_sampling_ratio/min": 0.7793566286563873, "sampling/sampling_logp_difference/max": 0.25817238092422484, "sampling/sampling_logp_difference/mean": 0.02145161684602499, "step": 6440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 193.5625, "completions/mean_terminated_length": 193.5625, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 1.0884141117334365, "epoch": 0.5929943918359842, "frac_reward_zero_std": 0.3, "grad_norm": 0.1767646223306656, "kl": 0.013352746691089124, "learning_rate": 4.707971353138124e-06, "loss": 0.0005, "num_tokens": 28063103.0, "reward": 0.05, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.2762964010238647, "sampling/importance_sampling_ratio/mean": 1.0006568908691407, "sampling/importance_sampling_ratio/min": 0.7499827921390534, "sampling/sampling_logp_difference/max": 0.3112846791744232, "sampling/sampling_logp_difference/mean": 0.02177255917340517, "step": 6450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 339.9, "completions/max_terminated_length": 336.8, "completions/mean_length": 235.0375, "completions/mean_terminated_length": 232.75892944335936, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.1045090168714524, "epoch": 0.5939137629861175, "frac_reward_zero_std": 0.4, "grad_norm": 0.13380984961986542, "kl": 0.014598804904380813, "learning_rate": 4.706781903794243e-06, "loss": 0.0013, "num_tokens": 28108122.0, "reward": -0.45625, "reward_std": 0.4798125073313713, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.4798125073313713, "sampling/importance_sampling_ratio/max": 1.2490201711654663, "sampling/importance_sampling_ratio/mean": 0.9997563064098358, "sampling/importance_sampling_ratio/min": 0.742404043674469, "sampling/sampling_logp_difference/max": 0.30784035921096803, "sampling/sampling_logp_difference/mean": 0.020972993783652784, "step": 6460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.0, "completions/max_terminated_length": 321.0, "completions/mean_length": 250.6125, "completions/mean_terminated_length": 250.6125, "completions/min_length": 173.7, "completions/min_terminated_length": 173.7, "entropy": 1.2196326814591885, "epoch": 0.5948331341362508, "frac_reward_zero_std": 0.7, "grad_norm": 0.1142936423420906, "kl": 0.015414643799886107, "learning_rate": 4.705590187908452e-06, "loss": 0.0009, "num_tokens": 28157267.0, "reward": -0.675, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.675, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2974353551864624, "sampling/importance_sampling_ratio/mean": 0.9998271465301514, "sampling/importance_sampling_ratio/min": 0.7694901525974274, "sampling/sampling_logp_difference/max": 0.29575313329696656, "sampling/sampling_logp_difference/mean": 0.022882797569036484, "step": 6470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.6, "completions/max_terminated_length": 329.6, "completions/mean_length": 242.45, "completions/mean_terminated_length": 242.45, "completions/min_length": 163.7, "completions/min_terminated_length": 163.7, "entropy": 1.1484448119997979, "epoch": 0.5957525052863841, "frac_reward_zero_std": 0.5, "grad_norm": 0.1336471438407898, "kl": 0.016411362565122545, "learning_rate": 4.7043962067047376e-06, "loss": 0.0009, "num_tokens": 28203695.0, "reward": -0.075, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3274279952049255, "sampling/importance_sampling_ratio/mean": 1.0002317488193513, "sampling/importance_sampling_ratio/min": 0.7503229171037674, "sampling/sampling_logp_difference/max": 0.34339562654495237, "sampling/sampling_logp_difference/mean": 0.021787341870367526, "step": 6480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 355.8, "completions/max_terminated_length": 336.0, "completions/mean_length": 251.275, "completions/mean_terminated_length": 248.13035888671874, "completions/min_length": 159.9, "completions/min_terminated_length": 159.9, "entropy": 1.1052244275808334, "epoch": 0.5966718764365174, "frac_reward_zero_std": 0.5, "grad_norm": 0.09575166553258896, "kl": 0.01261749699478969, "learning_rate": 4.703199961409415e-06, "loss": 0.0007, "num_tokens": 28251709.0, "reward": -0.33125, "reward_std": 0.4764178454875946, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.47641786336898806, "sampling/importance_sampling_ratio/max": 1.2630594372749329, "sampling/importance_sampling_ratio/mean": 0.9998807787895203, "sampling/importance_sampling_ratio/min": 0.753791457414627, "sampling/sampling_logp_difference/max": 0.29871925711631775, "sampling/sampling_logp_difference/mean": 0.02094956859946251, "step": 6490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 208.4875, "completions/mean_terminated_length": 208.4875, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.0899581748992204, "epoch": 0.5975912475866507, "frac_reward_zero_std": 0.4, "grad_norm": 0.12356915324926376, "kl": 0.012678639887599274, "learning_rate": 4.702001453251125e-06, "loss": 0.0005, "num_tokens": 28296964.0, "reward": -0.125, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2963548064231873, "sampling/importance_sampling_ratio/mean": 1.0000158309936524, "sampling/importance_sampling_ratio/min": 0.7879454374313355, "sampling/sampling_logp_difference/max": 0.26982401609420775, "sampling/sampling_logp_difference/mean": 0.02141719972714782, "step": 6500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.5, "completions/max_terminated_length": 287.5, "completions/mean_length": 204.025, "completions/mean_terminated_length": 204.025, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.110663302242756, "epoch": 0.598510618736784, "frac_reward_zero_std": 0.4, "grad_norm": 5.052488803863525, "kl": 0.012636405049124732, "learning_rate": 4.700800683460829e-06, "loss": 0.0005, "num_tokens": 28339486.0, "reward": -0.475, "reward_std": 0.4461353957653046, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.4461353957653046, "sampling/importance_sampling_ratio/max": 1.245255970954895, "sampling/importance_sampling_ratio/mean": 0.9997644782066345, "sampling/importance_sampling_ratio/min": 0.7739045560359955, "sampling/sampling_logp_difference/max": 0.2753369092941284, "sampling/sampling_logp_difference/mean": 0.021922191977500914, "step": 6510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.9, "completions/max_terminated_length": 325.9, "completions/mean_length": 233.2, "completions/mean_terminated_length": 233.2, "completions/min_length": 155.6, "completions/min_terminated_length": 155.6, "entropy": 1.1631937190890311, "epoch": 0.5994299898869173, "frac_reward_zero_std": 0.3, "grad_norm": 0.10646405816078186, "kl": 0.013381678139558062, "learning_rate": 4.699597653271816e-06, "loss": 0.0002, "num_tokens": 28387166.0, "reward": -0.1, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2616149663925171, "sampling/importance_sampling_ratio/mean": 1.000089854001999, "sampling/importance_sampling_ratio/min": 0.8024286568164826, "sampling/sampling_logp_difference/max": 0.2477761149406433, "sampling/sampling_logp_difference/mean": 0.02107716482132673, "step": 6520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 337.2, "completions/max_terminated_length": 315.0, "completions/mean_length": 235.775, "completions/mean_terminated_length": 232.60714416503907, "completions/min_length": 152.9, "completions/min_terminated_length": 152.9, "entropy": 1.1090887114405632, "epoch": 0.6003493610370506, "frac_reward_zero_std": 0.4, "grad_norm": 0.12257765978574753, "kl": 0.013817408360773697, "learning_rate": 4.698392363919694e-06, "loss": 0.0008, "num_tokens": 28433404.0, "reward": -0.25625, "reward_std": 0.5432875275611877, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.5432875394821167, "sampling/importance_sampling_ratio/max": 1.3057843327522278, "sampling/importance_sampling_ratio/mean": 0.9995329201221466, "sampling/importance_sampling_ratio/min": 0.7753621757030487, "sampling/sampling_logp_difference/max": 0.2942726373672485, "sampling/sampling_logp_difference/mean": 0.021153258346021175, "step": 6530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.6, "completions/max_terminated_length": 296.6, "completions/mean_length": 214.8875, "completions/mean_terminated_length": 214.8875, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.0568211883306504, "epoch": 0.601268732187184, "frac_reward_zero_std": 0.6, "grad_norm": 6.03017520904541, "kl": 0.012287242640741169, "learning_rate": 4.697184816642392e-06, "loss": 0.0007, "num_tokens": 28476395.0, "reward": -0.45625, "reward_std": 0.2844801887869835, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.28448020070791247, "sampling/importance_sampling_ratio/max": 1.2221644639968872, "sampling/importance_sampling_ratio/mean": 0.9999144017696381, "sampling/importance_sampling_ratio/min": 0.7668275117874146, "sampling/sampling_logp_difference/max": 0.2780591011047363, "sampling/sampling_logp_difference/mean": 0.020912213064730168, "step": 6540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 234.1875, "completions/mean_terminated_length": 234.1875, "completions/min_length": 169.9, "completions/min_terminated_length": 169.9, "entropy": 1.1288899391889573, "epoch": 0.6021881033373173, "frac_reward_zero_std": 0.1, "grad_norm": 5.938522815704346, "kl": 0.014043272216804325, "learning_rate": 4.6959750126801565e-06, "loss": 0.0006, "num_tokens": 28524242.0, "reward": -0.45, "reward_std": 0.7600752472877502, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.7600752472877502, "sampling/importance_sampling_ratio/max": 1.2568817496299745, "sampling/importance_sampling_ratio/mean": 0.9997751593589783, "sampling/importance_sampling_ratio/min": 0.7785943448543549, "sampling/sampling_logp_difference/max": 0.2659779071807861, "sampling/sampling_logp_difference/mean": 0.021541865728795528, "step": 6550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.4, "completions/max_terminated_length": 307.4, "completions/mean_length": 222.1625, "completions/mean_terminated_length": 222.1625, "completions/min_length": 149.3, "completions/min_terminated_length": 149.3, "entropy": 1.0738735746592283, "epoch": 0.6031074744874506, "frac_reward_zero_std": 0.5, "grad_norm": 0.10668086260557175, "kl": 0.012803468009224162, "learning_rate": 4.694762953275554e-06, "loss": 0.0006, "num_tokens": 28568959.0, "reward": -0.38125, "reward_std": 0.4508997440338135, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.4508997440338135, "sampling/importance_sampling_ratio/max": 1.271391797065735, "sampling/importance_sampling_ratio/mean": 1.000172060728073, "sampling/importance_sampling_ratio/min": 0.7994428038597107, "sampling/sampling_logp_difference/max": 0.2602361083030701, "sampling/sampling_logp_difference/mean": 0.021073590964078903, "step": 6560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.2, "completions/max_terminated_length": 318.2, "completions/mean_length": 221.075, "completions/mean_terminated_length": 221.075, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.1348481342196464, "epoch": 0.6040268456375839, "frac_reward_zero_std": 0.3, "grad_norm": 6.992284297943115, "kl": 0.01313080302788876, "learning_rate": 4.6935486396734666e-06, "loss": -0.0002, "num_tokens": 28615029.0, "reward": -0.3, "reward_std": 0.651453047990799, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.6514530599117279, "sampling/importance_sampling_ratio/max": 1.275177824497223, "sampling/importance_sampling_ratio/mean": 0.999828439950943, "sampling/importance_sampling_ratio/min": 0.7540915012359619, "sampling/sampling_logp_difference/max": 0.3007181644439697, "sampling/sampling_logp_difference/mean": 0.02165831793099642, "step": 6570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.5, "completions/max_terminated_length": 315.5, "completions/mean_length": 230.4625, "completions/mean_terminated_length": 230.4625, "completions/min_length": 150.6, "completions/min_terminated_length": 150.6, "entropy": 1.1991167813539505, "epoch": 0.6049462167877172, "frac_reward_zero_std": 0.2, "grad_norm": 6.772153854370117, "kl": 0.013629298348678275, "learning_rate": 4.692332073121092e-06, "loss": 0.0007, "num_tokens": 28661306.0, "reward": -0.05, "reward_std": 0.714614874124527, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.714614886045456, "sampling/importance_sampling_ratio/max": 1.2988760709762572, "sampling/importance_sampling_ratio/mean": 1.0002309918403625, "sampling/importance_sampling_ratio/min": 0.7796796202659607, "sampling/sampling_logp_difference/max": 0.3010742783546448, "sampling/sampling_logp_difference/mean": 0.022164711728692055, "step": 6580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.9, "completions/max_terminated_length": 283.0, "completions/mean_length": 199.1, "completions/mean_terminated_length": 195.37857208251953, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0375138729810716, "epoch": 0.6058655879378505, "frac_reward_zero_std": 0.8, "grad_norm": 5.536322593688965, "kl": 0.01411155168316327, "learning_rate": 4.691113254867942e-06, "loss": 0.0009, "num_tokens": 28702922.0, "reward": -0.23125, "reward_std": 0.20148030519485474, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.20148032307624816, "sampling/importance_sampling_ratio/max": 1.2578237414360047, "sampling/importance_sampling_ratio/mean": 0.9999438047409057, "sampling/importance_sampling_ratio/min": 0.8040703356266021, "sampling/sampling_logp_difference/max": 0.2608494162559509, "sampling/sampling_logp_difference/mean": 0.020521630346775056, "step": 6590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.1, "completions/max_terminated_length": 268.1, "completions/mean_length": 194.875, "completions/mean_terminated_length": 194.875, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.055034650862217, "epoch": 0.6067849590879838, "frac_reward_zero_std": 0.5, "grad_norm": 6.161749839782715, "kl": 0.015149449987802655, "learning_rate": 4.68989218616584e-06, "loss": 0.0005, "num_tokens": 28742376.0, "reward": -0.4, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.27205730676651, "sampling/importance_sampling_ratio/mean": 0.999992024898529, "sampling/importance_sampling_ratio/min": 0.7790205299854278, "sampling/sampling_logp_difference/max": 0.25789356231689453, "sampling/sampling_logp_difference/mean": 0.02118025757372379, "step": 6600 }, { "epoch": 0.6067849590879838, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 301.47294938917975, "eval_completions/max_terminated_length": 299.92844677137873, "eval_completions/mean_length": 212.9829842931937, "eval_completions/mean_terminated_length": 212.42985968731253, "eval_completions/min_length": 141.0174520069808, "eval_completions/min_terminated_length": 141.0174520069808, "eval_entropy": 1.1069506454114098, "eval_frac_reward_zero_std": 0.462478184991274, "eval_kl": 0.01379624093243979, "eval_loss": 0.0007139444351196289, "eval_num_tokens": 28742376.0, "eval_reward": -0.13219895287958114, "eval_reward_std": 0.4715418941999605, "eval_rewards/reward_fn/mean": -0.13219895287958114, "eval_rewards/reward_fn/std": 0.4715419085550059, "eval_runtime": 1753.3049, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2721446103450516, "eval_sampling/importance_sampling_ratio/mean": 1.0000369465579952, "eval_sampling/importance_sampling_ratio/min": 0.7771621261695709, "eval_sampling/sampling_logp_difference/max": 0.285978839555544, "eval_sampling/sampling_logp_difference/mean": 0.02112613743525421, "eval_steps_per_second": 0.041, "step": 6600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.6, "completions/max_terminated_length": 302.6, "completions/mean_length": 214.5625, "completions/mean_terminated_length": 214.5625, "completions/min_length": 148.1, "completions/min_terminated_length": 148.1, "entropy": 1.1190059266984462, "epoch": 0.6077043302381171, "frac_reward_zero_std": 0.1, "grad_norm": 0.1659095287322998, "kl": 0.013167573197279125, "learning_rate": 4.688668868268921e-06, "loss": 0.0004, "num_tokens": 28788813.0, "reward": -0.15, "reward_std": 0.7634542167186738, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.7634542286396027, "sampling/importance_sampling_ratio/max": 1.2374029397964477, "sampling/importance_sampling_ratio/mean": 0.9997837901115417, "sampling/importance_sampling_ratio/min": 0.7925926089286804, "sampling/sampling_logp_difference/max": 0.24245071411132812, "sampling/sampling_logp_difference/mean": 0.02065279334783554, "step": 6610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.7, "completions/max_terminated_length": 335.7, "completions/mean_length": 229.5625, "completions/mean_terminated_length": 229.5625, "completions/min_length": 157.6, "completions/min_terminated_length": 157.6, "entropy": 1.1018521316349505, "epoch": 0.6086237013882504, "frac_reward_zero_std": 0.2, "grad_norm": 5.882264614105225, "kl": 0.014318764500785619, "learning_rate": 4.687443302433631e-06, "loss": 0.0006, "num_tokens": 28836938.0, "reward": -0.40625, "reward_std": 0.6001220509409905, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.6001220747828484, "sampling/importance_sampling_ratio/max": 1.2464033961296082, "sampling/importance_sampling_ratio/mean": 0.9996146142482758, "sampling/importance_sampling_ratio/min": 0.7600459814071655, "sampling/sampling_logp_difference/max": 0.2963321328163147, "sampling/sampling_logp_difference/mean": 0.021311441063880922, "step": 6620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.0, "completions/max_terminated_length": 307.0, "completions/mean_length": 199.5125, "completions/mean_terminated_length": 199.5125, "completions/min_length": 116.2, "completions/min_terminated_length": 116.2, "entropy": 1.0613984540104866, "epoch": 0.6095430725383837, "frac_reward_zero_std": 0.5, "grad_norm": 6.803581714630127, "kl": 0.013431384979048744, "learning_rate": 4.686215489918724e-06, "loss": 0.0005, "num_tokens": 28879715.0, "reward": -0.175, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2429281949996949, "sampling/importance_sampling_ratio/mean": 0.9998455047607422, "sampling/importance_sampling_ratio/min": 0.8000204920768738, "sampling/sampling_logp_difference/max": 0.2437838315963745, "sampling/sampling_logp_difference/mean": 0.02100240606814623, "step": 6630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.3, "completions/max_terminated_length": 306.3, "completions/mean_length": 212.6125, "completions/mean_terminated_length": 212.6125, "completions/min_length": 148.8, "completions/min_terminated_length": 148.8, "entropy": 1.1408431567251682, "epoch": 0.6104624436885171, "frac_reward_zero_std": 0.5, "grad_norm": 6.993856906890869, "kl": 0.014417864131974056, "learning_rate": 4.684985431985263e-06, "loss": 0.0007, "num_tokens": 28922100.0, "reward": -0.225, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2866899013519286, "sampling/importance_sampling_ratio/mean": 0.999882036447525, "sampling/importance_sampling_ratio/min": 0.7968545854091644, "sampling/sampling_logp_difference/max": 0.2631964564323425, "sampling/sampling_logp_difference/mean": 0.021950706839561462, "step": 6640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.1, "completions/max_terminated_length": 299.8, "completions/mean_length": 226.4375, "completions/mean_terminated_length": 223.89642944335938, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0518347054719925, "epoch": 0.6113818148386504, "frac_reward_zero_std": 0.6, "grad_norm": 0.14879639446735382, "kl": 0.0143184905173257, "learning_rate": 4.6837531298966145e-06, "loss": 0.0007, "num_tokens": 28966831.0, "reward": -0.10625, "reward_std": 0.3442630469799042, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.34426305294036863, "sampling/importance_sampling_ratio/max": 1.339721393585205, "sampling/importance_sampling_ratio/mean": 0.9995287835597992, "sampling/importance_sampling_ratio/min": 0.7533026158809661, "sampling/sampling_logp_difference/max": 0.3465405464172363, "sampling/sampling_logp_difference/mean": 0.020366567373275756, "step": 6650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.4, "completions/max_terminated_length": 317.4, "completions/mean_length": 235.1875, "completions/mean_terminated_length": 235.1875, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 1.1129253931343555, "epoch": 0.6123011859887837, "frac_reward_zero_std": 0.5, "grad_norm": 5.922641277313232, "kl": 0.013537961791735142, "learning_rate": 4.682518584918453e-06, "loss": 0.0003, "num_tokens": 29012830.0, "reward": -0.5, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.308371865749359, "sampling/importance_sampling_ratio/mean": 0.9999157905578613, "sampling/importance_sampling_ratio/min": 0.7579428195953369, "sampling/sampling_logp_difference/max": 0.3044776201248169, "sampling/sampling_logp_difference/mean": 0.021360795386135577, "step": 6660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 325.6, "completions/max_terminated_length": 312.1, "completions/mean_length": 218.375, "completions/mean_terminated_length": 215.0250015258789, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.120529257506132, "epoch": 0.613220557138917, "frac_reward_zero_std": 0.5, "grad_norm": 5.142268657684326, "kl": 0.012410218542208895, "learning_rate": 4.681281798318754e-06, "loss": 0.0006, "num_tokens": 29060148.0, "reward": -0.33125, "reward_std": 0.4599475562572479, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.4599475622177124, "sampling/importance_sampling_ratio/max": 1.243469738960266, "sampling/importance_sampling_ratio/mean": 1.0002253234386445, "sampling/importance_sampling_ratio/min": 0.7879293203353882, "sampling/sampling_logp_difference/max": 0.2414475917816162, "sampling/sampling_logp_difference/mean": 0.0214101517572999, "step": 6670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 216.225, "completions/mean_terminated_length": 216.225, "completions/min_length": 146.2, "completions/min_terminated_length": 146.2, "entropy": 1.112018235772848, "epoch": 0.6141399282890503, "frac_reward_zero_std": 0.5, "grad_norm": 5.993546485900879, "kl": 0.012404230661923065, "learning_rate": 4.680042771367797e-06, "loss": 0.0001, "num_tokens": 29102590.0, "reward": -0.1, "reward_std": 0.520943820476532, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5209438681602478, "sampling/importance_sampling_ratio/max": 1.3059924364089965, "sampling/importance_sampling_ratio/mean": 1.0002347946166992, "sampling/importance_sampling_ratio/min": 0.8077674627304077, "sampling/sampling_logp_difference/max": 0.27605053782463074, "sampling/sampling_logp_difference/mean": 0.02106844987720251, "step": 6680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.9, "completions/max_terminated_length": 296.9, "completions/mean_length": 208.9125, "completions/mean_terminated_length": 208.9125, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.0619107510894537, "epoch": 0.6150592994391836, "frac_reward_zero_std": 0.3, "grad_norm": 6.528067588806152, "kl": 0.012731477053603158, "learning_rate": 4.678801505338163e-06, "loss": 0.0005, "num_tokens": 29148103.0, "reward": 0.125, "reward_std": 0.6186381995677948, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6186382234096527, "sampling/importance_sampling_ratio/max": 1.2894106268882752, "sampling/importance_sampling_ratio/mean": 1.0001568257808686, "sampling/importance_sampling_ratio/min": 0.7919514000415802, "sampling/sampling_logp_difference/max": 0.2748812258243561, "sampling/sampling_logp_difference/mean": 0.021149273961782455, "step": 6690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.0, "completions/max_terminated_length": 328.0, "completions/mean_length": 238.8625, "completions/mean_terminated_length": 238.8625, "completions/min_length": 155.3, "completions/min_terminated_length": 155.3, "entropy": 1.062290820479393, "epoch": 0.6159786705893169, "frac_reward_zero_std": 0.5, "grad_norm": 0.12721966207027435, "kl": 0.013717268226901069, "learning_rate": 4.67755800150473e-06, "loss": 0.0004, "num_tokens": 29195468.0, "reward": -0.3, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3031733632087708, "sampling/importance_sampling_ratio/mean": 1.0002612471580505, "sampling/importance_sampling_ratio/min": 0.7613613963127136, "sampling/sampling_logp_difference/max": 0.31599323749542235, "sampling/sampling_logp_difference/mean": 0.020578425377607346, "step": 6700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.8, "completions/max_terminated_length": 266.8, "completions/mean_length": 189.0, "completions/mean_terminated_length": 189.0, "completions/min_length": 110.5, "completions/min_terminated_length": 110.5, "entropy": 0.9704227991402149, "epoch": 0.6168980417394502, "frac_reward_zero_std": 0.5, "grad_norm": 0.1983964443206787, "kl": 0.012009758915519341, "learning_rate": 4.6763122611446765e-06, "loss": 0.0006, "num_tokens": 29233452.0, "reward": -0.2, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2527263760566711, "sampling/importance_sampling_ratio/mean": 1.000353330373764, "sampling/importance_sampling_ratio/min": 0.766628098487854, "sampling/sampling_logp_difference/max": 0.2763820946216583, "sampling/sampling_logp_difference/mean": 0.020503737032413483, "step": 6710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.0, "completions/max_terminated_length": 284.0, "completions/mean_length": 197.1375, "completions/mean_terminated_length": 197.1375, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.0863211624324323, "epoch": 0.6178174128895835, "frac_reward_zero_std": 0.5, "grad_norm": 0.08740390092134476, "kl": 0.013022797094890848, "learning_rate": 4.675064285537479e-06, "loss": 0.0004, "num_tokens": 29276007.0, "reward": -0.05, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2355541825294494, "sampling/importance_sampling_ratio/mean": 1.000337016582489, "sampling/importance_sampling_ratio/min": 0.8032303512096405, "sampling/sampling_logp_difference/max": 0.22747172117233277, "sampling/sampling_logp_difference/mean": 0.021115600131452084, "step": 6720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 286.4, "completions/max_terminated_length": 257.9, "completions/mean_length": 184.35, "completions/mean_terminated_length": 180.1750015258789, "completions/min_length": 120.1, "completions/min_terminated_length": 120.1, "entropy": 0.9917740263044834, "epoch": 0.6187367840397169, "frac_reward_zero_std": 0.7, "grad_norm": 0.1649557650089264, "kl": 0.012435258785262703, "learning_rate": 4.6738140759649064e-06, "loss": 0.0004, "num_tokens": 29315619.0, "reward": -0.53125, "reward_std": 0.28741282820701597, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.28741284012794494, "sampling/importance_sampling_ratio/max": 1.2582787036895753, "sampling/importance_sampling_ratio/mean": 1.0000815391540527, "sampling/importance_sampling_ratio/min": 0.7744428098201752, "sampling/sampling_logp_difference/max": 0.27908073663711547, "sampling/sampling_logp_difference/mean": 0.019876933470368386, "step": 6730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.2, "completions/max_terminated_length": 318.2, "completions/mean_length": 223.675, "completions/mean_terminated_length": 223.675, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0555565036833285, "epoch": 0.6196561551898502, "frac_reward_zero_std": 0.6, "grad_norm": 4.961339950561523, "kl": 0.01254017889732495, "learning_rate": 4.672561633711026e-06, "loss": 0.0005, "num_tokens": 29362665.0, "reward": 0.0, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.311578917503357, "sampling/importance_sampling_ratio/mean": 1.000170260667801, "sampling/importance_sampling_ratio/min": 0.7461402118206024, "sampling/sampling_logp_difference/max": 0.32207065224647524, "sampling/sampling_logp_difference/mean": 0.020664796233177185, "step": 6740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.8, "completions/max_terminated_length": 294.8, "completions/mean_length": 218.075, "completions/mean_terminated_length": 218.075, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0571951933205128, "epoch": 0.6205755263399835, "frac_reward_zero_std": 0.5, "grad_norm": 5.189789772033691, "kl": 0.012219166773138567, "learning_rate": 4.671306960062195e-06, "loss": 0.0002, "num_tokens": 29409191.0, "reward": 0.225, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2755597114562989, "sampling/importance_sampling_ratio/mean": 1.0002372801303863, "sampling/importance_sampling_ratio/min": 0.777616822719574, "sampling/sampling_logp_difference/max": 0.2632319688796997, "sampling/sampling_logp_difference/mean": 0.020077543146908284, "step": 6750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.4, "completions/max_terminated_length": 311.5, "completions/mean_length": 217.175, "completions/mean_terminated_length": 215.49107360839844, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 0.9206697119399905, "epoch": 0.6214948974901168, "frac_reward_zero_std": 0.4, "grad_norm": 0.1188785508275032, "kl": 0.012828078609891236, "learning_rate": 4.670050056307065e-06, "loss": 0.0004, "num_tokens": 29451181.0, "reward": -0.38125, "reward_std": 0.559176754951477, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.559176766872406, "sampling/importance_sampling_ratio/max": 1.3672340035438537, "sampling/importance_sampling_ratio/mean": 0.9999996423721313, "sampling/importance_sampling_ratio/min": 0.7891774892807006, "sampling/sampling_logp_difference/max": 0.31805256605148313, "sampling/sampling_logp_difference/mean": 0.019223827589303254, "step": 6760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.2, "completions/max_terminated_length": 305.2, "completions/mean_length": 214.7375, "completions/mean_terminated_length": 214.7375, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1559641808271408, "epoch": 0.6224142686402501, "frac_reward_zero_std": 0.2, "grad_norm": 6.416077613830566, "kl": 0.013408507267013193, "learning_rate": 4.668790923736577e-06, "loss": 0.0008, "num_tokens": 29498672.0, "reward": 0.25, "reward_std": 0.6422272384166717, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.6422272503376008, "sampling/importance_sampling_ratio/max": 1.2513112425804138, "sampling/importance_sampling_ratio/mean": 0.9997948944568634, "sampling/importance_sampling_ratio/min": 0.8051104426383973, "sampling/sampling_logp_difference/max": 0.243906831741333, "sampling/sampling_logp_difference/mean": 0.021571399457752705, "step": 6770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.0, "completions/max_terminated_length": 327.0, "completions/mean_length": 228.6375, "completions/mean_terminated_length": 228.6375, "completions/min_length": 166.1, "completions/min_terminated_length": 166.1, "entropy": 1.0521890662610531, "epoch": 0.6233336397903834, "frac_reward_zero_std": 0.5, "grad_norm": 5.617599964141846, "kl": 0.01175788757391274, "learning_rate": 4.667529563643959e-06, "loss": 0.0006, "num_tokens": 29545763.0, "reward": -0.35, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.277726125717163, "sampling/importance_sampling_ratio/mean": 1.0002332091331483, "sampling/importance_sampling_ratio/min": 0.7868040204048157, "sampling/sampling_logp_difference/max": 0.27329759001731874, "sampling/sampling_logp_difference/mean": 0.02036209087818861, "step": 6780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.3, "completions/max_terminated_length": 256.3, "completions/mean_length": 191.25, "completions/mean_terminated_length": 191.25, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.0863084867596626, "epoch": 0.6242530109405167, "frac_reward_zero_std": 0.7, "grad_norm": 0.2681926190853119, "kl": 0.012034327047877014, "learning_rate": 4.666265977324731e-06, "loss": 0.0003, "num_tokens": 29586855.0, "reward": -0.15, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2533173561096191, "sampling/importance_sampling_ratio/mean": 0.9996869623661041, "sampling/importance_sampling_ratio/min": 0.7971704602241516, "sampling/sampling_logp_difference/max": 0.2561259984970093, "sampling/sampling_logp_difference/mean": 0.021279106475412847, "step": 6790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.2, "completions/max_terminated_length": 280.2, "completions/mean_length": 204.2125, "completions/mean_terminated_length": 204.2125, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.1849117659032344, "epoch": 0.62517238209065, "frac_reward_zero_std": 0.3, "grad_norm": 5.244174957275391, "kl": 0.014809558994602411, "learning_rate": 4.665000166076697e-06, "loss": 0.001, "num_tokens": 29629096.0, "reward": -0.475, "reward_std": 0.5933722019195556, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5933722376823425, "sampling/importance_sampling_ratio/max": 1.2449828505516052, "sampling/importance_sampling_ratio/mean": 0.9999510943889618, "sampling/importance_sampling_ratio/min": 0.7814999222755432, "sampling/sampling_logp_difference/max": 0.26101909279823304, "sampling/sampling_logp_difference/mean": 0.02205946370959282, "step": 6800 }, { "epoch": 0.62517238209065, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 296.0785340314136, "eval_completions/max_terminated_length": 294.5776614310646, "eval_completions/mean_length": 205.66121291448516, "eval_completions/mean_terminated_length": 205.24589682374324, "eval_completions/min_length": 133.62478184991275, "eval_completions/min_terminated_length": 133.62478184991275, "eval_entropy": 1.0912293805293805, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.013413789575630197, "eval_loss": 0.0006586270174011588, "eval_num_tokens": 29629096.0, "eval_reward": -0.12227312390924956, "eval_reward_std": 0.45555907261621265, "eval_rewards/reward_fn/mean": -0.12227312390924956, "eval_rewards/reward_fn/std": 0.4555590881155007, "eval_runtime": 1725.2729, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.269957778133439, "eval_sampling/importance_sampling_ratio/mean": 1.0000647946177976, "eval_sampling/importance_sampling_ratio/min": 0.7744465545665011, "eval_sampling/sampling_logp_difference/max": 0.28921654841246613, "eval_sampling/sampling_logp_difference/mean": 0.020896380238989157, "eval_steps_per_second": 0.042, "step": 6800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.7, "completions/max_terminated_length": 318.7, "completions/mean_length": 209.1125, "completions/mean_terminated_length": 209.1125, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.139192096143961, "epoch": 0.6260917532407833, "frac_reward_zero_std": 0.6, "grad_norm": 0.11431828141212463, "kl": 0.013136305968509987, "learning_rate": 4.663732131199946e-06, "loss": 0.0006, "num_tokens": 29673313.0, "reward": -0.35, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2880698084831237, "sampling/importance_sampling_ratio/mean": 0.9998107373714447, "sampling/importance_sampling_ratio/min": 0.7968315601348877, "sampling/sampling_logp_difference/max": 0.26990058422088625, "sampling/sampling_logp_difference/mean": 0.021338170021772386, "step": 6810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.8, "completions/max_terminated_length": 316.8, "completions/mean_length": 214.625, "completions/mean_terminated_length": 214.625, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.1537302166223526, "epoch": 0.6270111243909167, "frac_reward_zero_std": 0.2, "grad_norm": 6.663516521453857, "kl": 0.013568983017466962, "learning_rate": 4.662461873996853e-06, "loss": 0.0007, "num_tokens": 29716235.0, "reward": -0.15, "reward_std": 0.656549721956253, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.656549733877182, "sampling/importance_sampling_ratio/max": 1.2441540002822875, "sampling/importance_sampling_ratio/mean": 1.0001797080039978, "sampling/importance_sampling_ratio/min": 0.7756579935550689, "sampling/sampling_logp_difference/max": 0.2663849115371704, "sampling/sampling_logp_difference/mean": 0.021392032317817212, "step": 6820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.2, "completions/max_terminated_length": 321.2, "completions/mean_length": 199.1875, "completions/mean_terminated_length": 199.1875, "completions/min_length": 116.5, "completions/min_terminated_length": 116.5, "entropy": 1.100270675867796, "epoch": 0.62793049554105, "frac_reward_zero_std": 0.5, "grad_norm": 11.527972221374512, "kl": 0.012747963366564363, "learning_rate": 4.6611893957720715e-06, "loss": 0.001, "num_tokens": 29764170.0, "reward": -0.325, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2343478560447694, "sampling/importance_sampling_ratio/mean": 0.9996278524398804, "sampling/importance_sampling_ratio/min": 0.7842954099178314, "sampling/sampling_logp_difference/max": 0.25992830991745, "sampling/sampling_logp_difference/mean": 0.0209498293697834, "step": 6830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 212.9875, "completions/mean_terminated_length": 212.9875, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.0899310097098351, "epoch": 0.6288498666911833, "frac_reward_zero_std": 0.4, "grad_norm": 7.675441265106201, "kl": 0.013095389376394451, "learning_rate": 4.6599146978325405e-06, "loss": 0.0003, "num_tokens": 29808945.0, "reward": -0.3, "reward_std": 0.5588710367679596, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5588710486888886, "sampling/importance_sampling_ratio/max": 1.270983636379242, "sampling/importance_sampling_ratio/mean": 1.0001568138599395, "sampling/importance_sampling_ratio/min": 0.7650015532970429, "sampling/sampling_logp_difference/max": 0.31362439393997193, "sampling/sampling_logp_difference/mean": 0.020231781341135503, "step": 6840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.5, "completions/max_terminated_length": 313.5, "completions/mean_length": 207.1875, "completions/mean_terminated_length": 207.1875, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.13693553134799, "epoch": 0.6297692378413166, "frac_reward_zero_std": 0.2, "grad_norm": 5.751143455505371, "kl": 0.01495915949344635, "learning_rate": 4.658637781487477e-06, "loss": 0.001, "num_tokens": 29851832.0, "reward": -0.05, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.323489010334015, "sampling/importance_sampling_ratio/mean": 1.0000997602939605, "sampling/importance_sampling_ratio/min": 0.7976732909679413, "sampling/sampling_logp_difference/max": 0.28316869735717776, "sampling/sampling_logp_difference/mean": 0.021467476151883602, "step": 6850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 196.3125, "completions/mean_terminated_length": 196.3125, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.1678206339478492, "epoch": 0.6306886089914499, "frac_reward_zero_std": 0.5, "grad_norm": 6.224295139312744, "kl": 0.01502502709045075, "learning_rate": 4.657358648048375e-06, "loss": 0.0007, "num_tokens": 29893689.0, "reward": -0.3, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3501280784606933, "sampling/importance_sampling_ratio/mean": 1.0007274568080902, "sampling/importance_sampling_ratio/min": 0.780909287929535, "sampling/sampling_logp_difference/max": 0.3240118384361267, "sampling/sampling_logp_difference/mean": 0.022132136300206183, "step": 6860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.8, "completions/max_terminated_length": 263.8, "completions/mean_length": 187.3125, "completions/mean_terminated_length": 187.3125, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.100642056018114, "epoch": 0.6316079801415831, "frac_reward_zero_std": 0.5, "grad_norm": 6.676766395568848, "kl": 0.013358874205732719, "learning_rate": 4.656077298829009e-06, "loss": 0.0004, "num_tokens": 29932466.0, "reward": -0.125, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2770989418029786, "sampling/importance_sampling_ratio/mean": 0.9997160851955413, "sampling/importance_sampling_ratio/min": 0.7971075296401977, "sampling/sampling_logp_difference/max": 0.26397920250892637, "sampling/sampling_logp_difference/mean": 0.021204890310764314, "step": 6870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.2, "completions/max_terminated_length": 269.2, "completions/mean_length": 199.0375, "completions/mean_terminated_length": 199.0375, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.1348198264837266, "epoch": 0.6325273512917164, "frac_reward_zero_std": 0.3, "grad_norm": 0.09815039485692978, "kl": 0.012368429341586306, "learning_rate": 4.654793735145425e-06, "loss": 0.0005, "num_tokens": 29976789.0, "reward": 0.05, "reward_std": 0.6985746860504151, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.698574709892273, "sampling/importance_sampling_ratio/max": 1.2591906189918518, "sampling/importance_sampling_ratio/mean": 0.9999608755111694, "sampling/importance_sampling_ratio/min": 0.7822715163230896, "sampling/sampling_logp_difference/max": 0.2755227625370026, "sampling/sampling_logp_difference/mean": 0.021202344819903372, "step": 6880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.2, "completions/max_terminated_length": 315.4, "completions/mean_length": 220.725, "completions/mean_terminated_length": 218.01607360839844, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.1755747459828854, "epoch": 0.6334467224418497, "frac_reward_zero_std": 0.1, "grad_norm": 6.615853309631348, "kl": 0.012971704581286759, "learning_rate": 4.653507958315948e-06, "loss": 0.0, "num_tokens": 30022063.0, "reward": -0.25625, "reward_std": 0.6708484217524529, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.6708484217524529, "sampling/importance_sampling_ratio/max": 1.3982303380966186, "sampling/importance_sampling_ratio/mean": 1.0002803027629852, "sampling/importance_sampling_ratio/min": 0.768500030040741, "sampling/sampling_logp_difference/max": 0.3549355804920197, "sampling/sampling_logp_difference/mean": 0.022225804813206195, "step": 6890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.8, "completions/max_terminated_length": 311.8, "completions/mean_length": 215.525, "completions/mean_terminated_length": 215.525, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0575472258031369, "epoch": 0.634366093591983, "frac_reward_zero_std": 0.6, "grad_norm": 5.950709819793701, "kl": 0.012647197116166353, "learning_rate": 4.652219969661174e-06, "loss": 0.0006, "num_tokens": 30067049.0, "reward": -0.05, "reward_std": 0.420828652381897, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.42082867622375486, "sampling/importance_sampling_ratio/max": 1.3180296659469604, "sampling/importance_sampling_ratio/mean": 1.0002132534980774, "sampling/importance_sampling_ratio/min": 0.776434862613678, "sampling/sampling_logp_difference/max": 0.3152373909950256, "sampling/sampling_logp_difference/mean": 0.0203783237375319, "step": 6900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.6, "completions/max_terminated_length": 267.6, "completions/mean_length": 207.625, "completions/mean_terminated_length": 207.625, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0853083588182926, "epoch": 0.6352854647421164, "frac_reward_zero_std": 0.4, "grad_norm": 0.15372659265995026, "kl": 0.012476367386989295, "learning_rate": 4.650929770503968e-06, "loss": 0.001, "num_tokens": 30112131.0, "reward": 0.075, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2450814247131348, "sampling/importance_sampling_ratio/mean": 1.0004511058330536, "sampling/importance_sampling_ratio/min": 0.7816547572612762, "sampling/sampling_logp_difference/max": 0.2522275000810623, "sampling/sampling_logp_difference/mean": 0.02111142687499523, "step": 6910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 309.6, "completions/max_terminated_length": 309.6, "completions/mean_length": 241.175, "completions/mean_terminated_length": 231.9, "completions/min_length": 158.6, "completions/min_terminated_length": 158.6, "entropy": 1.0720074601471423, "epoch": 0.6362048358922497, "frac_reward_zero_std": 0.5, "grad_norm": 0.14932659268379211, "kl": 0.015071413718396797, "learning_rate": 4.64963736216947e-06, "loss": 0.0006, "num_tokens": 30158641.0, "reward": -0.60625, "reward_std": 0.4325863480567932, "rewards/reward_fn/mean": -0.60625, "rewards/reward_fn/std": 0.4325863540172577, "sampling/importance_sampling_ratio/max": 1.366134512424469, "sampling/importance_sampling_ratio/mean": 0.9998899936676026, "sampling/importance_sampling_ratio/min": 0.706684696674347, "sampling/sampling_logp_difference/max": 0.4147831678390503, "sampling/sampling_logp_difference/mean": 0.02061451692134142, "step": 6920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.9, "completions/max_terminated_length": 317.9, "completions/mean_length": 210.925, "completions/mean_terminated_length": 210.925, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.049093709141016, "epoch": 0.637124207042383, "frac_reward_zero_std": 0.7, "grad_norm": 7.322968006134033, "kl": 0.01169348051189445, "learning_rate": 4.648342745985087e-06, "loss": 0.0003, "num_tokens": 30205707.0, "reward": 0.45, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2340118408203125, "sampling/importance_sampling_ratio/mean": 1.000343906879425, "sampling/importance_sampling_ratio/min": 0.74562509059906, "sampling/sampling_logp_difference/max": 0.3514559745788574, "sampling/sampling_logp_difference/mean": 0.020456557720899583, "step": 6930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.7, "completions/max_terminated_length": 267.7, "completions/mean_length": 194.775, "completions/mean_terminated_length": 194.775, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.116311491280794, "epoch": 0.6380435781925163, "frac_reward_zero_std": 0.4, "grad_norm": 9.609569549560547, "kl": 0.013173996191471814, "learning_rate": 4.647045923280493e-06, "loss": 0.0003, "num_tokens": 30247833.0, "reward": -0.15, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.3426828384399414, "sampling/importance_sampling_ratio/mean": 0.9997741520404816, "sampling/importance_sampling_ratio/min": 0.7878404200077057, "sampling/sampling_logp_difference/max": 0.32070615589618684, "sampling/sampling_logp_difference/mean": 0.02236398197710514, "step": 6940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.9, "completions/max_terminated_length": 250.9, "completions/mean_length": 174.6875, "completions/mean_terminated_length": 174.6875, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 0.9974558562040329, "epoch": 0.6389629493426496, "frac_reward_zero_std": 0.3, "grad_norm": 0.09004458785057068, "kl": 0.012838218727847562, "learning_rate": 4.645746895387628e-06, "loss": 0.0005, "num_tokens": 30286592.0, "reward": -0.225, "reward_std": 0.5277738928794861, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5277739048004151, "sampling/importance_sampling_ratio/max": 1.244395911693573, "sampling/importance_sampling_ratio/mean": 0.9997526347637177, "sampling/importance_sampling_ratio/min": 0.7657565116882324, "sampling/sampling_logp_difference/max": 0.28994229435920715, "sampling/sampling_logp_difference/mean": 0.020509791933000086, "step": 6950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 199.15, "completions/mean_terminated_length": 199.15, "completions/min_length": 117.6, "completions/min_terminated_length": 117.6, "entropy": 1.0454927116632462, "epoch": 0.6398823204927829, "frac_reward_zero_std": 0.3, "grad_norm": 4.786548137664795, "kl": 0.01274618151364848, "learning_rate": 4.6444456636407e-06, "loss": 0.0008, "num_tokens": 30328124.0, "reward": 0.09375, "reward_std": 0.6089177608489991, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.6089177668094635, "sampling/importance_sampling_ratio/max": 1.2596526861190795, "sampling/importance_sampling_ratio/mean": 1.0003575563430787, "sampling/importance_sampling_ratio/min": 0.7883762419223785, "sampling/sampling_logp_difference/max": 0.2544731378555298, "sampling/sampling_logp_difference/mean": 0.020805941335856913, "step": 6960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.4, "completions/max_terminated_length": 262.4, "completions/mean_length": 196.7, "completions/mean_terminated_length": 196.7, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.1625725962221622, "epoch": 0.6408016916429162, "frac_reward_zero_std": 0.6, "grad_norm": 0.11862431466579437, "kl": 0.014373186870943754, "learning_rate": 4.643142229376178e-06, "loss": 0.0003, "num_tokens": 30372612.0, "reward": 0.175, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.2771398901939393, "sampling/importance_sampling_ratio/mean": 0.999726539850235, "sampling/importance_sampling_ratio/min": 0.788287079334259, "sampling/sampling_logp_difference/max": 0.2703669548034668, "sampling/sampling_logp_difference/mean": 0.021910946257412432, "step": 6970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.5, "completions/max_terminated_length": 279.5, "completions/mean_length": 201.8875, "completions/mean_terminated_length": 201.8875, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0583705589175225, "epoch": 0.6417210627930495, "frac_reward_zero_std": 0.4, "grad_norm": 0.10379751771688461, "kl": 0.013229988503735513, "learning_rate": 4.64183659393279e-06, "loss": 0.0009, "num_tokens": 30412107.0, "reward": -0.375, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.3010121703147888, "sampling/importance_sampling_ratio/mean": 1.0001993119716643, "sampling/importance_sampling_ratio/min": 0.8039737164974212, "sampling/sampling_logp_difference/max": 0.26371116638183595, "sampling/sampling_logp_difference/mean": 0.02153287548571825, "step": 6980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.2, "completions/max_terminated_length": 318.2, "completions/mean_length": 207.3, "completions/mean_terminated_length": 207.3, "completions/min_length": 126.3, "completions/min_terminated_length": 126.3, "entropy": 1.0708966724574567, "epoch": 0.6426404339431828, "frac_reward_zero_std": 0.5, "grad_norm": 6.704335689544678, "kl": 0.012586241832468658, "learning_rate": 4.640528758651532e-06, "loss": 0.0007, "num_tokens": 30455059.0, "reward": -0.2, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2573201298713683, "sampling/importance_sampling_ratio/mean": 0.999875009059906, "sampling/importance_sampling_ratio/min": 0.7710090577602386, "sampling/sampling_logp_difference/max": 0.30048797726631166, "sampling/sampling_logp_difference/mean": 0.02111421674489975, "step": 6990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 352.2, "completions/max_terminated_length": 328.7, "completions/mean_length": 218.4125, "completions/mean_terminated_length": 214.67857208251954, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 0.9276309177279473, "epoch": 0.6435598050933161, "frac_reward_zero_std": 0.5, "grad_norm": 5.2724289894104, "kl": 0.012088383792433887, "learning_rate": 4.639218724875655e-06, "loss": 0.0007, "num_tokens": 30494852.0, "reward": -0.38125, "reward_std": 0.446427184343338, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.4464271903038025, "sampling/importance_sampling_ratio/max": 1.3094436883926392, "sampling/importance_sampling_ratio/mean": 1.0002029478549956, "sampling/importance_sampling_ratio/min": 0.7807109534740448, "sampling/sampling_logp_difference/max": 0.30850088596343994, "sampling/sampling_logp_difference/mean": 0.01887863837182522, "step": 7000 }, { "epoch": 0.6435598050933161, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 284.1640488656195, "eval_completions/max_terminated_length": 282.85340314136124, "eval_completions/mean_length": 196.8852530541012, "eval_completions/mean_terminated_length": 196.61530803261002, "eval_completions/min_length": 125.19895287958116, "eval_completions/min_terminated_length": 125.19895287958116, "eval_entropy": 1.071847322359551, "eval_frac_reward_zero_std": 0.45200698080279234, "eval_kl": 0.013672170641281062, "eval_loss": 0.0007036181632429361, "eval_num_tokens": 30494852.0, "eval_reward": -0.10111256544502618, "eval_reward_std": 0.48201471944233926, "eval_rewards/reward_fn/mean": -0.10111256544502618, "eval_rewards/reward_fn/std": 0.48201473483760526, "eval_runtime": 1664.2816, "eval_samples_per_second": 0.344, "eval_sampling/importance_sampling_ratio/max": 1.277143068546608, "eval_sampling/importance_sampling_ratio/mean": 1.0000387695447313, "eval_sampling/importance_sampling_ratio/min": 0.7766850197242401, "eval_sampling/sampling_logp_difference/max": 0.28616350283277386, "eval_sampling/sampling_logp_difference/mean": 0.021055045534399897, "eval_steps_per_second": 0.043, "step": 7000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.9, "completions/max_terminated_length": 302.9, "completions/mean_length": 209.4125, "completions/mean_terminated_length": 209.4125, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1222404755651951, "epoch": 0.6444791762434495, "frac_reward_zero_std": 0.5, "grad_norm": 4.499269485473633, "kl": 0.01390745629905723, "learning_rate": 4.637906493950668e-06, "loss": 0.0009, "num_tokens": 30537957.0, "reward": -0.5, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2888835668563843, "sampling/importance_sampling_ratio/mean": 0.9999236285686492, "sampling/importance_sampling_ratio/min": 0.7749639213085174, "sampling/sampling_logp_difference/max": 0.2886025905609131, "sampling/sampling_logp_difference/mean": 0.021849279664456843, "step": 7010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 212.175, "completions/mean_terminated_length": 212.175, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.087604008615017, "epoch": 0.6453985473935828, "frac_reward_zero_std": 0.5, "grad_norm": 0.1855287104845047, "kl": 0.012688208738109098, "learning_rate": 4.636592067224337e-06, "loss": 0.0007, "num_tokens": 30581443.0, "reward": -0.03125, "reward_std": 0.5224424958229065, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5224425315856933, "sampling/importance_sampling_ratio/max": 1.2437695503234862, "sampling/importance_sampling_ratio/mean": 0.9999536216259003, "sampling/importance_sampling_ratio/min": 0.7664796113967896, "sampling/sampling_logp_difference/max": 0.28432133197784426, "sampling/sampling_logp_difference/mean": 0.021528596058487892, "step": 7020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.9, "completions/max_terminated_length": 318.9, "completions/mean_length": 221.25, "completions/mean_terminated_length": 221.25, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.2106634810566903, "epoch": 0.6463179185437161, "frac_reward_zero_std": 0.3, "grad_norm": 0.11777020245790482, "kl": 0.012946854712208734, "learning_rate": 4.635275446046685e-06, "loss": 0.0008, "num_tokens": 30627095.0, "reward": -0.325, "reward_std": 0.6111050426959992, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6111050426959992, "sampling/importance_sampling_ratio/max": 1.2264029502868652, "sampling/importance_sampling_ratio/mean": 0.9996461808681488, "sampling/importance_sampling_ratio/min": 0.7928543746471405, "sampling/sampling_logp_difference/max": 0.2367732286453247, "sampling/sampling_logp_difference/mean": 0.02239028885960579, "step": 7030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 331.2, "completions/max_terminated_length": 315.3, "completions/mean_length": 232.6, "completions/mean_terminated_length": 229.1750015258789, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.123713205009699, "epoch": 0.6472372896938494, "frac_reward_zero_std": 0.5, "grad_norm": 0.1075737476348877, "kl": 0.01530649869237095, "learning_rate": 4.633956631769986e-06, "loss": 0.0007, "num_tokens": 30673751.0, "reward": -0.68125, "reward_std": 0.32239171117544174, "rewards/reward_fn/mean": -0.68125, "rewards/reward_fn/std": 0.32239171117544174, "sampling/importance_sampling_ratio/max": 1.2435141921043396, "sampling/importance_sampling_ratio/mean": 1.0001457929611206, "sampling/importance_sampling_ratio/min": 0.7957070946693421, "sampling/sampling_logp_difference/max": 0.2419174551963806, "sampling/sampling_logp_difference/mean": 0.021192418225109578, "step": 7040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 218.7875, "completions/mean_terminated_length": 218.7875, "completions/min_length": 149.3, "completions/min_terminated_length": 149.3, "entropy": 1.1174160428345203, "epoch": 0.6481566608439827, "frac_reward_zero_std": 0.2, "grad_norm": 5.925088405609131, "kl": 0.014045749767683446, "learning_rate": 4.632635625748768e-06, "loss": 0.0006, "num_tokens": 30715678.0, "reward": -0.1, "reward_std": 0.6422272384166717, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6422272503376008, "sampling/importance_sampling_ratio/max": 1.282818365097046, "sampling/importance_sampling_ratio/mean": 0.9999416708946228, "sampling/importance_sampling_ratio/min": 0.7498128294944764, "sampling/sampling_logp_difference/max": 0.3212683081626892, "sampling/sampling_logp_difference/mean": 0.021746222488582136, "step": 7050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.8, "completions/max_terminated_length": 306.8, "completions/mean_length": 220.9125, "completions/mean_terminated_length": 220.9125, "completions/min_length": 152.8, "completions/min_terminated_length": 152.8, "entropy": 1.0097453318536282, "epoch": 0.649076031994116, "frac_reward_zero_std": 0.6, "grad_norm": 0.1176348403096199, "kl": 0.011836492939619348, "learning_rate": 4.631312429339811e-06, "loss": 0.0008, "num_tokens": 30758359.0, "reward": 0.15, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2569056630134583, "sampling/importance_sampling_ratio/mean": 0.9999224722385407, "sampling/importance_sampling_ratio/min": 0.7961991131305695, "sampling/sampling_logp_difference/max": 0.25167847871780397, "sampling/sampling_logp_difference/mean": 0.02000873303040862, "step": 7060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 363.8, "completions/max_terminated_length": 336.5, "completions/mean_length": 247.5625, "completions/mean_terminated_length": 241.5964324951172, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.13183763474226, "epoch": 0.6499954031442493, "frac_reward_zero_std": 0.3, "grad_norm": 0.10691054165363312, "kl": 0.01326324868714437, "learning_rate": 4.6299870439021446e-06, "loss": 0.0005, "num_tokens": 30805124.0, "reward": -0.24375, "reward_std": 0.5947821602225304, "rewards/reward_fn/mean": -0.24375, "rewards/reward_fn/std": 0.5947821721434593, "sampling/importance_sampling_ratio/max": 1.2597053050994873, "sampling/importance_sampling_ratio/mean": 1.0000607550144196, "sampling/importance_sampling_ratio/min": 0.7917557120323181, "sampling/sampling_logp_difference/max": 0.2544626951217651, "sampling/sampling_logp_difference/mean": 0.021227643452584745, "step": 7070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.5, "completions/max_terminated_length": 311.5, "completions/mean_length": 221.9875, "completions/mean_terminated_length": 221.9875, "completions/min_length": 152.3, "completions/min_terminated_length": 152.3, "entropy": 1.0553542710840702, "epoch": 0.6509147742943826, "frac_reward_zero_std": 0.2, "grad_norm": 6.998108863830566, "kl": 0.012483163439901546, "learning_rate": 4.628659470797044e-06, "loss": 0.0003, "num_tokens": 30848787.0, "reward": -0.05, "reward_std": 0.7658906996250152, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.7658907353878022, "sampling/importance_sampling_ratio/max": 1.266426968574524, "sampling/importance_sampling_ratio/mean": 1.0000426054000855, "sampling/importance_sampling_ratio/min": 0.8016277372837066, "sampling/sampling_logp_difference/max": 0.25018908977508547, "sampling/sampling_logp_difference/mean": 0.02059396877884865, "step": 7080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 220.4375, "completions/mean_terminated_length": 220.4375, "completions/min_length": 134.9, "completions/min_terminated_length": 134.9, "entropy": 1.1339677639305592, "epoch": 0.6518341454445159, "frac_reward_zero_std": 0.5, "grad_norm": 0.13222593069076538, "kl": 0.01622246265760623, "learning_rate": 4.627329711388033e-06, "loss": 0.0012, "num_tokens": 30894438.0, "reward": -0.3, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.2464215755462646, "sampling/importance_sampling_ratio/mean": 1.0004166126251222, "sampling/importance_sampling_ratio/min": 0.7726547241210937, "sampling/sampling_logp_difference/max": 0.2681657075881958, "sampling/sampling_logp_difference/mean": 0.0222994539886713, "step": 7090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.4, "completions/max_terminated_length": 300.4, "completions/mean_length": 195.5625, "completions/mean_terminated_length": 195.5625, "completions/min_length": 117.2, "completions/min_terminated_length": 117.2, "entropy": 1.1113900147378444, "epoch": 0.6527535165946493, "frac_reward_zero_std": 0.2, "grad_norm": 5.826842784881592, "kl": 0.014433144708164036, "learning_rate": 4.625997767040883e-06, "loss": 0.001, "num_tokens": 30933635.0, "reward": -0.025, "reward_std": 0.7802288770675659, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.7802288889884949, "sampling/importance_sampling_ratio/max": 1.2982587814331055, "sampling/importance_sampling_ratio/mean": 0.9999959766864777, "sampling/importance_sampling_ratio/min": 0.7995989859104157, "sampling/sampling_logp_difference/max": 0.2802507996559143, "sampling/sampling_logp_difference/mean": 0.021926282811909913, "step": 7100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 196.4125, "completions/mean_terminated_length": 196.4125, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.0740439862012863, "epoch": 0.6536728877447826, "frac_reward_zero_std": 0.2, "grad_norm": 0.11543919146060944, "kl": 0.013095683348365129, "learning_rate": 4.624663639123607e-06, "loss": 0.0008, "num_tokens": 30977988.0, "reward": -0.2, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.2589791536331176, "sampling/importance_sampling_ratio/mean": 0.9999546885490418, "sampling/importance_sampling_ratio/min": 0.7961708188056946, "sampling/sampling_logp_difference/max": 0.258049464225769, "sampling/sampling_logp_difference/mean": 0.021516186743974687, "step": 7110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.5, "completions/max_terminated_length": 337.0, "completions/mean_length": 216.5, "completions/mean_terminated_length": 213.68035888671875, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0592115007340908, "epoch": 0.6545922588949159, "frac_reward_zero_std": 0.5, "grad_norm": 7.709811687469482, "kl": 0.012965208530658856, "learning_rate": 4.623327329006461e-06, "loss": 0.0007, "num_tokens": 31021612.0, "reward": -0.28125, "reward_std": 0.43548367023468015, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.43548368811607363, "sampling/importance_sampling_ratio/max": 1.3051105380058288, "sampling/importance_sampling_ratio/mean": 0.9998682022094727, "sampling/importance_sampling_ratio/min": 0.741610848903656, "sampling/sampling_logp_difference/max": 0.3210714817047119, "sampling/sampling_logp_difference/mean": 0.021227958798408508, "step": 7120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 200.6375, "completions/mean_terminated_length": 200.6375, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0800247654318809, "epoch": 0.6555116300450492, "frac_reward_zero_std": 0.2, "grad_norm": 6.0163187980651855, "kl": 0.013485591963399202, "learning_rate": 4.621988838061942e-06, "loss": 0.0006, "num_tokens": 31063039.0, "reward": -0.35, "reward_std": 0.6750263929367065, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.6750264167785645, "sampling/importance_sampling_ratio/max": 1.2986309766769408, "sampling/importance_sampling_ratio/mean": 0.9998098373413086, "sampling/importance_sampling_ratio/min": 0.7950408518314361, "sampling/sampling_logp_difference/max": 0.27585268020629883, "sampling/sampling_logp_difference/mean": 0.02135090958327055, "step": 7130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 319.5, "completions/max_terminated_length": 302.8, "completions/mean_length": 223.725, "completions/mean_terminated_length": 218.0125, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.0499363131821156, "epoch": 0.6564310011951825, "frac_reward_zero_std": 0.5, "grad_norm": 5.662688732147217, "kl": 0.011674515943741425, "learning_rate": 4.620648167664789e-06, "loss": 0.0006, "num_tokens": 31105689.0, "reward": -0.2625, "reward_std": 0.40438567399978637, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.40438570976257326, "sampling/importance_sampling_ratio/max": 1.3221560835838317, "sampling/importance_sampling_ratio/mean": 1.0003158688545226, "sampling/importance_sampling_ratio/min": 0.7975208520889282, "sampling/sampling_logp_difference/max": 0.2907664865255356, "sampling/sampling_logp_difference/mean": 0.021052646078169347, "step": 7140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 194.9375, "completions/mean_terminated_length": 194.9375, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.1004159934818745, "epoch": 0.6573503723453158, "frac_reward_zero_std": 0.5, "grad_norm": 0.127359077334404, "kl": 0.014598513842793181, "learning_rate": 4.619305319191975e-06, "loss": 0.0008, "num_tokens": 31148748.0, "reward": -0.05625, "reward_std": 0.4206503689289093, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.42065038084983825, "sampling/importance_sampling_ratio/max": 1.2476365327835084, "sampling/importance_sampling_ratio/mean": 1.0000236690044404, "sampling/importance_sampling_ratio/min": 0.7693938195705414, "sampling/sampling_logp_difference/max": 0.28074294328689575, "sampling/sampling_logp_difference/mean": 0.02170372921973467, "step": 7150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.5, "completions/max_terminated_length": 264.5, "completions/mean_length": 192.4875, "completions/mean_terminated_length": 192.4875, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.0737413495779038, "epoch": 0.6582697434954491, "frac_reward_zero_std": 0.5, "grad_norm": 6.5150227546691895, "kl": 0.013738722022389993, "learning_rate": 4.617960294022717e-06, "loss": 0.0008, "num_tokens": 31192515.0, "reward": -0.175, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.333419668674469, "sampling/importance_sampling_ratio/mean": 1.0000180661678315, "sampling/importance_sampling_ratio/min": 0.7475740134716033, "sampling/sampling_logp_difference/max": 0.33612738847732543, "sampling/sampling_logp_difference/mean": 0.021398316882550717, "step": 7160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 204.7125, "completions/mean_terminated_length": 204.7125, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0547878183424473, "epoch": 0.6591891146455824, "frac_reward_zero_std": 0.6, "grad_norm": 5.582183361053467, "kl": 0.012809601312619634, "learning_rate": 4.616613093538461e-06, "loss": 0.0006, "num_tokens": 31237916.0, "reward": 0.3, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.2653074502944945, "sampling/importance_sampling_ratio/mean": 1.0002614200115203, "sampling/importance_sampling_ratio/min": 0.7713806331157684, "sampling/sampling_logp_difference/max": 0.2937042474746704, "sampling/sampling_logp_difference/mean": 0.020799442008137704, "step": 7170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.2, "completions/max_terminated_length": 274.2, "completions/mean_length": 180.3625, "completions/mean_terminated_length": 180.3625, "completions/min_length": 108.0, "completions/min_terminated_length": 108.0, "entropy": 1.0666462995111943, "epoch": 0.6601084857957157, "frac_reward_zero_std": 0.3, "grad_norm": 7.489744663238525, "kl": 0.01315185495186597, "learning_rate": 4.615263719122891e-06, "loss": 0.0007, "num_tokens": 31277873.0, "reward": -0.125, "reward_std": 0.538717406988144, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.538717406988144, "sampling/importance_sampling_ratio/max": 1.32176730632782, "sampling/importance_sampling_ratio/mean": 1.0007331907749175, "sampling/importance_sampling_ratio/min": 0.7786247670650482, "sampling/sampling_logp_difference/max": 0.34165329933166505, "sampling/sampling_logp_difference/mean": 0.021101703122258186, "step": 7180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 200.95, "completions/mean_terminated_length": 200.95, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.048249178379774, "epoch": 0.6610278569458491, "frac_reward_zero_std": 0.7, "grad_norm": 0.10223764181137085, "kl": 0.012529148213798181, "learning_rate": 4.613912172161923e-06, "loss": 0.0006, "num_tokens": 31319701.0, "reward": -0.275, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.3040303707122802, "sampling/importance_sampling_ratio/mean": 0.9998725116252899, "sampling/importance_sampling_ratio/min": 0.7885439157485962, "sampling/sampling_logp_difference/max": 0.27669855058193205, "sampling/sampling_logp_difference/mean": 0.022195605747401714, "step": 7190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.4, "completions/max_terminated_length": 305.4, "completions/mean_length": 212.85, "completions/mean_terminated_length": 212.85, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 0.9927649531513453, "epoch": 0.6619472280959824, "frac_reward_zero_std": 0.3, "grad_norm": 5.467513561248779, "kl": 0.011597259389236569, "learning_rate": 4.612558454043702e-06, "loss": 0.0006, "num_tokens": 31364057.0, "reward": -0.15, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2340598821640014, "sampling/importance_sampling_ratio/mean": 0.9995432436466217, "sampling/importance_sampling_ratio/min": 0.7852667510509491, "sampling/sampling_logp_difference/max": 0.2510343611240387, "sampling/sampling_logp_difference/mean": 0.019636253360658884, "step": 7200 }, { "epoch": 0.6619472280959824, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 305.6876090750436, "eval_completions/max_terminated_length": 302.8237347294939, "eval_completions/mean_length": 212.53097731239092, "eval_completions/mean_terminated_length": 211.95216303054784, "eval_completions/min_length": 138.73123909249563, "eval_completions/min_terminated_length": 138.73123909249563, "eval_entropy": 1.0986937209573715, "eval_frac_reward_zero_std": 0.4642233856893543, "eval_kl": 0.013213522861472754, "eval_loss": 0.0006504422053694725, "eval_num_tokens": 31364057.0, "eval_reward": -0.11725567190226877, "eval_reward_std": 0.4671095831231921, "eval_rewards/reward_fn/mean": -0.11725567190226877, "eval_rewards/reward_fn/std": 0.4671095993506347, "eval_runtime": 1772.0232, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2709084137572044, "eval_sampling/importance_sampling_ratio/mean": 0.9999895157198215, "eval_sampling/importance_sampling_ratio/min": 0.7753419616563574, "eval_sampling/sampling_logp_difference/max": 0.2828384679961579, "eval_sampling/sampling_logp_difference/mean": 0.021289987346550764, "eval_steps_per_second": 0.041, "step": 7200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.8, "completions/max_terminated_length": 307.8, "completions/mean_length": 223.125, "completions/mean_terminated_length": 223.125, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.0410732313990594, "epoch": 0.6628665992461157, "frac_reward_zero_std": 0.2, "grad_norm": 6.807127952575684, "kl": 0.012633659230777994, "learning_rate": 4.611202566158607e-06, "loss": 0.0002, "num_tokens": 31408763.0, "reward": 0.15, "reward_std": 0.7549471855163574, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.7549472332000733, "sampling/importance_sampling_ratio/max": 1.2522562861442565, "sampling/importance_sampling_ratio/mean": 1.0003236413002015, "sampling/importance_sampling_ratio/min": 0.7914454162120819, "sampling/sampling_logp_difference/max": 0.2554145336151123, "sampling/sampling_logp_difference/mean": 0.019987775944173336, "step": 7210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.2, "completions/max_terminated_length": 298.2, "completions/mean_length": 215.825, "completions/mean_terminated_length": 215.825, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.05211578682065, "epoch": 0.663785970396249, "frac_reward_zero_std": 0.3, "grad_norm": 0.11826147884130478, "kl": 0.013826043257722631, "learning_rate": 4.609844509899243e-06, "loss": 0.0007, "num_tokens": 31454373.0, "reward": -0.40625, "reward_std": 0.5878328442573547, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.5878328740596771, "sampling/importance_sampling_ratio/max": 1.3829928517341614, "sampling/importance_sampling_ratio/mean": 1.0001531422138215, "sampling/importance_sampling_ratio/min": 0.7795573592185974, "sampling/sampling_logp_difference/max": 0.34242090582847595, "sampling/sampling_logp_difference/mean": 0.021138047985732555, "step": 7220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 282.8, "completions/max_terminated_length": 280.3, "completions/mean_length": 195.625, "completions/mean_terminated_length": 192.86607360839844, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.0296320348978043, "epoch": 0.6647053415463823, "frac_reward_zero_std": 0.5, "grad_norm": 0.1349049061536789, "kl": 0.014292489044601097, "learning_rate": 4.6084842866604426e-06, "loss": 0.0008, "num_tokens": 31495047.0, "reward": 0.21875, "reward_std": 0.4946424126625061, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.4946424126625061, "sampling/importance_sampling_ratio/max": 1.275464916229248, "sampling/importance_sampling_ratio/mean": 0.9998139083385468, "sampling/importance_sampling_ratio/min": 0.7782023072242736, "sampling/sampling_logp_difference/max": 0.2682417154312134, "sampling/sampling_logp_difference/mean": 0.021343384683132172, "step": 7230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 201.15, "completions/mean_terminated_length": 201.15, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.0483795806765557, "epoch": 0.6656247126965156, "frac_reward_zero_std": 0.4, "grad_norm": 0.13609015941619873, "kl": 0.014316833287011832, "learning_rate": 4.607121897839264e-06, "loss": 0.0002, "num_tokens": 31540211.0, "reward": -0.175, "reward_std": 0.5807423710823059, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5807423830032349, "sampling/importance_sampling_ratio/max": 1.330616331100464, "sampling/importance_sampling_ratio/mean": 0.9999494016170501, "sampling/importance_sampling_ratio/min": 0.795012092590332, "sampling/sampling_logp_difference/max": 0.2892928242683411, "sampling/sampling_logp_difference/mean": 0.021040657348930836, "step": 7240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 357.8, "completions/max_terminated_length": 334.1, "completions/mean_length": 232.4625, "completions/mean_terminated_length": 226.21785888671874, "completions/min_length": 151.2, "completions/min_terminated_length": 151.2, "entropy": 1.0883030854165554, "epoch": 0.6665440838466489, "frac_reward_zero_std": 0.4, "grad_norm": 5.106171607971191, "kl": 0.012125975196249783, "learning_rate": 4.605757344834989e-06, "loss": 0.0005, "num_tokens": 31585968.0, "reward": -0.2625, "reward_std": 0.5126763999462127, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.5126764059066773, "sampling/importance_sampling_ratio/max": 1.243828535079956, "sampling/importance_sampling_ratio/mean": 0.9996449708938598, "sampling/importance_sampling_ratio/min": 0.7760604560375214, "sampling/sampling_logp_difference/max": 0.2630816876888275, "sampling/sampling_logp_difference/mean": 0.021224530786275862, "step": 7250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.4, "completions/max_terminated_length": 312.4, "completions/mean_length": 207.4625, "completions/mean_terminated_length": 207.4625, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0673184350132943, "epoch": 0.6674634549967822, "frac_reward_zero_std": 0.5, "grad_norm": 4.516080379486084, "kl": 0.014982869953382761, "learning_rate": 4.604390629049125e-06, "loss": 0.001, "num_tokens": 31631949.0, "reward": -0.075, "reward_std": 0.4696836888790131, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4696836888790131, "sampling/importance_sampling_ratio/max": 1.3281073927879334, "sampling/importance_sampling_ratio/mean": 0.9999613285064697, "sampling/importance_sampling_ratio/min": 0.7802676558494568, "sampling/sampling_logp_difference/max": 0.3103169322013855, "sampling/sampling_logp_difference/mean": 0.021106972359120846, "step": 7260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.3, "completions/max_terminated_length": 309.3, "completions/mean_length": 226.9125, "completions/mean_terminated_length": 226.9125, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.0653372682631015, "epoch": 0.6683828261469155, "frac_reward_zero_std": 0.7, "grad_norm": 0.13786208629608154, "kl": 0.013468979965546168, "learning_rate": 4.603021751885397e-06, "loss": 0.0006, "num_tokens": 31675414.0, "reward": -0.325, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2397347331047057, "sampling/importance_sampling_ratio/mean": 1.0001973807811737, "sampling/importance_sampling_ratio/min": 0.7342250764369964, "sampling/sampling_logp_difference/max": 0.3157650947570801, "sampling/sampling_logp_difference/mean": 0.021303127333521844, "step": 7270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 200.0625, "completions/mean_terminated_length": 200.0625, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 0.9823125764727593, "epoch": 0.6693021972970489, "frac_reward_zero_std": 0.3, "grad_norm": 0.1257641762495041, "kl": 0.01322268505464308, "learning_rate": 4.601650714749752e-06, "loss": 0.0013, "num_tokens": 31715435.0, "reward": -0.35, "reward_std": 0.6261870503425598, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.6261870741844178, "sampling/importance_sampling_ratio/max": 1.2477649331092835, "sampling/importance_sampling_ratio/mean": 1.0005633175373077, "sampling/importance_sampling_ratio/min": 0.7928510308265686, "sampling/sampling_logp_difference/max": 0.25440030097961425, "sampling/sampling_logp_difference/mean": 0.019906306639313698, "step": 7280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.9, "completions/max_terminated_length": 295.9, "completions/mean_length": 208.9625, "completions/mean_terminated_length": 208.9625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.058735778182745, "epoch": 0.6702215684471822, "frac_reward_zero_std": 0.5, "grad_norm": 5.251330375671387, "kl": 0.013585271057672798, "learning_rate": 4.600277519050355e-06, "loss": 0.0006, "num_tokens": 31756896.0, "reward": -0.225, "reward_std": 0.48400617241859434, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.48400617241859434, "sampling/importance_sampling_ratio/max": 1.243813443183899, "sampling/importance_sampling_ratio/mean": 0.9993820726871491, "sampling/importance_sampling_ratio/min": 0.7592293202877045, "sampling/sampling_logp_difference/max": 0.2884256362915039, "sampling/sampling_logp_difference/mean": 0.021113764308393002, "step": 7290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 216.1625, "completions/mean_terminated_length": 216.1625, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.1301624044775962, "epoch": 0.6711409395973155, "frac_reward_zero_std": 0.4, "grad_norm": 0.28059670329093933, "kl": 0.01373261078260839, "learning_rate": 4.598902166197589e-06, "loss": 0.0004, "num_tokens": 31801309.0, "reward": -0.375, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2343312501907349, "sampling/importance_sampling_ratio/mean": 0.9997979938983917, "sampling/importance_sampling_ratio/min": 0.7933791518211365, "sampling/sampling_logp_difference/max": 0.23679301738739014, "sampling/sampling_logp_difference/mean": 0.02147707622498274, "step": 7300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.3, "completions/max_terminated_length": 285.3, "completions/mean_length": 208.1, "completions/mean_terminated_length": 208.1, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.148018406331539, "epoch": 0.6720603107474488, "frac_reward_zero_std": 0.5, "grad_norm": 0.5835710763931274, "kl": 0.013098191533936187, "learning_rate": 4.597524657604049e-06, "loss": 0.0007, "num_tokens": 31848485.0, "reward": -0.575, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2896743655204772, "sampling/importance_sampling_ratio/mean": 0.9999469637870788, "sampling/importance_sampling_ratio/min": 0.7659646272659302, "sampling/sampling_logp_difference/max": 0.3005671977996826, "sampling/sampling_logp_difference/mean": 0.021824601478874682, "step": 7310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 309.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 218.9875, "completions/mean_terminated_length": 215.60714416503907, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 1.0519354112446309, "epoch": 0.6729796818975821, "frac_reward_zero_std": 0.3, "grad_norm": 0.09417544305324554, "kl": 0.014495330589124934, "learning_rate": 4.5961449946845495e-06, "loss": 0.0007, "num_tokens": 31894812.0, "reward": 0.21875, "reward_std": 0.6080160975456238, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.6080161094665527, "sampling/importance_sampling_ratio/max": 1.3294335007667542, "sampling/importance_sampling_ratio/mean": 0.9998958289623261, "sampling/importance_sampling_ratio/min": 0.7737886428833007, "sampling/sampling_logp_difference/max": 0.3331479370594025, "sampling/sampling_logp_difference/mean": 0.020785226486623287, "step": 7320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 208.8625, "completions/mean_terminated_length": 208.8625, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.1048367410898208, "epoch": 0.6738990530477154, "frac_reward_zero_std": 0.5, "grad_norm": 6.374660015106201, "kl": 0.012759653676766902, "learning_rate": 4.594763178856113e-06, "loss": 0.0006, "num_tokens": 31938721.0, "reward": -0.3, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.2513394474983215, "sampling/importance_sampling_ratio/mean": 1.000237900018692, "sampling/importance_sampling_ratio/min": 0.80212881565094, "sampling/sampling_logp_difference/max": 0.24143928289413452, "sampling/sampling_logp_difference/mean": 0.021231202967464925, "step": 7330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 215.575, "completions/mean_terminated_length": 215.575, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0721113413572312, "epoch": 0.6748184241978487, "frac_reward_zero_std": 0.6, "grad_norm": 5.825749397277832, "kl": 0.013220571278361603, "learning_rate": 4.593379211537977e-06, "loss": 0.0006, "num_tokens": 31983015.0, "reward": -0.225, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2658117413520813, "sampling/importance_sampling_ratio/mean": 0.9996518850326538, "sampling/importance_sampling_ratio/min": 0.7926708698272705, "sampling/sampling_logp_difference/max": 0.2659019470214844, "sampling/sampling_logp_difference/mean": 0.02108414303511381, "step": 7340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.6, "completions/max_terminated_length": 326.6, "completions/mean_length": 214.4125, "completions/mean_terminated_length": 214.4125, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.1187768876552582, "epoch": 0.675737795347982, "frac_reward_zero_std": 0.7, "grad_norm": 0.1188107505440712, "kl": 0.012524129246594384, "learning_rate": 4.591993094151586e-06, "loss": 0.0009, "num_tokens": 32030008.0, "reward": -0.05, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2478779315948487, "sampling/importance_sampling_ratio/mean": 1.0000717699527741, "sampling/importance_sampling_ratio/min": 0.8051528990268707, "sampling/sampling_logp_difference/max": 0.23995606899261473, "sampling/sampling_logp_difference/mean": 0.020897566340863705, "step": 7350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 213.975, "completions/mean_terminated_length": 213.975, "completions/min_length": 134.9, "completions/min_terminated_length": 134.9, "entropy": 1.0616620533168315, "epoch": 0.6766571664981152, "frac_reward_zero_std": 0.4, "grad_norm": 5.189362525939941, "kl": 0.012554944213479757, "learning_rate": 4.590604828120593e-06, "loss": 0.0005, "num_tokens": 32073998.0, "reward": -0.43125, "reward_std": 0.4805720314383507, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.4805720552802086, "sampling/importance_sampling_ratio/max": 1.2674453973770141, "sampling/importance_sampling_ratio/mean": 1.0002794086933136, "sampling/importance_sampling_ratio/min": 0.8002573370933532, "sampling/sampling_logp_difference/max": 0.24773343801498413, "sampling/sampling_logp_difference/mean": 0.020656614564359188, "step": 7360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 206.6125, "completions/mean_terminated_length": 206.6125, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0031670525670051, "epoch": 0.6775765376482485, "frac_reward_zero_std": 0.2, "grad_norm": 5.812961578369141, "kl": 0.011402834230102599, "learning_rate": 4.589214414870858e-06, "loss": 0.0006, "num_tokens": 32121191.0, "reward": 0.125, "reward_std": 0.7002766966819763, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.7002767324447632, "sampling/importance_sampling_ratio/max": 1.2517301201820374, "sampling/importance_sampling_ratio/mean": 1.0001061141490937, "sampling/importance_sampling_ratio/min": 0.7544350922107697, "sampling/sampling_logp_difference/max": 0.3079416751861572, "sampling/sampling_logp_difference/mean": 0.0202798867598176, "step": 7370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 200.2375, "completions/mean_terminated_length": 200.2375, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 0.9126736426725983, "epoch": 0.678495908798382, "frac_reward_zero_std": 0.5, "grad_norm": 3.1218576431274414, "kl": 0.011367647504084744, "learning_rate": 4.58782185583045e-06, "loss": 0.0008, "num_tokens": 32164250.0, "reward": 0.075, "reward_std": 0.4587401747703552, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4587401866912842, "sampling/importance_sampling_ratio/max": 1.271173858642578, "sampling/importance_sampling_ratio/mean": 1.0000624895095824, "sampling/importance_sampling_ratio/min": 0.7945567190647125, "sampling/sampling_logp_difference/max": 0.26567124128341674, "sampling/sampling_logp_difference/mean": 0.018318990897387265, "step": 7380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 188.0625, "completions/mean_terminated_length": 188.0625, "completions/min_length": 115.8, "completions/min_terminated_length": 115.8, "entropy": 1.0357622116804124, "epoch": 0.6794152799485152, "frac_reward_zero_std": 0.5, "grad_norm": 0.17545770108699799, "kl": 0.020082729076966642, "learning_rate": 4.5864271524296355e-06, "loss": 0.0011, "num_tokens": 32203623.0, "reward": 0.025, "reward_std": 0.45536120533943175, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.45536120533943175, "sampling/importance_sampling_ratio/max": 1.2728724360466004, "sampling/importance_sampling_ratio/mean": 0.9999041557312012, "sampling/importance_sampling_ratio/min": 0.7776906132698059, "sampling/sampling_logp_difference/max": 0.28098747730255125, "sampling/sampling_logp_difference/mean": 0.02050054892897606, "step": 7390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 189.5125, "completions/mean_terminated_length": 189.5125, "completions/min_length": 119.1, "completions/min_terminated_length": 119.1, "entropy": 1.0010408282279968, "epoch": 0.6803346510986485, "frac_reward_zero_std": 0.5, "grad_norm": 5.389584541320801, "kl": 0.0162218737998046, "learning_rate": 4.585030306100887e-06, "loss": 0.0002, "num_tokens": 32243304.0, "reward": -0.175, "reward_std": 0.42594102025032043, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.42594102025032043, "sampling/importance_sampling_ratio/max": 1.2216119527816773, "sampling/importance_sampling_ratio/mean": 0.9998199820518494, "sampling/importance_sampling_ratio/min": 0.7630745887756347, "sampling/sampling_logp_difference/max": 0.2797402560710907, "sampling/sampling_logp_difference/mean": 0.02027832642197609, "step": 7400 }, { "epoch": 0.6803346510986485, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 287.9685863874345, "eval_completions/max_terminated_length": 286.4223385689354, "eval_completions/mean_length": 199.09467713787086, "eval_completions/mean_terminated_length": 198.70415745813392, "eval_completions/min_length": 127.03315881326353, "eval_completions/min_terminated_length": 127.03315881326353, "eval_entropy": 1.068321117711941, "eval_frac_reward_zero_std": 0.47469458987783597, "eval_kl": 0.014091189950704575, "eval_loss": 0.0006992268608883023, "eval_num_tokens": 32243304.0, "eval_reward": -0.1087478184991274, "eval_reward_std": 0.4642822731279786, "eval_rewards/reward_fn/mean": -0.1087478184991274, "eval_rewards/reward_fn/std": 0.464282287587046, "eval_runtime": 1682.9078, "eval_samples_per_second": 0.34, "eval_sampling/importance_sampling_ratio/max": 1.2698188391863572, "eval_sampling/importance_sampling_ratio/mean": 1.0000646089384069, "eval_sampling/importance_sampling_ratio/min": 0.7744522804157064, "eval_sampling/sampling_logp_difference/max": 0.28828560895112587, "eval_sampling/sampling_logp_difference/mean": 0.021072522358380478, "eval_steps_per_second": 0.043, "step": 7400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.9, "completions/max_terminated_length": 261.9, "completions/mean_length": 195.3, "completions/mean_terminated_length": 195.3, "completions/min_length": 117.7, "completions/min_terminated_length": 117.7, "entropy": 1.1375061012804508, "epoch": 0.6812540222487818, "frac_reward_zero_std": 0.5, "grad_norm": 0.13014572858810425, "kl": 0.012365566485095769, "learning_rate": 4.5836313182788804e-06, "loss": 0.0005, "num_tokens": 32284296.0, "reward": 0.4, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2634915947914123, "sampling/importance_sampling_ratio/mean": 0.9995404183864594, "sampling/importance_sampling_ratio/min": 0.7927441596984863, "sampling/sampling_logp_difference/max": 0.2544417858123779, "sampling/sampling_logp_difference/mean": 0.02185969762504101, "step": 7410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 214.075, "completions/mean_terminated_length": 214.075, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0757948748767376, "epoch": 0.6821733933989151, "frac_reward_zero_std": 0.3, "grad_norm": 0.11409033834934235, "kl": 0.017890160239767285, "learning_rate": 4.582230190400486e-06, "loss": 0.0006, "num_tokens": 32331502.0, "reward": -0.275, "reward_std": 0.6767033517360688, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6767033755779266, "sampling/importance_sampling_ratio/max": 1.296743893623352, "sampling/importance_sampling_ratio/mean": 1.0006622552871705, "sampling/importance_sampling_ratio/min": 0.7876415908336639, "sampling/sampling_logp_difference/max": 0.27911301255226134, "sampling/sampling_logp_difference/mean": 0.020274607092142106, "step": 7420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 199.825, "completions/mean_terminated_length": 199.825, "completions/min_length": 118.6, "completions/min_terminated_length": 118.6, "entropy": 0.9550028210505843, "epoch": 0.6830927645490484, "frac_reward_zero_std": 0.5, "grad_norm": 5.933973789215088, "kl": 0.012520952371414752, "learning_rate": 4.580826923904774e-06, "loss": 0.0004, "num_tokens": 32375664.0, "reward": 0.025, "reward_std": 0.45536120533943175, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.45536120533943175, "sampling/importance_sampling_ratio/max": 1.2579827785491944, "sampling/importance_sampling_ratio/mean": 0.9999578833580017, "sampling/importance_sampling_ratio/min": 0.7514145374298096, "sampling/sampling_logp_difference/max": 0.3190728545188904, "sampling/sampling_logp_difference/mean": 0.01953996755182743, "step": 7430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 195.9625, "completions/mean_terminated_length": 195.9625, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.0466359481215477, "epoch": 0.6840121356991817, "frac_reward_zero_std": 0.4, "grad_norm": 7.04184627532959, "kl": 0.014690878486726433, "learning_rate": 4.579421520233013e-06, "loss": 0.0004, "num_tokens": 32419949.0, "reward": -0.025, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.3238658905029297, "sampling/importance_sampling_ratio/mean": 0.9998562693595886, "sampling/importance_sampling_ratio/min": 0.7800685465335846, "sampling/sampling_logp_difference/max": 0.29748511910438535, "sampling/sampling_logp_difference/mean": 0.021052373573184013, "step": 7440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.6, "completions/max_terminated_length": 330.6, "completions/mean_length": 217.175, "completions/mean_terminated_length": 217.175, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.1432682383805512, "epoch": 0.684931506849315, "frac_reward_zero_std": 0.3, "grad_norm": 6.732651233673096, "kl": 0.013543446728726849, "learning_rate": 4.578013980828664e-06, "loss": 0.0011, "num_tokens": 32469211.0, "reward": 0.125, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.3095009803771973, "sampling/importance_sampling_ratio/mean": 0.9994521021842957, "sampling/importance_sampling_ratio/min": 0.7629216015338898, "sampling/sampling_logp_difference/max": 0.3178255558013916, "sampling/sampling_logp_difference/mean": 0.021629870496690275, "step": 7450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.1, "completions/max_terminated_length": 319.1, "completions/mean_length": 231.5, "completions/mean_terminated_length": 231.5, "completions/min_length": 158.4, "completions/min_terminated_length": 158.4, "entropy": 1.186948573589325, "epoch": 0.6858508779994483, "frac_reward_zero_std": 0.4, "grad_norm": 6.0993475914001465, "kl": 0.014296312333317473, "learning_rate": 4.5766043071373845e-06, "loss": 0.0009, "num_tokens": 32514163.0, "reward": -0.425, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.282647466659546, "sampling/importance_sampling_ratio/mean": 1.0001420497894287, "sampling/importance_sampling_ratio/min": 0.7757820904254913, "sampling/sampling_logp_difference/max": 0.29073081016540525, "sampling/sampling_logp_difference/mean": 0.0220105292275548, "step": 7460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 209.825, "completions/mean_terminated_length": 209.825, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.0707103170454502, "epoch": 0.6867702491495817, "frac_reward_zero_std": 0.3, "grad_norm": 4.354079723358154, "kl": 0.013598239800194279, "learning_rate": 4.57519250060702e-06, "loss": 0.0007, "num_tokens": 32557725.0, "reward": -0.2, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.226995873451233, "sampling/importance_sampling_ratio/mean": 0.9991967022418976, "sampling/importance_sampling_ratio/min": 0.7734785854816437, "sampling/sampling_logp_difference/max": 0.2639762282371521, "sampling/sampling_logp_difference/mean": 0.020776829309761526, "step": 7470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.2, "completions/max_terminated_length": 280.2, "completions/mean_length": 196.425, "completions/mean_terminated_length": 196.425, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.0584315650165081, "epoch": 0.687689620299715, "frac_reward_zero_std": 0.5, "grad_norm": 0.12023525685071945, "kl": 0.01227387169492431, "learning_rate": 4.573778562687612e-06, "loss": 0.001, "num_tokens": 32598927.0, "reward": 0.0, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2386721849441529, "sampling/importance_sampling_ratio/mean": 0.9998223185539246, "sampling/importance_sampling_ratio/min": 0.8055253267288208, "sampling/sampling_logp_difference/max": 0.23766615390777587, "sampling/sampling_logp_difference/mean": 0.02083153333514929, "step": 7480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 223.225, "completions/mean_terminated_length": 223.225, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.1298728056252003, "epoch": 0.6886089914498483, "frac_reward_zero_std": 0.6, "grad_norm": 0.10385890305042267, "kl": 0.01306133281905204, "learning_rate": 4.572362494831386e-06, "loss": 0.0007, "num_tokens": 32642673.0, "reward": -0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2419705986976624, "sampling/importance_sampling_ratio/mean": 1.0002587199211121, "sampling/importance_sampling_ratio/min": 0.7786792635917663, "sampling/sampling_logp_difference/max": 0.2577428936958313, "sampling/sampling_logp_difference/mean": 0.020883635990321635, "step": 7490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.5, "completions/max_terminated_length": 271.5, "completions/mean_length": 204.45, "completions/mean_terminated_length": 204.45, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.1334423467516899, "epoch": 0.6895283625999816, "frac_reward_zero_std": 0.4, "grad_norm": 0.09474965184926987, "kl": 0.013369963341392576, "learning_rate": 4.5709442984927564e-06, "loss": 0.0005, "num_tokens": 32685549.0, "reward": -0.3, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.2840457439422608, "sampling/importance_sampling_ratio/mean": 0.9993945598602295, "sampling/importance_sampling_ratio/min": 0.7898337483406067, "sampling/sampling_logp_difference/max": 0.2548990249633789, "sampling/sampling_logp_difference/mean": 0.021754392236471177, "step": 7500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.4, "completions/max_terminated_length": 310.4, "completions/mean_length": 208.7875, "completions/mean_terminated_length": 208.7875, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0952791519463063, "epoch": 0.6904477337501149, "frac_reward_zero_std": 0.4, "grad_norm": 0.11462806165218353, "kl": 0.011976730602327734, "learning_rate": 4.5695239751283275e-06, "loss": 0.0007, "num_tokens": 32732052.0, "reward": -0.3, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.3485276818275451, "sampling/importance_sampling_ratio/mean": 1.0004139184951781, "sampling/importance_sampling_ratio/min": 0.7774294376373291, "sampling/sampling_logp_difference/max": 0.3159065306186676, "sampling/sampling_logp_difference/mean": 0.02135732416063547, "step": 7510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 212.4625, "completions/mean_terminated_length": 212.4625, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.1110522039234638, "epoch": 0.6913671049002482, "frac_reward_zero_std": 0.3, "grad_norm": 0.10660427063703537, "kl": 0.01175245044287294, "learning_rate": 4.568101526196882e-06, "loss": 0.0004, "num_tokens": 32775337.0, "reward": 0.3, "reward_std": 0.6261870503425598, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.6261870741844178, "sampling/importance_sampling_ratio/max": 1.2539007782936096, "sampling/importance_sampling_ratio/mean": 1.0000085413455964, "sampling/importance_sampling_ratio/min": 0.7852171301841736, "sampling/sampling_logp_difference/max": 0.2615157335996628, "sampling/sampling_logp_difference/mean": 0.021166692301630975, "step": 7520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.4, "completions/max_terminated_length": 320.4, "completions/mean_length": 218.3125, "completions/mean_terminated_length": 218.3125, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.0494569063186645, "epoch": 0.6922864760503815, "frac_reward_zero_std": 0.5, "grad_norm": 0.17673015594482422, "kl": 0.013312400120776146, "learning_rate": 4.566676953159392e-06, "loss": 0.0006, "num_tokens": 32820858.0, "reward": -0.325, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2776724457740785, "sampling/importance_sampling_ratio/mean": 0.9995474100112915, "sampling/importance_sampling_ratio/min": 0.758964017033577, "sampling/sampling_logp_difference/max": 0.3070987641811371, "sampling/sampling_logp_difference/mean": 0.02017091093584895, "step": 7530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.4, "completions/max_terminated_length": 289.4, "completions/mean_length": 208.75, "completions/mean_terminated_length": 208.75, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.1369595650583506, "epoch": 0.6932058472005148, "frac_reward_zero_std": 0.6, "grad_norm": 0.3300573527812958, "kl": 0.014526094106258824, "learning_rate": 4.565250257479004e-06, "loss": 0.0009, "num_tokens": 32864694.0, "reward": 0.275, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.2568421840667725, "sampling/importance_sampling_ratio/mean": 1.000301617383957, "sampling/importance_sampling_ratio/min": 0.7771108746528625, "sampling/sampling_logp_difference/max": 0.26367324590682983, "sampling/sampling_logp_difference/mean": 0.021137416549026965, "step": 7540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 209.95, "completions/mean_terminated_length": 209.95, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.143422019481659, "epoch": 0.6941252183506481, "frac_reward_zero_std": 0.4, "grad_norm": 5.663791179656982, "kl": 0.013664722023531795, "learning_rate": 4.563821440621055e-06, "loss": 0.0003, "num_tokens": 32909322.0, "reward": -0.575, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2357025384902953, "sampling/importance_sampling_ratio/mean": 1.0002930164337158, "sampling/importance_sampling_ratio/min": 0.8047625958919525, "sampling/sampling_logp_difference/max": 0.2389779806137085, "sampling/sampling_logp_difference/mean": 0.021991441026329994, "step": 7550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 187.8375, "completions/mean_terminated_length": 187.8375, "completions/min_length": 111.8, "completions/min_terminated_length": 111.8, "entropy": 1.0516787521541118, "epoch": 0.6950445895007815, "frac_reward_zero_std": 0.3, "grad_norm": 6.809999942779541, "kl": 0.012189005134860053, "learning_rate": 4.562390504053049e-06, "loss": 0.001, "num_tokens": 32949061.0, "reward": 0.0, "reward_std": 0.6371305644512176, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6371305763721467, "sampling/importance_sampling_ratio/max": 1.2557111740112306, "sampling/importance_sampling_ratio/mean": 0.9995312690734863, "sampling/importance_sampling_ratio/min": 0.794962352514267, "sampling/sampling_logp_difference/max": 0.24627496004104615, "sampling/sampling_logp_difference/mean": 0.020747230760753153, "step": 7560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.6, "completions/max_terminated_length": 295.6, "completions/mean_length": 209.2625, "completions/mean_terminated_length": 209.2625, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.1205576620995998, "epoch": 0.6959639606509148, "frac_reward_zero_std": 0.5, "grad_norm": 0.11271718144416809, "kl": 0.013041552173672245, "learning_rate": 4.560957449244675e-06, "loss": 0.001, "num_tokens": 32993050.0, "reward": 0.025, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2421066284179687, "sampling/importance_sampling_ratio/mean": 0.999755609035492, "sampling/importance_sampling_ratio/min": 0.766630619764328, "sampling/sampling_logp_difference/max": 0.2716700792312622, "sampling/sampling_logp_difference/mean": 0.021048104017972948, "step": 7570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 290.8, "completions/max_terminated_length": 276.6, "completions/mean_length": 206.4625, "completions/mean_terminated_length": 203.7125, "completions/min_length": 150.1, "completions/min_terminated_length": 150.1, "entropy": 1.078347396850586, "epoch": 0.6968833318010481, "frac_reward_zero_std": 0.2, "grad_norm": 4.5677995681762695, "kl": 0.013976041867863387, "learning_rate": 4.5595222776677935e-06, "loss": 0.0011, "num_tokens": 33033815.0, "reward": 0.16875, "reward_std": 0.7275521874427795, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.727552205324173, "sampling/importance_sampling_ratio/max": 1.2469866633415223, "sampling/importance_sampling_ratio/mean": 1.0002372980117797, "sampling/importance_sampling_ratio/min": 0.7966596364974976, "sampling/sampling_logp_difference/max": 0.2469363331794739, "sampling/sampling_logp_difference/mean": 0.021382903307676317, "step": 7580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.5, "completions/max_terminated_length": 280.5, "completions/mean_length": 209.4, "completions/mean_terminated_length": 209.4, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.1299501053988934, "epoch": 0.6978027029511814, "frac_reward_zero_std": 0.3, "grad_norm": 5.996198654174805, "kl": 0.013944997842190787, "learning_rate": 4.5580849907964435e-06, "loss": 0.0008, "num_tokens": 33074767.0, "reward": 0.025, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2502052187919617, "sampling/importance_sampling_ratio/mean": 0.9996806681156158, "sampling/importance_sampling_ratio/min": 0.7479320943355561, "sampling/sampling_logp_difference/max": 0.3103110074996948, "sampling/sampling_logp_difference/mean": 0.021967416629195213, "step": 7590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 204.9375, "completions/mean_terminated_length": 204.9375, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0872042320668698, "epoch": 0.6987220741013147, "frac_reward_zero_std": 0.1, "grad_norm": 6.660717010498047, "kl": 0.01419613611069508, "learning_rate": 4.5566455901068305e-06, "loss": 0.0012, "num_tokens": 33117762.0, "reward": 0.3, "reward_std": 0.792874401807785, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.792874413728714, "sampling/importance_sampling_ratio/max": 1.2487792253494263, "sampling/importance_sampling_ratio/mean": 0.9997764766216278, "sampling/importance_sampling_ratio/min": 0.7633844316005707, "sampling/sampling_logp_difference/max": 0.2762184262275696, "sampling/sampling_logp_difference/mean": 0.021015827544033527, "step": 7600 }, { "epoch": 0.6987220741013147, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0006544502617801048, "eval_completions/max_length": 305.3176265270506, "eval_completions/max_terminated_length": 304.7678883071553, "eval_completions/mean_length": 216.21815008726003, "eval_completions/mean_terminated_length": 216.0781289944474, "eval_completions/min_length": 143.57940663176265, "eval_completions/min_terminated_length": 143.57940663176265, "eval_entropy": 1.1051548468192836, "eval_frac_reward_zero_std": 0.4572425828970332, "eval_kl": 0.01286670464671251, "eval_loss": 0.0006287605501711369, "eval_num_tokens": 33117762.0, "eval_reward": -0.1143106457242583, "eval_reward_std": 0.47488945072442035, "eval_rewards/reward_fn/mean": -0.1143106457242583, "eval_rewards/reward_fn/std": 0.47488946507946567, "eval_runtime": 1773.3998, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.263111858051693, "eval_sampling/importance_sampling_ratio/mean": 1.0000141171468697, "eval_sampling/importance_sampling_ratio/min": 0.7812600030220824, "eval_sampling/sampling_logp_difference/max": 0.2729007971016316, "eval_sampling/sampling_logp_difference/mean": 0.020776947724582533, "eval_steps_per_second": 0.041, "step": 7600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.1, "completions/max_terminated_length": 300.1, "completions/mean_length": 215.0125, "completions/mean_terminated_length": 215.0125, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.1416768856346606, "epoch": 0.699641445251448, "frac_reward_zero_std": 0.7, "grad_norm": 0.14178860187530518, "kl": 0.014247893361607566, "learning_rate": 4.555204077077338e-06, "loss": 0.0005, "num_tokens": 33161579.0, "reward": -0.85625, "reward_std": 0.19189817756414412, "rewards/reward_fn/mean": -0.85625, "rewards/reward_fn/std": 0.1918981894850731, "sampling/importance_sampling_ratio/max": 1.2699844598770142, "sampling/importance_sampling_ratio/mean": 0.9996096014976501, "sampling/importance_sampling_ratio/min": 0.7865210592746734, "sampling/sampling_logp_difference/max": 0.26673702597618104, "sampling/sampling_logp_difference/mean": 0.02177044879645109, "step": 7610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.8, "completions/max_terminated_length": 296.8, "completions/mean_length": 208.9875, "completions/mean_terminated_length": 208.9875, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.1147437863051892, "epoch": 0.7005608164015813, "frac_reward_zero_std": 0.5, "grad_norm": 0.3334629535675049, "kl": 0.013073995558079332, "learning_rate": 4.553760453188513e-06, "loss": 0.001, "num_tokens": 33204906.0, "reward": -0.025, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.2418126940727234, "sampling/importance_sampling_ratio/mean": 0.9992565751075745, "sampling/importance_sampling_ratio/min": 0.7509912192821503, "sampling/sampling_logp_difference/max": 0.3009630560874939, "sampling/sampling_logp_difference/mean": 0.02142366785556078, "step": 7620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.9, "completions/max_terminated_length": 295.9, "completions/mean_length": 200.9625, "completions/mean_terminated_length": 200.9625, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 1.0487634927034377, "epoch": 0.7014801875517146, "frac_reward_zero_std": 0.6, "grad_norm": 6.008181571960449, "kl": 0.013615646766265855, "learning_rate": 4.552314719923073e-06, "loss": 0.0006, "num_tokens": 33247271.0, "reward": -0.225, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2672451615333558, "sampling/importance_sampling_ratio/mean": 1.0001518905162812, "sampling/importance_sampling_ratio/min": 0.7706095337867737, "sampling/sampling_logp_difference/max": 0.2795531749725342, "sampling/sampling_logp_difference/mean": 0.021493326313793658, "step": 7630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 329.4, "completions/max_terminated_length": 289.3, "completions/mean_length": 218.85, "completions/mean_terminated_length": 211.94821472167968, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0840099252760411, "epoch": 0.7023995587018479, "frac_reward_zero_std": 0.4, "grad_norm": 5.550228595733643, "kl": 0.011601889884332195, "learning_rate": 4.550866878765903e-06, "loss": 0.0005, "num_tokens": 33293187.0, "reward": -0.4125, "reward_std": 0.4820707067847252, "rewards/reward_fn/mean": -0.4125, "rewards/reward_fn/std": 0.48207071870565416, "sampling/importance_sampling_ratio/max": 1.2616230487823485, "sampling/importance_sampling_ratio/mean": 1.0001303613185883, "sampling/importance_sampling_ratio/min": 0.7653930425643921, "sampling/sampling_logp_difference/max": 0.29657193422317507, "sampling/sampling_logp_difference/mean": 0.020509148947894573, "step": 7640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.5, "completions/max_terminated_length": 331.5, "completions/mean_length": 226.475, "completions/mean_terminated_length": 226.475, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.054324644804001, "epoch": 0.7033189298519812, "frac_reward_zero_std": 0.7, "grad_norm": 0.09934248775243759, "kl": 0.012379286414943635, "learning_rate": 4.549416931204051e-06, "loss": 0.0008, "num_tokens": 33338049.0, "reward": 0.15, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.2734720349311828, "sampling/importance_sampling_ratio/mean": 1.0001199543476105, "sampling/importance_sampling_ratio/min": 0.7779594123363495, "sampling/sampling_logp_difference/max": 0.2846710681915283, "sampling/sampling_logp_difference/mean": 0.020255954191088676, "step": 7650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.6, "completions/max_terminated_length": 273.6, "completions/mean_length": 188.3625, "completions/mean_terminated_length": 188.3625, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.1072974406182765, "epoch": 0.7042383010021146, "frac_reward_zero_std": 0.3, "grad_norm": 0.13758757710456848, "kl": 0.01229650130844675, "learning_rate": 4.547964878726729e-06, "loss": 0.0007, "num_tokens": 33378710.0, "reward": 0.01875, "reward_std": 0.581199049949646, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.581199049949646, "sampling/importance_sampling_ratio/max": 1.2964622020721435, "sampling/importance_sampling_ratio/mean": 0.9996895253658294, "sampling/importance_sampling_ratio/min": 0.8076421320438385, "sampling/sampling_logp_difference/max": 0.2655660629272461, "sampling/sampling_logp_difference/mean": 0.0213504608720541, "step": 7660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.5, "completions/max_terminated_length": 259.5, "completions/mean_length": 190.9, "completions/mean_terminated_length": 190.9, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 1.1463838160037993, "epoch": 0.7051576721522479, "frac_reward_zero_std": 0.2, "grad_norm": 8.830795288085938, "kl": 0.011723669565981254, "learning_rate": 4.5465107228253135e-06, "loss": 0.0006, "num_tokens": 33418814.0, "reward": -0.225, "reward_std": 0.6346783876419068, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6346783995628357, "sampling/importance_sampling_ratio/max": 1.2632263898849487, "sampling/importance_sampling_ratio/mean": 1.0002202808856964, "sampling/importance_sampling_ratio/min": 0.7903464198112488, "sampling/sampling_logp_difference/max": 0.2711119890213013, "sampling/sampling_logp_difference/mean": 0.02228588666766882, "step": 7670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.2, "completions/max_terminated_length": 280.2, "completions/mean_length": 208.8, "completions/mean_terminated_length": 208.8, "completions/min_length": 146.2, "completions/min_terminated_length": 146.2, "entropy": 0.9905225478112698, "epoch": 0.7060770433023812, "frac_reward_zero_std": 0.6, "grad_norm": 4.655591011047363, "kl": 0.012903569760965183, "learning_rate": 4.5450544649933375e-06, "loss": 0.0004, "num_tokens": 33459486.0, "reward": -0.25625, "reward_std": 0.26600351780653, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.26600351780653, "sampling/importance_sampling_ratio/max": 1.253794264793396, "sampling/importance_sampling_ratio/mean": 1.0000717043876648, "sampling/importance_sampling_ratio/min": 0.7908954739570617, "sampling/sampling_logp_difference/max": 0.2523775637149811, "sampling/sampling_logp_difference/mean": 0.019776570424437524, "step": 7680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 211.0875, "completions/mean_terminated_length": 211.0875, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.0969453290104867, "epoch": 0.7069964144525145, "frac_reward_zero_std": 0.4, "grad_norm": 6.8091959953308105, "kl": 0.012727080803597347, "learning_rate": 4.543596106726494e-06, "loss": 0.0003, "num_tokens": 33500965.0, "reward": -0.275, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.2468703150749207, "sampling/importance_sampling_ratio/mean": 0.9995214521884919, "sampling/importance_sampling_ratio/min": 0.7868207037448883, "sampling/sampling_logp_difference/max": 0.2557219326496124, "sampling/sampling_logp_difference/mean": 0.021707679145038128, "step": 7690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.4, "completions/max_terminated_length": 287.4, "completions/mean_length": 188.7875, "completions/mean_terminated_length": 188.7875, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.0049482021480798, "epoch": 0.7079157856026478, "frac_reward_zero_std": 0.6, "grad_norm": 5.610608100891113, "kl": 0.01211997150676325, "learning_rate": 4.5421356495226356e-06, "loss": 0.0009, "num_tokens": 33543180.0, "reward": -0.65, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2209412813186646, "sampling/importance_sampling_ratio/mean": 0.9997904896736145, "sampling/importance_sampling_ratio/min": 0.7460877299308777, "sampling/sampling_logp_difference/max": 0.30244273543357847, "sampling/sampling_logp_difference/mean": 0.02045807773247361, "step": 7700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.6, "completions/max_terminated_length": 292.6, "completions/mean_length": 207.8125, "completions/mean_terminated_length": 207.8125, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.088780291378498, "epoch": 0.7088351567527811, "frac_reward_zero_std": 0.3, "grad_norm": 5.990338325500488, "kl": 0.01210208511329256, "learning_rate": 4.540673094881768e-06, "loss": 0.0008, "num_tokens": 33585629.0, "reward": -0.25, "reward_std": 0.5496452271938324, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5496452391147614, "sampling/importance_sampling_ratio/max": 1.3122787714004516, "sampling/importance_sampling_ratio/mean": 0.9993920803070069, "sampling/importance_sampling_ratio/min": 0.7662236034870148, "sampling/sampling_logp_difference/max": 0.3335108458995819, "sampling/sampling_logp_difference/mean": 0.021526631712913514, "step": 7710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.0, "completions/max_terminated_length": 259.0, "completions/mean_length": 198.8625, "completions/mean_terminated_length": 198.8625, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.0855985805392265, "epoch": 0.7097545279029144, "frac_reward_zero_std": 0.5, "grad_norm": 6.699244976043701, "kl": 0.011983204825082795, "learning_rate": 4.539208444306051e-06, "loss": 0.0006, "num_tokens": 33629330.0, "reward": -0.325, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.251526403427124, "sampling/importance_sampling_ratio/mean": 0.9998815178871154, "sampling/importance_sampling_ratio/min": 0.8052299380302429, "sampling/sampling_logp_difference/max": 0.2408873677253723, "sampling/sampling_logp_difference/mean": 0.021098622493445872, "step": 7720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.6, "completions/max_terminated_length": 323.6, "completions/mean_length": 235.6, "completions/mean_terminated_length": 235.6, "completions/min_length": 151.1, "completions/min_terminated_length": 151.1, "entropy": 1.1324720583856105, "epoch": 0.7106738990530477, "frac_reward_zero_std": 0.5, "grad_norm": 6.484548091888428, "kl": 0.011749640654306859, "learning_rate": 4.5377416992998e-06, "loss": 0.0005, "num_tokens": 33675474.0, "reward": -0.175, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.335338544845581, "sampling/importance_sampling_ratio/mean": 0.9999153852462769, "sampling/importance_sampling_ratio/min": 0.7652119576931, "sampling/sampling_logp_difference/max": 0.3390638530254364, "sampling/sampling_logp_difference/mean": 0.02156429346650839, "step": 7730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 217.0125, "completions/mean_terminated_length": 217.0125, "completions/min_length": 152.4, "completions/min_terminated_length": 152.4, "entropy": 1.046103696525097, "epoch": 0.711593270203181, "frac_reward_zero_std": 0.7, "grad_norm": 0.09440566599369049, "kl": 0.01196893872693181, "learning_rate": 4.536272861369478e-06, "loss": 0.0007, "num_tokens": 33719091.0, "reward": -0.55625, "reward_std": 0.2516810357570648, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.2516810417175293, "sampling/importance_sampling_ratio/max": 1.306090271472931, "sampling/importance_sampling_ratio/mean": 1.000227689743042, "sampling/importance_sampling_ratio/min": 0.7947349190711975, "sampling/sampling_logp_difference/max": 0.2864459276199341, "sampling/sampling_logp_difference/mean": 0.020243253372609616, "step": 7740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.9, "completions/max_terminated_length": 300.9, "completions/mean_length": 203.9, "completions/mean_terminated_length": 203.9, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.122155174612999, "epoch": 0.7125126413533144, "frac_reward_zero_std": 0.7, "grad_norm": 0.09372245520353317, "kl": 0.013970675220480188, "learning_rate": 4.534801932023701e-06, "loss": 0.0004, "num_tokens": 33762459.0, "reward": 0.125, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.2786078691482543, "sampling/importance_sampling_ratio/mean": 0.9998780906200408, "sampling/importance_sampling_ratio/min": 0.7720388472080231, "sampling/sampling_logp_difference/max": 0.29762768745422363, "sampling/sampling_logp_difference/mean": 0.021775362826883794, "step": 7750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.1, "completions/max_terminated_length": 310.1, "completions/mean_length": 212.1375, "completions/mean_terminated_length": 212.1375, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.0408035792410373, "epoch": 0.7134320125034477, "frac_reward_zero_std": 0.4, "grad_norm": 0.09632840752601624, "kl": 0.011703932122327388, "learning_rate": 4.53332891277323e-06, "loss": 0.0008, "num_tokens": 33805150.0, "reward": 0.05, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.2709550380706787, "sampling/importance_sampling_ratio/mean": 1.0001371920108795, "sampling/importance_sampling_ratio/min": 0.8069843471050262, "sampling/sampling_logp_difference/max": 0.24740784168243407, "sampling/sampling_logp_difference/mean": 0.02107594721019268, "step": 7760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.2, "completions/max_terminated_length": 311.2, "completions/mean_length": 233.625, "completions/mean_terminated_length": 233.625, "completions/min_length": 150.8, "completions/min_terminated_length": 150.8, "entropy": 1.1412988848984242, "epoch": 0.714351383653581, "frac_reward_zero_std": 0.4, "grad_norm": 0.11383591592311859, "kl": 0.013404991137213074, "learning_rate": 4.531853805130975e-06, "loss": 0.0012, "num_tokens": 33852376.0, "reward": -0.15, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2483739614486695, "sampling/importance_sampling_ratio/mean": 0.9999114096164703, "sampling/importance_sampling_ratio/min": 0.7602717101573944, "sampling/sampling_logp_difference/max": 0.2867527425289154, "sampling/sampling_logp_difference/mean": 0.022138280048966407, "step": 7770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.3, "completions/max_terminated_length": 264.3, "completions/mean_length": 177.5375, "completions/mean_terminated_length": 177.5375, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 1.0889952294528484, "epoch": 0.7152707548037143, "frac_reward_zero_std": 0.7, "grad_norm": 0.09376151114702225, "kl": 0.011932455166243017, "learning_rate": 4.530376610611988e-06, "loss": 0.0005, "num_tokens": 33891619.0, "reward": 0.15, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2372679948806762, "sampling/importance_sampling_ratio/mean": 0.9993834137916565, "sampling/importance_sampling_ratio/min": 0.7819735050201416, "sampling/sampling_logp_difference/max": 0.2545257806777954, "sampling/sampling_logp_difference/mean": 0.020981532521545887, "step": 7780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.6, "completions/max_terminated_length": 317.6, "completions/mean_length": 213.3375, "completions/mean_terminated_length": 213.3375, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 0.9978972434997558, "epoch": 0.7161901259538476, "frac_reward_zero_std": 0.3, "grad_norm": 5.2302937507629395, "kl": 0.011727024300489575, "learning_rate": 4.528897330733467e-06, "loss": 0.0005, "num_tokens": 33935758.0, "reward": 0.0875, "reward_std": 0.6745985448360443, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.6745985567569732, "sampling/importance_sampling_ratio/max": 1.2542707920074463, "sampling/importance_sampling_ratio/mean": 1.0004034161567688, "sampling/importance_sampling_ratio/min": 0.7876003444194793, "sampling/sampling_logp_difference/max": 0.2739355802536011, "sampling/sampling_logp_difference/mean": 0.019733317848294973, "step": 7790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 184.15, "completions/mean_terminated_length": 184.15, "completions/min_length": 103.4, "completions/min_terminated_length": 103.4, "entropy": 0.928548451513052, "epoch": 0.7171094971039809, "frac_reward_zero_std": 0.6, "grad_norm": 5.467036247253418, "kl": 0.010884142320719548, "learning_rate": 4.527415967014752e-06, "loss": 0.0007, "num_tokens": 33977450.0, "reward": -0.1, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2540866255760192, "sampling/importance_sampling_ratio/mean": 0.9995656192302704, "sampling/importance_sampling_ratio/min": 0.7846047282218933, "sampling/sampling_logp_difference/max": 0.26770131587982177, "sampling/sampling_logp_difference/mean": 0.019505896233022214, "step": 7800 }, { "epoch": 0.7171094971039809, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 288.80104712041884, "eval_completions/max_terminated_length": 288.02443280977315, "eval_completions/mean_length": 202.10558464223385, "eval_completions/mean_terminated_length": 201.8860634209598, "eval_completions/min_length": 128.15357766143106, "eval_completions/min_terminated_length": 128.15357766143106, "eval_entropy": 1.060717930552431, "eval_frac_reward_zero_std": 0.4799301919720768, "eval_kl": 0.012782559815443155, "eval_loss": 0.000684626866132021, "eval_num_tokens": 33977450.0, "eval_reward": -0.11834642233856893, "eval_reward_std": 0.45445252531069946, "eval_rewards/reward_fn/mean": -0.11834642233856893, "eval_rewards/reward_fn/std": 0.45445253945770064, "eval_runtime": 1689.0674, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.274384078138994, "eval_sampling/importance_sampling_ratio/mean": 1.0001233914967815, "eval_sampling/importance_sampling_ratio/min": 0.7766847202707128, "eval_sampling/sampling_logp_difference/max": 0.28354395899993173, "eval_sampling/sampling_logp_difference/mean": 0.02050270055653121, "eval_steps_per_second": 0.043, "step": 7800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.2, "completions/max_terminated_length": 323.2, "completions/mean_length": 225.025, "completions/mean_terminated_length": 225.025, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0898796930909156, "epoch": 0.7180288682541142, "frac_reward_zero_std": 0.3, "grad_norm": 5.084307670593262, "kl": 0.014839508716249838, "learning_rate": 4.525932520977322e-06, "loss": 0.0005, "num_tokens": 34026452.0, "reward": -0.45, "reward_std": 0.6261870503425598, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.6261870741844178, "sampling/importance_sampling_ratio/max": 1.3269975662231446, "sampling/importance_sampling_ratio/mean": 1.0001397669315337, "sampling/importance_sampling_ratio/min": 0.7209860354661941, "sampling/sampling_logp_difference/max": 0.39565067291259765, "sampling/sampling_logp_difference/mean": 0.020828712731599808, "step": 7810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 205.5625, "completions/mean_terminated_length": 205.5625, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0231329262256623, "epoch": 0.7189482394042475, "frac_reward_zero_std": 0.5, "grad_norm": 0.11433705687522888, "kl": 0.01152013125247322, "learning_rate": 4.524446994144795e-06, "loss": 0.0006, "num_tokens": 34070193.0, "reward": 0.225, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.256628441810608, "sampling/importance_sampling_ratio/mean": 1.0000002264976502, "sampling/importance_sampling_ratio/min": 0.7783495604991912, "sampling/sampling_logp_difference/max": 0.27004218101501465, "sampling/sampling_logp_difference/mean": 0.020354905910789967, "step": 7820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.8, "completions/max_terminated_length": 312.8, "completions/mean_length": 221.6375, "completions/mean_terminated_length": 221.6375, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.0051250889897347, "epoch": 0.7198676105543808, "frac_reward_zero_std": 0.3, "grad_norm": 5.0841875076293945, "kl": 0.012239983805920928, "learning_rate": 4.5229593880429245e-06, "loss": 0.0005, "num_tokens": 34115620.0, "reward": -0.05, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2590021729469298, "sampling/importance_sampling_ratio/mean": 1.0000342309474946, "sampling/importance_sampling_ratio/min": 0.7607444524765015, "sampling/sampling_logp_difference/max": 0.2901815176010132, "sampling/sampling_logp_difference/mean": 0.020146839693188666, "step": 7830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.5, "completions/max_terminated_length": 316.5, "completions/mean_length": 219.4, "completions/mean_terminated_length": 219.4, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.0954822234809398, "epoch": 0.7207869817045142, "frac_reward_zero_std": 0.2, "grad_norm": 7.15207576751709, "kl": 0.011783897370332853, "learning_rate": 4.521469704199603e-06, "loss": 0.0002, "num_tokens": 34159788.0, "reward": -0.45, "reward_std": 0.6750263929367065, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.6750264167785645, "sampling/importance_sampling_ratio/max": 1.2334399819374084, "sampling/importance_sampling_ratio/mean": 1.000223845243454, "sampling/importance_sampling_ratio/min": 0.7668735325336457, "sampling/sampling_logp_difference/max": 0.2830051302909851, "sampling/sampling_logp_difference/mean": 0.021048289723694324, "step": 7840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.9, "completions/max_terminated_length": 324.9, "completions/mean_length": 219.25, "completions/mean_terminated_length": 219.25, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 0.9555469121783972, "epoch": 0.7217063528546475, "frac_reward_zero_std": 0.3, "grad_norm": 4.324455261230469, "kl": 0.011664616456255317, "learning_rate": 4.519977944144854e-06, "loss": 0.0001, "num_tokens": 34202984.0, "reward": -0.18125, "reward_std": 0.610797768831253, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.610797780752182, "sampling/importance_sampling_ratio/max": 1.2321448922157288, "sampling/importance_sampling_ratio/mean": 0.9999436914920807, "sampling/importance_sampling_ratio/min": 0.7384652435779572, "sampling/sampling_logp_difference/max": 0.3111132323741913, "sampling/sampling_logp_difference/mean": 0.019147028028964997, "step": 7850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 209.3375, "completions/mean_terminated_length": 209.3375, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0670596078038215, "epoch": 0.7226257240047808, "frac_reward_zero_std": 0.6, "grad_norm": 0.8870770931243896, "kl": 0.014166604360798373, "learning_rate": 4.518484109410837e-06, "loss": 0.0005, "num_tokens": 34246363.0, "reward": -0.25, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.4114805936813355, "sampling/importance_sampling_ratio/mean": 0.9999588012695313, "sampling/importance_sampling_ratio/min": 0.7756626963615417, "sampling/sampling_logp_difference/max": 0.3913330852985382, "sampling/sampling_logp_difference/mean": 0.02083588931709528, "step": 7860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.8, "completions/max_terminated_length": 335.8, "completions/mean_length": 227.4375, "completions/mean_terminated_length": 227.4375, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.0857323482632637, "epoch": 0.723545095154914, "frac_reward_zero_std": 0.5, "grad_norm": 6.237811088562012, "kl": 0.013221461547072978, "learning_rate": 4.516988201531837e-06, "loss": 0.0007, "num_tokens": 34291998.0, "reward": -0.15, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2349757075309753, "sampling/importance_sampling_ratio/mean": 1.000060075521469, "sampling/importance_sampling_ratio/min": 0.8026494979858398, "sampling/sampling_logp_difference/max": 0.23344600200653076, "sampling/sampling_logp_difference/mean": 0.020623236149549484, "step": 7870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 365.3, "completions/max_terminated_length": 365.3, "completions/mean_length": 234.575, "completions/mean_terminated_length": 234.575, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.045611882954836, "epoch": 0.7244644663050473, "frac_reward_zero_std": 0.6, "grad_norm": 7.7251200675964355, "kl": 0.013196555461036041, "learning_rate": 4.515490222044273e-06, "loss": 0.0007, "num_tokens": 34339732.0, "reward": -0.2, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.3068642973899842, "sampling/importance_sampling_ratio/mean": 1.0002929985523223, "sampling/importance_sampling_ratio/min": 0.7944088220596314, "sampling/sampling_logp_difference/max": 0.2835546374320984, "sampling/sampling_logp_difference/mean": 0.020206162147223948, "step": 7880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.1, "completions/max_terminated_length": 306.0, "completions/mean_length": 229.6875, "completions/mean_terminated_length": 226.82142944335936, "completions/min_length": 156.2, "completions/min_terminated_length": 156.2, "entropy": 1.0366659320890903, "epoch": 0.7253838374551806, "frac_reward_zero_std": 0.4, "grad_norm": 0.1552855223417282, "kl": 0.013528899732045829, "learning_rate": 4.513990172486689e-06, "loss": 0.0006, "num_tokens": 34384459.0, "reward": -0.43125, "reward_std": 0.5751873195171356, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.575187349319458, "sampling/importance_sampling_ratio/max": 1.2917468905448914, "sampling/importance_sampling_ratio/mean": 1.0003337502479552, "sampling/importance_sampling_ratio/min": 0.7804825723171234, "sampling/sampling_logp_difference/max": 0.2790086567401886, "sampling/sampling_logp_difference/mean": 0.02050896305590868, "step": 7890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 217.95, "completions/mean_terminated_length": 217.95, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.1229249581694603, "epoch": 0.7263032086053139, "frac_reward_zero_std": 0.5, "grad_norm": 6.265895843505859, "kl": 0.012378497858298943, "learning_rate": 4.512488054399758e-06, "loss": 0.0006, "num_tokens": 34431127.0, "reward": 0.125, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.2822787642478943, "sampling/importance_sampling_ratio/mean": 1.00043585896492, "sampling/importance_sampling_ratio/min": 0.7877587497234344, "sampling/sampling_logp_difference/max": 0.26999974250793457, "sampling/sampling_logp_difference/mean": 0.02120590005069971, "step": 7900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.9, "completions/max_terminated_length": 304.6, "completions/mean_length": 211.3125, "completions/mean_terminated_length": 207.73214416503907, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.0197883643209935, "epoch": 0.7272225797554472, "frac_reward_zero_std": 0.3, "grad_norm": 5.88197660446167, "kl": 0.01234890622436069, "learning_rate": 4.510983869326273e-06, "loss": 0.0009, "num_tokens": 34472832.0, "reward": -0.10625, "reward_std": 0.6469915866851806, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6469915986061097, "sampling/importance_sampling_ratio/max": 1.3230201244354247, "sampling/importance_sampling_ratio/mean": 0.9999683499336243, "sampling/importance_sampling_ratio/min": 0.7960600912570953, "sampling/sampling_logp_difference/max": 0.27659912109375, "sampling/sampling_logp_difference/mean": 0.01991453468799591, "step": 7910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 204.9375, "completions/mean_terminated_length": 204.9375, "completions/min_length": 126.9, "completions/min_terminated_length": 126.9, "entropy": 1.050185988843441, "epoch": 0.7281419509055805, "frac_reward_zero_std": 0.6, "grad_norm": 6.053203105926514, "kl": 0.011961052514379844, "learning_rate": 4.509477618811156e-06, "loss": 0.0004, "num_tokens": 34515851.0, "reward": 0.05, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.2489317774772644, "sampling/importance_sampling_ratio/mean": 0.9995578408241272, "sampling/importance_sampling_ratio/min": 0.7616906225681305, "sampling/sampling_logp_difference/max": 0.2846506148576736, "sampling/sampling_logp_difference/mean": 0.020563923195004462, "step": 7920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 297.7, "completions/max_terminated_length": 282.1, "completions/mean_length": 198.9625, "completions/mean_terminated_length": 195.87142944335938, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0783646024763585, "epoch": 0.729061322055714, "frac_reward_zero_std": 0.5, "grad_norm": 6.644243240356445, "kl": 0.013218981976388022, "learning_rate": 4.507969304401443e-06, "loss": 0.0008, "num_tokens": 34559144.0, "reward": -0.08125, "reward_std": 0.5010819792747497, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5010820209980011, "sampling/importance_sampling_ratio/max": 1.2524287462234498, "sampling/importance_sampling_ratio/mean": 1.0001303851604462, "sampling/importance_sampling_ratio/min": 0.7765160977840424, "sampling/sampling_logp_difference/max": 0.27204169034957887, "sampling/sampling_logp_difference/mean": 0.021299427933990957, "step": 7930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.4, "completions/max_terminated_length": 296.2, "completions/mean_length": 223.7375, "completions/mean_terminated_length": 221.08214416503907, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0132794439792634, "epoch": 0.7299806932058472, "frac_reward_zero_std": 0.6, "grad_norm": 0.12792444229125977, "kl": 0.013522760837804525, "learning_rate": 4.506458927646297e-06, "loss": 0.0006, "num_tokens": 34601075.0, "reward": 0.19375, "reward_std": 0.3661186873912811, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.3661187171936035, "sampling/importance_sampling_ratio/max": 1.2627922773361206, "sampling/importance_sampling_ratio/mean": 1.0002568125724793, "sampling/importance_sampling_ratio/min": 0.7886207282543183, "sampling/sampling_logp_difference/max": 0.271436071395874, "sampling/sampling_logp_difference/mean": 0.01971248220652342, "step": 7940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.3, "completions/max_terminated_length": 330.3, "completions/mean_length": 228.3625, "completions/mean_terminated_length": 228.3625, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.158314074575901, "epoch": 0.7309000643559805, "frac_reward_zero_std": 0.3, "grad_norm": 0.1173718199133873, "kl": 0.01392504406394437, "learning_rate": 4.504946490096995e-06, "loss": 0.0008, "num_tokens": 34643688.0, "reward": -0.325, "reward_std": 0.6001615285873413, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6001615405082703, "sampling/importance_sampling_ratio/max": 1.29703848361969, "sampling/importance_sampling_ratio/mean": 0.9998954594135284, "sampling/importance_sampling_ratio/min": 0.7769752562046051, "sampling/sampling_logp_difference/max": 0.2815298795700073, "sampling/sampling_logp_difference/mean": 0.02231447771191597, "step": 7950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.0, "completions/max_terminated_length": 338.0, "completions/mean_length": 211.1, "completions/mean_terminated_length": 211.1, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0597498930990696, "epoch": 0.7318194355061138, "frac_reward_zero_std": 0.5, "grad_norm": 6.641622066497803, "kl": 0.01173171637346968, "learning_rate": 4.503431993306931e-06, "loss": 0.0007, "num_tokens": 34684888.0, "reward": 0.05, "reward_std": 0.4847656965255737, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.48476572036743165, "sampling/importance_sampling_ratio/max": 1.2551561355590821, "sampling/importance_sampling_ratio/mean": 1.0002061247825622, "sampling/importance_sampling_ratio/min": 0.8066501319408417, "sampling/sampling_logp_difference/max": 0.23911242485046386, "sampling/sampling_logp_difference/mean": 0.020517088659107684, "step": 7960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.8, "completions/max_terminated_length": 292.8, "completions/mean_length": 205.45, "completions/mean_terminated_length": 205.45, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.097807016223669, "epoch": 0.7327388066562471, "frac_reward_zero_std": 0.5, "grad_norm": 4.4960174560546875, "kl": 0.012536961917066946, "learning_rate": 4.5019154388316165e-06, "loss": 0.0002, "num_tokens": 34730060.0, "reward": -0.275, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2664496779441834, "sampling/importance_sampling_ratio/mean": 0.9999650239944458, "sampling/importance_sampling_ratio/min": 0.7524630963802338, "sampling/sampling_logp_difference/max": 0.29182469844818115, "sampling/sampling_logp_difference/mean": 0.021039102412760257, "step": 7970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.3, "completions/max_terminated_length": 322.3, "completions/mean_length": 211.8625, "completions/mean_terminated_length": 211.8625, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0527584604918956, "epoch": 0.7336581778063804, "frac_reward_zero_std": 0.7, "grad_norm": 0.10207977890968323, "kl": 0.014820568886352702, "learning_rate": 4.500396828228673e-06, "loss": 0.0008, "num_tokens": 34775265.0, "reward": 0.025, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.2773639917373658, "sampling/importance_sampling_ratio/mean": 1.000165992975235, "sampling/importance_sampling_ratio/min": 0.721134889125824, "sampling/sampling_logp_difference/max": 0.3686139941215515, "sampling/sampling_logp_difference/mean": 0.020936667919158936, "step": 7980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 211.3375, "completions/mean_terminated_length": 211.3375, "completions/min_length": 142.8, "completions/min_terminated_length": 142.8, "entropy": 1.0581104256212712, "epoch": 0.7345775489565137, "frac_reward_zero_std": 0.4, "grad_norm": 5.592179775238037, "kl": 0.01158688751165755, "learning_rate": 4.498876163057834e-06, "loss": 0.0005, "num_tokens": 34820028.0, "reward": 0.2, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2345170259475708, "sampling/importance_sampling_ratio/mean": 1.0000490605831147, "sampling/importance_sampling_ratio/min": 0.7912254631519318, "sampling/sampling_logp_difference/max": 0.24015414714813232, "sampling/sampling_logp_difference/mean": 0.020838825032114983, "step": 7990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.9, "completions/max_terminated_length": 310.9, "completions/mean_length": 216.8, "completions/mean_terminated_length": 216.8, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.073033381998539, "epoch": 0.735496920106647, "frac_reward_zero_std": 0.4, "grad_norm": 0.1064324602484703, "kl": 0.012321639846777543, "learning_rate": 4.497353444880946e-06, "loss": 0.0008, "num_tokens": 34861716.0, "reward": 0.26875, "reward_std": 0.5163200557231903, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.5163200855255127, "sampling/importance_sampling_ratio/max": 1.2902882933616637, "sampling/importance_sampling_ratio/mean": 1.0002009391784668, "sampling/importance_sampling_ratio/min": 0.7866641104221344, "sampling/sampling_logp_difference/max": 0.2805799961090088, "sampling/sampling_logp_difference/mean": 0.0208828404545784, "step": 8000 }, { "epoch": 0.735496920106647, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 307.020942408377, "eval_completions/max_terminated_length": 304.6038394415358, "eval_completions/mean_length": 211.89419720767887, "eval_completions/mean_terminated_length": 211.3929820168913, "eval_completions/min_length": 135.1937172774869, "eval_completions/min_terminated_length": 135.1937172774869, "eval_entropy": 1.0907467671817925, "eval_frac_reward_zero_std": 0.4851657940663176, "eval_kl": 0.012451450720780495, "eval_loss": 0.0006132197449915111, "eval_num_tokens": 34861716.0, "eval_reward": -0.13579842931937172, "eval_reward_std": 0.44761053398641615, "eval_rewards/reward_fn/mean": -0.13579842931937172, "eval_rewards/reward_fn/std": 0.44761054906961595, "eval_runtime": 1778.9002, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.2693412000180122, "eval_sampling/importance_sampling_ratio/mean": 0.9999428770096931, "eval_sampling/importance_sampling_ratio/min": 0.7730308804940594, "eval_sampling/sampling_logp_difference/max": 0.28729757474147005, "eval_sampling/sampling_logp_difference/mean": 0.02101208265857807, "eval_steps_per_second": 0.04, "step": 8000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 341.5, "completions/max_terminated_length": 313.6, "completions/mean_length": 204.975, "completions/mean_terminated_length": 200.86964416503906, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0978618092834949, "epoch": 0.7364162912567803, "frac_reward_zero_std": 0.6, "grad_norm": 0.11246131360530853, "kl": 0.01326041083666496, "learning_rate": 4.495828675261964e-06, "loss": 0.0008, "num_tokens": 34907186.0, "reward": -0.23125, "reward_std": 0.39431732296943667, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.3943173348903656, "sampling/importance_sampling_ratio/max": 1.2905421733856202, "sampling/importance_sampling_ratio/mean": 1.0001361787319183, "sampling/importance_sampling_ratio/min": 0.7811485946178436, "sampling/sampling_logp_difference/max": 0.2857553958892822, "sampling/sampling_logp_difference/mean": 0.02123757228255272, "step": 8010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.6, "completions/max_terminated_length": 282.6, "completions/mean_length": 206.2375, "completions/mean_terminated_length": 206.2375, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1210158534348011, "epoch": 0.7373356624069136, "frac_reward_zero_std": 0.5, "grad_norm": 5.241743087768555, "kl": 0.014208092662738637, "learning_rate": 4.494301855766947e-06, "loss": 0.0012, "num_tokens": 34948845.0, "reward": -0.325, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2946329832077026, "sampling/importance_sampling_ratio/mean": 1.0002695739269256, "sampling/importance_sampling_ratio/min": 0.775946170091629, "sampling/sampling_logp_difference/max": 0.2878941476345062, "sampling/sampling_logp_difference/mean": 0.021890595182776452, "step": 8020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 345.6, "completions/max_terminated_length": 345.6, "completions/mean_length": 225.775, "completions/mean_terminated_length": 225.775, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 1.08005213662982, "epoch": 0.738255033557047, "frac_reward_zero_std": 0.2, "grad_norm": 5.533916473388672, "kl": 0.013351417589001357, "learning_rate": 4.492772987964061e-06, "loss": 0.0004, "num_tokens": 34990931.0, "reward": -0.225, "reward_std": 0.7440193653106689, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.7440194010734558, "sampling/importance_sampling_ratio/max": 1.2726633071899414, "sampling/importance_sampling_ratio/mean": 1.0000362277030945, "sampling/importance_sampling_ratio/min": 0.7956205844879151, "sampling/sampling_logp_difference/max": 0.2488576889038086, "sampling/sampling_logp_difference/mean": 0.021109837293624877, "step": 8030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.5, "completions/max_terminated_length": 279.5, "completions/mean_length": 203.5875, "completions/mean_terminated_length": 203.5875, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.1266114212572576, "epoch": 0.7391744047071803, "frac_reward_zero_std": 0.2, "grad_norm": 8.678178787231445, "kl": 0.013378884154371917, "learning_rate": 4.491242073423575e-06, "loss": 0.0014, "num_tokens": 35033858.0, "reward": -0.50625, "reward_std": 0.6220090791583062, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.6220090791583062, "sampling/importance_sampling_ratio/max": 1.2636826992034913, "sampling/importance_sampling_ratio/mean": 1.000044310092926, "sampling/importance_sampling_ratio/min": 0.7707274973392486, "sampling/sampling_logp_difference/max": 0.27308831214904783, "sampling/sampling_logp_difference/mean": 0.021772509440779686, "step": 8040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.3, "completions/max_terminated_length": 297.3, "completions/mean_length": 201.225, "completions/mean_terminated_length": 201.225, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0412396281957625, "epoch": 0.7400937758573136, "frac_reward_zero_std": 0.6, "grad_norm": 0.14768904447555542, "kl": 0.013401343242730946, "learning_rate": 4.489709113717864e-06, "loss": 0.0006, "num_tokens": 35076124.0, "reward": -0.7, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.7, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.3199504137039184, "sampling/importance_sampling_ratio/mean": 0.9999884128570556, "sampling/importance_sampling_ratio/min": 0.7614287555217742, "sampling/sampling_logp_difference/max": 0.3609073609113693, "sampling/sampling_logp_difference/mean": 0.02137789875268936, "step": 8050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 221.2, "completions/mean_terminated_length": 221.2, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.0861206188797952, "epoch": 0.7410131470074469, "frac_reward_zero_std": 0.2, "grad_norm": 5.208590507507324, "kl": 0.012650728545850143, "learning_rate": 4.488174110421397e-06, "loss": 0.0007, "num_tokens": 35121908.0, "reward": -0.225, "reward_std": 0.7112202107906341, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.7112202346324921, "sampling/importance_sampling_ratio/max": 1.2454363226890564, "sampling/importance_sampling_ratio/mean": 1.000141018629074, "sampling/importance_sampling_ratio/min": 0.7919990599155426, "sampling/sampling_logp_difference/max": 0.23873124718666078, "sampling/sampling_logp_difference/mean": 0.0212739285081625, "step": 8060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 212.5, "completions/mean_terminated_length": 212.5, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.0745020352303982, "epoch": 0.7419325181575802, "frac_reward_zero_std": 0.5, "grad_norm": 0.1587846279144287, "kl": 0.012676413572626189, "learning_rate": 4.4866370651107475e-06, "loss": 0.0007, "num_tokens": 35165036.0, "reward": 0.1, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2403552412986756, "sampling/importance_sampling_ratio/mean": 0.9993901014328003, "sampling/importance_sampling_ratio/min": 0.7902234017848968, "sampling/sampling_logp_difference/max": 0.24351913034915923, "sampling/sampling_logp_difference/mean": 0.020469009689986706, "step": 8070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.8, "completions/max_terminated_length": 327.8, "completions/mean_length": 223.05, "completions/mean_terminated_length": 223.05, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.1282448165118695, "epoch": 0.7428518893077135, "frac_reward_zero_std": 0.3, "grad_norm": 6.438852787017822, "kl": 0.013418558664852753, "learning_rate": 4.485097979364583e-06, "loss": 0.0009, "num_tokens": 35212224.0, "reward": -0.375, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.2755277276039123, "sampling/importance_sampling_ratio/mean": 0.9997746229171753, "sampling/importance_sampling_ratio/min": 0.7706569194793701, "sampling/sampling_logp_difference/max": 0.2856948167085648, "sampling/sampling_logp_difference/mean": 0.02081215791404247, "step": 8080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 217.825, "completions/mean_terminated_length": 217.825, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.1162737056612968, "epoch": 0.7437712604578468, "frac_reward_zero_std": 0.7, "grad_norm": 0.7153237462043762, "kl": 0.013904567412100732, "learning_rate": 4.48355685476367e-06, "loss": 0.0007, "num_tokens": 35257226.0, "reward": -0.425, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2525946259498597, "sampling/importance_sampling_ratio/mean": 0.9995617985725402, "sampling/importance_sampling_ratio/min": 0.759384223818779, "sampling/sampling_logp_difference/max": 0.31262428760528566, "sampling/sampling_logp_difference/mean": 0.021446756832301617, "step": 8090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.9, "completions/max_terminated_length": 308.9, "completions/mean_length": 201.5625, "completions/mean_terminated_length": 201.5625, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0401764988899231, "epoch": 0.7446906316079801, "frac_reward_zero_std": 0.4, "grad_norm": 0.1352423131465912, "kl": 0.013288275199010968, "learning_rate": 4.482013692890864e-06, "loss": 0.0007, "num_tokens": 35297799.0, "reward": -0.1, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2909022450447083, "sampling/importance_sampling_ratio/mean": 1.0001994490623474, "sampling/importance_sampling_ratio/min": 0.7959495306015014, "sampling/sampling_logp_difference/max": 0.2701645255088806, "sampling/sampling_logp_difference/mean": 0.02112232055515051, "step": 8100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.4, "completions/max_terminated_length": 296.4, "completions/mean_length": 209.375, "completions/mean_terminated_length": 209.375, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0848962225019931, "epoch": 0.7456100027581134, "frac_reward_zero_std": 0.5, "grad_norm": 7.500487327575684, "kl": 0.012161192152416334, "learning_rate": 4.480468495331118e-06, "loss": 0.0007, "num_tokens": 35342709.0, "reward": -0.325, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.284670066833496, "sampling/importance_sampling_ratio/mean": 1.0006303787231445, "sampling/importance_sampling_ratio/min": 0.7802027285099029, "sampling/sampling_logp_difference/max": 0.28779600858688353, "sampling/sampling_logp_difference/mean": 0.021081316098570824, "step": 8110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.6, "completions/max_terminated_length": 339.6, "completions/mean_length": 242.9375, "completions/mean_terminated_length": 242.9375, "completions/min_length": 157.1, "completions/min_terminated_length": 157.1, "entropy": 1.1070052690804004, "epoch": 0.7465293739082468, "frac_reward_zero_std": 0.5, "grad_norm": 7.068181037902832, "kl": 0.014703968877438457, "learning_rate": 4.4789212636714745e-06, "loss": 0.0006, "num_tokens": 35390024.0, "reward": -0.45, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2717514157295227, "sampling/importance_sampling_ratio/mean": 1.00000302195549, "sampling/importance_sampling_ratio/min": 0.771361231803894, "sampling/sampling_logp_difference/max": 0.2900703907012939, "sampling/sampling_logp_difference/mean": 0.02123609557747841, "step": 8120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 309.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 216.05, "completions/mean_terminated_length": 209.84583435058593, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.0933148287236691, "epoch": 0.7474487450583801, "frac_reward_zero_std": 0.6, "grad_norm": 0.12545832991600037, "kl": 0.01344182957545854, "learning_rate": 4.477371999501063e-06, "loss": 0.0008, "num_tokens": 35436468.0, "reward": 0.1375, "reward_std": 0.3492541253566742, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.3492541253566742, "sampling/importance_sampling_ratio/max": 1.3586755275726319, "sampling/importance_sampling_ratio/mean": 0.9997727811336518, "sampling/importance_sampling_ratio/min": 0.7593908846378327, "sampling/sampling_logp_difference/max": 0.3233489155769348, "sampling/sampling_logp_difference/mean": 0.021504420414566995, "step": 8130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 363.7, "completions/max_terminated_length": 343.6, "completions/mean_length": 254.95, "completions/mean_terminated_length": 246.23428802490236, "completions/min_length": 155.1, "completions/min_terminated_length": 155.1, "entropy": 1.1016400553286076, "epoch": 0.7483681162085134, "frac_reward_zero_std": 0.3, "grad_norm": 5.211227893829346, "kl": 0.013478017353918403, "learning_rate": 4.475820704411102e-06, "loss": 0.0003, "num_tokens": 35483920.0, "reward": -0.3, "reward_std": 0.6155787825584411, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.6155787825584411, "sampling/importance_sampling_ratio/max": 1.2703124165534974, "sampling/importance_sampling_ratio/mean": 0.9998896539211273, "sampling/importance_sampling_ratio/min": 0.776882004737854, "sampling/sampling_logp_difference/max": 0.28458271026611326, "sampling/sampling_logp_difference/mean": 0.021040708012878894, "step": 8140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 222.0, "completions/mean_terminated_length": 222.0, "completions/min_length": 156.4, "completions/min_terminated_length": 156.4, "entropy": 0.9960038658231497, "epoch": 0.7492874873586467, "frac_reward_zero_std": 0.3, "grad_norm": 5.694696426391602, "kl": 0.01375974646070972, "learning_rate": 4.474267379994895e-06, "loss": 0.0004, "num_tokens": 35526664.0, "reward": -0.3, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.266595470905304, "sampling/importance_sampling_ratio/mean": 1.0004478096961975, "sampling/importance_sampling_ratio/min": 0.7943660140037536, "sampling/sampling_logp_difference/max": 0.24771275520324706, "sampling/sampling_logp_difference/mean": 0.020525914058089255, "step": 8150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.3, "completions/max_terminated_length": 286.3, "completions/mean_length": 199.9, "completions/mean_terminated_length": 199.9, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.0497367165982723, "epoch": 0.75020685850878, "frac_reward_zero_std": 0.3, "grad_norm": 6.058277130126953, "kl": 0.01288883998640813, "learning_rate": 4.4727120278478335e-06, "loss": 0.0004, "num_tokens": 35573584.0, "reward": 0.0, "reward_std": 0.6186538934707642, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6186538934707642, "sampling/importance_sampling_ratio/max": 1.2367583751678466, "sampling/importance_sampling_ratio/mean": 0.9998661935329437, "sampling/importance_sampling_ratio/min": 0.7526630640029908, "sampling/sampling_logp_difference/max": 0.30686196088790896, "sampling/sampling_logp_difference/mean": 0.02044058945029974, "step": 8160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 330.5, "completions/max_terminated_length": 328.1, "completions/mean_length": 221.325, "completions/mean_terminated_length": 218.7732147216797, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.1315288238227368, "epoch": 0.7511262296589133, "frac_reward_zero_std": 0.4, "grad_norm": 0.09331376850605011, "kl": 0.014294246485223993, "learning_rate": 4.471154649567386e-06, "loss": 0.0004, "num_tokens": 35620186.0, "reward": -0.23125, "reward_std": 0.5434817552566529, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5434817552566529, "sampling/importance_sampling_ratio/max": 1.3134258151054383, "sampling/importance_sampling_ratio/mean": 0.9996997892856598, "sampling/importance_sampling_ratio/min": 0.7802802622318268, "sampling/sampling_logp_difference/max": 0.282110595703125, "sampling/sampling_logp_difference/mean": 0.02147434987127781, "step": 8170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 210.1875, "completions/mean_terminated_length": 210.1875, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.1286060720682145, "epoch": 0.7520456008090466, "frac_reward_zero_std": 0.4, "grad_norm": 7.045644760131836, "kl": 0.013353185809683055, "learning_rate": 4.469595246753107e-06, "loss": 0.0008, "num_tokens": 35665401.0, "reward": -0.375, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.301651382446289, "sampling/importance_sampling_ratio/mean": 1.0002481043338776, "sampling/importance_sampling_ratio/min": 0.7833608150482178, "sampling/sampling_logp_difference/max": 0.289893114566803, "sampling/sampling_logp_difference/mean": 0.02151589747518301, "step": 8180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.3, "completions/max_terminated_length": 311.5, "completions/mean_length": 229.8625, "completions/mean_terminated_length": 227.8107147216797, "completions/min_length": 157.6, "completions/min_terminated_length": 157.6, "entropy": 1.1701050758361817, "epoch": 0.7529649719591799, "frac_reward_zero_std": 0.5, "grad_norm": 4.9442853927612305, "kl": 0.013474837585818022, "learning_rate": 4.4680338210066255e-06, "loss": 0.0005, "num_tokens": 35709030.0, "reward": -0.00625, "reward_std": 0.39138468354940414, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.3913846954703331, "sampling/importance_sampling_ratio/max": 1.2312663674354554, "sampling/importance_sampling_ratio/mean": 0.999917733669281, "sampling/importance_sampling_ratio/min": 0.7864776313304901, "sampling/sampling_logp_difference/max": 0.24761542677879333, "sampling/sampling_logp_difference/mean": 0.02141481190919876, "step": 8190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.1, "completions/max_terminated_length": 303.1, "completions/mean_length": 230.6875, "completions/mean_terminated_length": 230.6875, "completions/min_length": 159.3, "completions/min_terminated_length": 159.3, "entropy": 1.0959716208279133, "epoch": 0.7538843431093132, "frac_reward_zero_std": 0.4, "grad_norm": 0.12791407108306885, "kl": 0.014437156758503988, "learning_rate": 4.466470373931654e-06, "loss": 0.0009, "num_tokens": 35754005.0, "reward": -0.05, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.2515873193740845, "sampling/importance_sampling_ratio/mean": 0.9999491453170777, "sampling/importance_sampling_ratio/min": 0.7747252762317658, "sampling/sampling_logp_difference/max": 0.2596626400947571, "sampling/sampling_logp_difference/mean": 0.021005046926438807, "step": 8200 }, { "epoch": 0.7538843431093132, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 303.8411867364747, "eval_completions/max_terminated_length": 301.24083769633506, "eval_completions/mean_length": 210.22883944153577, "eval_completions/mean_terminated_length": 209.70989170041176, "eval_completions/min_length": 135.85514834205932, "eval_completions/min_terminated_length": 135.85514834205932, "eval_entropy": 1.1018926049298226, "eval_frac_reward_zero_std": 0.44851657940663175, "eval_kl": 0.012976040507798419, "eval_loss": 0.0006870377692393959, "eval_num_tokens": 35754005.0, "eval_reward": -0.1275087260034904, "eval_reward_std": 0.4894189638773184, "eval_rewards/reward_fn/mean": -0.1275087260034904, "eval_rewards/reward_fn/std": 0.48941897927258443, "eval_runtime": 1764.119, "eval_samples_per_second": 0.325, "eval_sampling/importance_sampling_ratio/max": 1.2728948911447175, "eval_sampling/importance_sampling_ratio/mean": 0.9999522922550821, "eval_sampling/importance_sampling_ratio/min": 0.7834606231507206, "eval_sampling/sampling_logp_difference/max": 0.27730211826207124, "eval_sampling/sampling_logp_difference/mean": 0.021037752685636076, "eval_steps_per_second": 0.041, "step": 8200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.1, "completions/max_terminated_length": 297.1, "completions/mean_length": 220.5375, "completions/mean_terminated_length": 220.5375, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.1590476416051387, "epoch": 0.7548037142594466, "frac_reward_zero_std": 0.5, "grad_norm": 0.09138042479753494, "kl": 0.01206546772737056, "learning_rate": 4.464904907133976e-06, "loss": 0.0006, "num_tokens": 35800560.0, "reward": -0.225, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.266919243335724, "sampling/importance_sampling_ratio/mean": 1.0000191748142242, "sampling/importance_sampling_ratio/min": 0.7899800419807435, "sampling/sampling_logp_difference/max": 0.2508130967617035, "sampling/sampling_logp_difference/mean": 0.02187879104167223, "step": 8210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 192.575, "completions/mean_terminated_length": 192.575, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 1.107027067989111, "epoch": 0.7557230854095799, "frac_reward_zero_std": 0.3, "grad_norm": 0.13801929354667664, "kl": 0.014159001404186711, "learning_rate": 4.463337422221452e-06, "loss": 0.0007, "num_tokens": 35842270.0, "reward": -0.30625, "reward_std": 0.5328060373663902, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5328060492873192, "sampling/importance_sampling_ratio/max": 1.2750577449798584, "sampling/importance_sampling_ratio/mean": 0.999759966135025, "sampling/importance_sampling_ratio/min": 0.7995847463607788, "sampling/sampling_logp_difference/max": 0.2634114921092987, "sampling/sampling_logp_difference/mean": 0.02129540964961052, "step": 8220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 351.0, "completions/max_terminated_length": 328.3, "completions/mean_length": 216.925, "completions/mean_terminated_length": 210.9607162475586, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.0797892071306705, "epoch": 0.7566424565597132, "frac_reward_zero_std": 0.3, "grad_norm": 0.1524190604686737, "kl": 0.012848711153492332, "learning_rate": 4.461767920804015e-06, "loss": 0.0007, "num_tokens": 35885856.0, "reward": -0.1875, "reward_std": 0.5339977249503136, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.533997754752636, "sampling/importance_sampling_ratio/max": 1.2448729991912841, "sampling/importance_sampling_ratio/mean": 1.0000406742095946, "sampling/importance_sampling_ratio/min": 0.7838064014911652, "sampling/sampling_logp_difference/max": 0.25891268253326416, "sampling/sampling_logp_difference/mean": 0.020775564201176166, "step": 8230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.9, "completions/max_terminated_length": 289.9, "completions/mean_length": 208.1125, "completions/mean_terminated_length": 208.1125, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0384016752243042, "epoch": 0.7575618277098465, "frac_reward_zero_std": 0.5, "grad_norm": 0.12386240065097809, "kl": 0.012377075391123071, "learning_rate": 4.460196404493669e-06, "loss": 0.0006, "num_tokens": 35931713.0, "reward": -0.45, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.2531754970550537, "sampling/importance_sampling_ratio/mean": 1.000171458721161, "sampling/importance_sampling_ratio/min": 0.7922884702682496, "sampling/sampling_logp_difference/max": 0.2473638892173767, "sampling/sampling_logp_difference/mean": 0.020273392274975776, "step": 8240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 214.4625, "completions/mean_terminated_length": 214.4625, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 1.0856870383024215, "epoch": 0.7584811988599798, "frac_reward_zero_std": 0.3, "grad_norm": 0.12115811556577682, "kl": 0.01335740071372129, "learning_rate": 4.458622874904486e-06, "loss": 0.0005, "num_tokens": 35974374.0, "reward": -0.4, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2737164616584777, "sampling/importance_sampling_ratio/mean": 1.0006861805915832, "sampling/importance_sampling_ratio/min": 0.7921424329280853, "sampling/sampling_logp_difference/max": 0.2653595328330994, "sampling/sampling_logp_difference/mean": 0.020486729219555855, "step": 8250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.4, "completions/max_terminated_length": 277.4, "completions/mean_length": 192.4875, "completions/mean_terminated_length": 192.4875, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.1889863289892673, "epoch": 0.7594005700101131, "frac_reward_zero_std": 0.3, "grad_norm": 5.169882297515869, "kl": 0.01589753308799118, "learning_rate": 4.457047333652607e-06, "loss": 0.0011, "num_tokens": 36018733.0, "reward": -0.2125, "reward_std": 0.5637092038989067, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.5637092277407646, "sampling/importance_sampling_ratio/max": 1.266532790660858, "sampling/importance_sampling_ratio/mean": 0.9998309910297394, "sampling/importance_sampling_ratio/min": 0.7885038197040558, "sampling/sampling_logp_difference/max": 0.2555086135864258, "sampling/sampling_logp_difference/mean": 0.02298225983977318, "step": 8260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.7, "completions/max_terminated_length": 300.7, "completions/mean_length": 200.0125, "completions/mean_terminated_length": 200.0125, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0733463123440743, "epoch": 0.7603199411602464, "frac_reward_zero_std": 0.5, "grad_norm": 1.6610380411148071, "kl": 0.01345902875182219, "learning_rate": 4.455469782356241e-06, "loss": 0.0005, "num_tokens": 36059662.0, "reward": -0.325, "reward_std": 0.48816035985946654, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4881603717803955, "sampling/importance_sampling_ratio/max": 1.269036638736725, "sampling/importance_sampling_ratio/mean": 1.0001213133335114, "sampling/importance_sampling_ratio/min": 0.7761441528797149, "sampling/sampling_logp_difference/max": 0.2875147104263306, "sampling/sampling_logp_difference/mean": 0.02063455358147621, "step": 8270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 197.6, "completions/mean_terminated_length": 197.6, "completions/min_length": 115.2, "completions/min_terminated_length": 115.2, "entropy": 0.9942729439586401, "epoch": 0.7612393123103797, "frac_reward_zero_std": 0.4, "grad_norm": 7.252109527587891, "kl": 0.01269307307084091, "learning_rate": 4.453890222635657e-06, "loss": 0.0008, "num_tokens": 36098174.0, "reward": 0.0, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.2649033665657043, "sampling/importance_sampling_ratio/mean": 1.0000628292560578, "sampling/importance_sampling_ratio/min": 0.8051225304603576, "sampling/sampling_logp_difference/max": 0.2438078761100769, "sampling/sampling_logp_difference/mean": 0.019889290630817413, "step": 8280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 353.7, "completions/max_terminated_length": 353.7, "completions/mean_length": 233.6875, "completions/mean_terminated_length": 233.6875, "completions/min_length": 162.5, "completions/min_terminated_length": 162.5, "entropy": 1.1719118051230908, "epoch": 0.762158683460513, "frac_reward_zero_std": 0.4, "grad_norm": 0.19336587190628052, "kl": 0.014362471119966358, "learning_rate": 4.452308656113191e-06, "loss": 0.0008, "num_tokens": 36145101.0, "reward": -0.40625, "reward_std": 0.5075400412082672, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.5075400710105896, "sampling/importance_sampling_ratio/max": 1.3165544748306275, "sampling/importance_sampling_ratio/mean": 0.9994021832942963, "sampling/importance_sampling_ratio/min": 0.792543351650238, "sampling/sampling_logp_difference/max": 0.29051942825317384, "sampling/sampling_logp_difference/mean": 0.022138492204248905, "step": 8290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.6, "completions/max_terminated_length": 283.6, "completions/mean_length": 202.325, "completions/mean_terminated_length": 202.325, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.0889384031295777, "epoch": 0.7630780546106464, "frac_reward_zero_std": 0.5, "grad_norm": 6.5470805168151855, "kl": 0.013930426089791581, "learning_rate": 4.45072508441324e-06, "loss": 0.0008, "num_tokens": 36187327.0, "reward": -0.175, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2623833894729615, "sampling/importance_sampling_ratio/mean": 0.999942147731781, "sampling/importance_sampling_ratio/min": 0.7754827082157135, "sampling/sampling_logp_difference/max": 0.2785060048103333, "sampling/sampling_logp_difference/mean": 0.021390562132000922, "step": 8300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 311.7, "completions/max_terminated_length": 292.6, "completions/mean_length": 219.425, "completions/mean_terminated_length": 212.475, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.202222003787756, "epoch": 0.7639974257607797, "frac_reward_zero_std": 0.6, "grad_norm": 0.10532724112272263, "kl": 0.013530123780947178, "learning_rate": 4.449139509162257e-06, "loss": 0.0012, "num_tokens": 36231569.0, "reward": 0.0125, "reward_std": 0.4424567699432373, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.44245678186416626, "sampling/importance_sampling_ratio/max": 1.2584846019744873, "sampling/importance_sampling_ratio/mean": 0.9997498393058777, "sampling/importance_sampling_ratio/min": 0.7534295797348023, "sampling/sampling_logp_difference/max": 0.3013087332248688, "sampling/sampling_logp_difference/mean": 0.022582073323428632, "step": 8310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 289.2, "completions/max_terminated_length": 275.4, "completions/mean_length": 208.65, "completions/mean_terminated_length": 201.08250122070314, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0480360381305218, "epoch": 0.764916796910913, "frac_reward_zero_std": 0.3, "grad_norm": 5.3447699546813965, "kl": 0.011797912931069732, "learning_rate": 4.447551931988755e-06, "loss": 0.0006, "num_tokens": 36272261.0, "reward": 0.05625, "reward_std": 0.6216289579868317, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.6216289699077606, "sampling/importance_sampling_ratio/max": 1.2879630327224731, "sampling/importance_sampling_ratio/mean": 0.9999438166618347, "sampling/importance_sampling_ratio/min": 0.7933026134967804, "sampling/sampling_logp_difference/max": 0.26983222365379333, "sampling/sampling_logp_difference/mean": 0.020880364626646043, "step": 8320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 216.6625, "completions/mean_terminated_length": 216.6625, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.1362594462931157, "epoch": 0.7658361680610463, "frac_reward_zero_std": 0.1, "grad_norm": 5.097079753875732, "kl": 0.012550245516467839, "learning_rate": 4.445962354523304e-06, "loss": 0.0004, "num_tokens": 36317018.0, "reward": 0.05, "reward_std": 0.7566648900508881, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.756664925813675, "sampling/importance_sampling_ratio/max": 1.2515528559684754, "sampling/importance_sampling_ratio/mean": 0.9999805688858032, "sampling/importance_sampling_ratio/min": 0.7737800180912018, "sampling/sampling_logp_difference/max": 0.2608341336250305, "sampling/sampling_logp_difference/mean": 0.02212666515260935, "step": 8330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.8, "completions/max_terminated_length": 291.8, "completions/mean_length": 200.05, "completions/mean_terminated_length": 200.05, "completions/min_length": 115.4, "completions/min_terminated_length": 115.4, "entropy": 1.0516941398382187, "epoch": 0.7667555392111796, "frac_reward_zero_std": 0.6, "grad_norm": 0.1549377739429474, "kl": 0.014296807261416689, "learning_rate": 4.4443707783985255e-06, "loss": 0.001, "num_tokens": 36358134.0, "reward": -0.025, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2868567824363708, "sampling/importance_sampling_ratio/mean": 1.0000163912773132, "sampling/importance_sampling_ratio/min": 0.7690602719783783, "sampling/sampling_logp_difference/max": 0.28216448426246643, "sampling/sampling_logp_difference/mean": 0.020814174041152, "step": 8340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 211.625, "completions/mean_terminated_length": 211.625, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 0.9756084911525249, "epoch": 0.7676749103613129, "frac_reward_zero_std": 0.6, "grad_norm": 0.11339589208364487, "kl": 0.011332496447721496, "learning_rate": 4.442777205249098e-06, "loss": 0.0005, "num_tokens": 36401776.0, "reward": -0.075, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2692540764808655, "sampling/importance_sampling_ratio/mean": 0.9996983349323273, "sampling/importance_sampling_ratio/min": 0.7470689982175827, "sampling/sampling_logp_difference/max": 0.3233334243297577, "sampling/sampling_logp_difference/mean": 0.019691317714750765, "step": 8350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.6, "completions/max_terminated_length": 304.6, "completions/mean_length": 213.325, "completions/mean_terminated_length": 213.325, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.065622616559267, "epoch": 0.7685942815114462, "frac_reward_zero_std": 0.3, "grad_norm": 0.0851113349199295, "kl": 0.012023266218602658, "learning_rate": 4.441181636711749e-06, "loss": 0.0003, "num_tokens": 36445034.0, "reward": 0.35, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.2335820317268371, "sampling/importance_sampling_ratio/mean": 0.9999973595142364, "sampling/importance_sampling_ratio/min": 0.7901502072811126, "sampling/sampling_logp_difference/max": 0.24760582447052001, "sampling/sampling_logp_difference/mean": 0.020483828522264956, "step": 8360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.7, "completions/max_terminated_length": 301.2, "completions/mean_length": 220.6625, "completions/mean_terminated_length": 217.63214416503905, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.088075990602374, "epoch": 0.7695136526615795, "frac_reward_zero_std": 0.5, "grad_norm": 5.829736709594727, "kl": 0.013579464197391644, "learning_rate": 4.439584074425253e-06, "loss": 0.0005, "num_tokens": 36489311.0, "reward": -0.33125, "reward_std": 0.41325601786375044, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.4132560297846794, "sampling/importance_sampling_ratio/max": 1.270370078086853, "sampling/importance_sampling_ratio/mean": 0.9993828713893891, "sampling/importance_sampling_ratio/min": 0.7897556006908417, "sampling/sampling_logp_difference/max": 0.27832276821136476, "sampling/sampling_logp_difference/mean": 0.0213587555103004, "step": 8370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 361.1, "completions/max_terminated_length": 341.4, "completions/mean_length": 233.0375, "completions/mean_terminated_length": 229.65178680419922, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.0560547532513738, "epoch": 0.7704330238117127, "frac_reward_zero_std": 0.5, "grad_norm": 2.233879566192627, "kl": 0.013418292670394294, "learning_rate": 4.437984520030437e-06, "loss": 0.0006, "num_tokens": 36533538.0, "reward": -0.45625, "reward_std": 0.30052037686109545, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.30052037686109545, "sampling/importance_sampling_ratio/max": 1.247363817691803, "sampling/importance_sampling_ratio/mean": 0.9998499274253845, "sampling/importance_sampling_ratio/min": 0.7610420703887939, "sampling/sampling_logp_difference/max": 0.2930307239294052, "sampling/sampling_logp_difference/mean": 0.02046879045665264, "step": 8380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.5, "completions/max_terminated_length": 298.5, "completions/mean_length": 212.7125, "completions/mean_terminated_length": 212.7125, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.1338731415569783, "epoch": 0.771352394961846, "frac_reward_zero_std": 0.4, "grad_norm": 6.843141078948975, "kl": 0.012913969720830209, "learning_rate": 4.43638297517017e-06, "loss": 0.0005, "num_tokens": 36577715.0, "reward": -0.1, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.2647264838218688, "sampling/importance_sampling_ratio/mean": 1.000447529554367, "sampling/importance_sampling_ratio/min": 0.8076965987682343, "sampling/sampling_logp_difference/max": 0.24773030281066893, "sampling/sampling_logp_difference/mean": 0.021816761791706087, "step": 8390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 328.9, "completions/max_terminated_length": 307.0, "completions/mean_length": 230.1875, "completions/mean_terminated_length": 226.60357208251952, "completions/min_length": 162.1, "completions/min_terminated_length": 162.1, "entropy": 1.0537135303020477, "epoch": 0.7722717661119795, "frac_reward_zero_std": 0.1, "grad_norm": 5.951216220855713, "kl": 0.01433980640140362, "learning_rate": 4.434779441489367e-06, "loss": 0.0006, "num_tokens": 36624586.0, "reward": -0.03125, "reward_std": 0.858778429031372, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.858778464794159, "sampling/importance_sampling_ratio/max": 1.2474345564842224, "sampling/importance_sampling_ratio/mean": 0.9997388660907746, "sampling/importance_sampling_ratio/min": 0.769368839263916, "sampling/sampling_logp_difference/max": 0.2690757095813751, "sampling/sampling_logp_difference/mean": 0.02051892802119255, "step": 8400 }, { "epoch": 0.7722717661119795, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 314.82722513089004, "eval_completions/max_terminated_length": 312.4171029668412, "eval_completions/mean_length": 220.45833333333334, "eval_completions/mean_terminated_length": 219.82626957918336, "eval_completions/min_length": 142.7539267015707, "eval_completions/min_terminated_length": 142.7539267015707, "eval_entropy": 1.0841898469816744, "eval_frac_reward_zero_std": 0.4712041884816754, "eval_kl": 0.013303948335518595, "eval_loss": 0.0006475884583778679, "eval_num_tokens": 36624586.0, "eval_reward": -0.10613001745200698, "eval_reward_std": 0.4630323057087304, "eval_rewards/reward_fn/mean": -0.10613001745200698, "eval_rewards/reward_fn/std": 0.46303231777529025, "eval_runtime": 1818.8531, "eval_samples_per_second": 0.315, "eval_sampling/importance_sampling_ratio/max": 1.2758885486795848, "eval_sampling/importance_sampling_ratio/mean": 0.9999544228350721, "eval_sampling/importance_sampling_ratio/min": 0.7801626067719118, "eval_sampling/sampling_logp_difference/max": 0.28227259699794843, "eval_sampling/sampling_logp_difference/mean": 0.02072641804463264, "eval_steps_per_second": 0.04, "step": 8400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 217.1375, "completions/mean_terminated_length": 217.1375, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.107875358313322, "epoch": 0.7731911372621127, "frac_reward_zero_std": 0.2, "grad_norm": 4.679193496704102, "kl": 0.013949061266612262, "learning_rate": 4.433173920634986e-06, "loss": 0.0006, "num_tokens": 36669685.0, "reward": -0.275, "reward_std": 0.6927435398101807, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6927435517311096, "sampling/importance_sampling_ratio/max": 1.306684422492981, "sampling/importance_sampling_ratio/mean": 1.0003615260124206, "sampling/importance_sampling_ratio/min": 0.7533694177865982, "sampling/sampling_logp_difference/max": 0.3286025643348694, "sampling/sampling_logp_difference/mean": 0.021115275099873543, "step": 8410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.1, "completions/max_terminated_length": 319.4, "completions/mean_length": 224.5125, "completions/mean_terminated_length": 221.63928833007813, "completions/min_length": 151.7, "completions/min_terminated_length": 151.7, "entropy": 1.050481027364731, "epoch": 0.774110508412246, "frac_reward_zero_std": 0.5, "grad_norm": 4.9894700050354, "kl": 0.012378092331346124, "learning_rate": 4.431566414256026e-06, "loss": 0.0007, "num_tokens": 36714726.0, "reward": 0.36875, "reward_std": 0.47753825783729553, "rewards/reward_fn/mean": 0.36875, "rewards/reward_fn/std": 0.47753825783729553, "sampling/importance_sampling_ratio/max": 1.3309808731079102, "sampling/importance_sampling_ratio/mean": 1.0002917230129242, "sampling/importance_sampling_ratio/min": 0.7614434897899628, "sampling/sampling_logp_difference/max": 0.3185242056846619, "sampling/sampling_logp_difference/mean": 0.020792209915816783, "step": 8420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.2, "completions/max_terminated_length": 311.2, "completions/mean_length": 218.4, "completions/mean_terminated_length": 218.4, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.2024700708687306, "epoch": 0.7750298795623793, "frac_reward_zero_std": 0.6, "grad_norm": 6.212398052215576, "kl": 0.012515843973960727, "learning_rate": 4.4299569240035245e-06, "loss": 0.0004, "num_tokens": 36763294.0, "reward": -0.25, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2816298604011536, "sampling/importance_sampling_ratio/mean": 1.0004276931285858, "sampling/importance_sampling_ratio/min": 0.7981801390647888, "sampling/sampling_logp_difference/max": 0.26148860454559325, "sampling/sampling_logp_difference/mean": 0.022285322099924086, "step": 8430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 211.775, "completions/mean_terminated_length": 211.775, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.1479266606271268, "epoch": 0.7759492507125126, "frac_reward_zero_std": 0.4, "grad_norm": 8.181745529174805, "kl": 0.014252031553769483, "learning_rate": 4.428345451530557e-06, "loss": 0.0012, "num_tokens": 36806732.0, "reward": -0.625, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2763408899307251, "sampling/importance_sampling_ratio/mean": 0.9999563574790955, "sampling/importance_sampling_ratio/min": 0.7623327851295472, "sampling/sampling_logp_difference/max": 0.2905093789100647, "sampling/sampling_logp_difference/mean": 0.0225266357883811, "step": 8440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 281.0, "completions/max_terminated_length": 254.7, "completions/mean_length": 181.4875, "completions/mean_terminated_length": 177.51071472167968, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 0.9758511655032635, "epoch": 0.7768686218626459, "frac_reward_zero_std": 0.4, "grad_norm": 0.15793122351169586, "kl": 0.014301901333965361, "learning_rate": 4.426731998492236e-06, "loss": 0.001, "num_tokens": 36843411.0, "reward": 0.0, "reward_std": 0.5294508516788483, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5294508635997772, "sampling/importance_sampling_ratio/max": 1.2838852405548096, "sampling/importance_sampling_ratio/mean": 1.0000818073749542, "sampling/importance_sampling_ratio/min": 0.7569388151168823, "sampling/sampling_logp_difference/max": 0.3225701868534088, "sampling/sampling_logp_difference/mean": 0.020531637780368327, "step": 8450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.4, "completions/max_terminated_length": 271.4, "completions/mean_length": 202.4375, "completions/mean_terminated_length": 202.4375, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 0.986922837048769, "epoch": 0.7777879930127792, "frac_reward_zero_std": 0.5, "grad_norm": 0.15830595791339874, "kl": 0.013422417547553777, "learning_rate": 4.425116566545706e-06, "loss": 0.001, "num_tokens": 36888198.0, "reward": -0.25, "reward_std": 0.49155502319335936, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.49155502319335936, "sampling/importance_sampling_ratio/max": 1.2821635723114013, "sampling/importance_sampling_ratio/mean": 0.9997081995010376, "sampling/importance_sampling_ratio/min": 0.7605595409870147, "sampling/sampling_logp_difference/max": 0.3147682845592499, "sampling/sampling_logp_difference/mean": 0.0196896780282259, "step": 8460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.3, "completions/max_terminated_length": 313.3, "completions/mean_length": 219.25, "completions/mean_terminated_length": 219.25, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0644735746085643, "epoch": 0.7787073641629125, "frac_reward_zero_std": 0.5, "grad_norm": 4.661945343017578, "kl": 0.01417288754018955, "learning_rate": 4.4234991573501426e-06, "loss": 0.0009, "num_tokens": 36934282.0, "reward": -0.1, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2603379607200622, "sampling/importance_sampling_ratio/mean": 0.9995934009552002, "sampling/importance_sampling_ratio/min": 0.7822761118412018, "sampling/sampling_logp_difference/max": 0.2563796520233154, "sampling/sampling_logp_difference/mean": 0.02120006363838911, "step": 8470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.3, "completions/max_terminated_length": 335.3, "completions/mean_length": 227.55, "completions/mean_terminated_length": 224.87857360839843, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.175598443299532, "epoch": 0.7796267353130458, "frac_reward_zero_std": 0.3, "grad_norm": 5.915812015533447, "kl": 0.01274326372658834, "learning_rate": 4.42187977256676e-06, "loss": 0.0007, "num_tokens": 36979134.0, "reward": -0.30625, "reward_std": 0.5328060373663902, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5328060492873192, "sampling/importance_sampling_ratio/max": 1.2796715140342712, "sampling/importance_sampling_ratio/mean": 1.0001187026500702, "sampling/importance_sampling_ratio/min": 0.8008165121078491, "sampling/sampling_logp_difference/max": 0.26383963227272034, "sampling/sampling_logp_difference/mean": 0.021772984601557254, "step": 8480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.3, "completions/max_terminated_length": 321.3, "completions/mean_length": 211.35, "completions/mean_terminated_length": 211.35, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.048290991783142, "epoch": 0.7805461064631792, "frac_reward_zero_std": 0.6, "grad_norm": 0.13205601274967194, "kl": 0.013687166449381038, "learning_rate": 4.420258413858792e-06, "loss": 0.0006, "num_tokens": 37019690.0, "reward": -0.15625, "reward_std": 0.35302736759185793, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.3530273973941803, "sampling/importance_sampling_ratio/max": 1.2521010279655456, "sampling/importance_sampling_ratio/mean": 1.00027037858963, "sampling/importance_sampling_ratio/min": 0.788463169336319, "sampling/sampling_logp_difference/max": 0.2549134254455566, "sampling/sampling_logp_difference/mean": 0.020574127696454524, "step": 8490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.4, "completions/max_terminated_length": 315.4, "completions/mean_length": 222.025, "completions/mean_terminated_length": 222.025, "completions/min_length": 146.7, "completions/min_terminated_length": 146.7, "entropy": 1.1249325320124626, "epoch": 0.7814654776133125, "frac_reward_zero_std": 0.5, "grad_norm": 5.61703634262085, "kl": 0.015344861848279833, "learning_rate": 4.418635082891506e-06, "loss": 0.001, "num_tokens": 37065460.0, "reward": -0.4, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.3527862191200257, "sampling/importance_sampling_ratio/mean": 1.0000836670398712, "sampling/importance_sampling_ratio/min": 0.7896876931190491, "sampling/sampling_logp_difference/max": 0.31323239803314207, "sampling/sampling_logp_difference/mean": 0.02136600259691477, "step": 8500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.6, "completions/max_terminated_length": 279.6, "completions/mean_length": 204.0, "completions/mean_terminated_length": 204.0, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0770075641572476, "epoch": 0.7823848487634458, "frac_reward_zero_std": 0.3, "grad_norm": 7.641733169555664, "kl": 0.013076091837137937, "learning_rate": 4.417009781332192e-06, "loss": 0.0007, "num_tokens": 37109700.0, "reward": 0.275, "reward_std": 0.6767033517360688, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.6767033755779266, "sampling/importance_sampling_ratio/max": 1.3715340137481689, "sampling/importance_sampling_ratio/mean": 1.000289762020111, "sampling/importance_sampling_ratio/min": 0.7539548575878143, "sampling/sampling_logp_difference/max": 0.34364392757415774, "sampling/sampling_logp_difference/mean": 0.021527023613452913, "step": 8510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.5, "completions/max_terminated_length": 310.5, "completions/mean_length": 210.7625, "completions/mean_terminated_length": 210.7625, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.0248918984085322, "epoch": 0.7833042199135791, "frac_reward_zero_std": 0.3, "grad_norm": 6.484497547149658, "kl": 0.01252762160729617, "learning_rate": 4.4153825108501665e-06, "loss": 0.0002, "num_tokens": 37149497.0, "reward": 0.35, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2544386625289916, "sampling/importance_sampling_ratio/mean": 1.0003900051116943, "sampling/importance_sampling_ratio/min": 0.7763875722885132, "sampling/sampling_logp_difference/max": 0.27275350987911223, "sampling/sampling_logp_difference/mean": 0.020262175239622593, "step": 8520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.6, "completions/max_terminated_length": 306.6, "completions/mean_length": 216.3125, "completions/mean_terminated_length": 216.3125, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.0473342902958394, "epoch": 0.7842235910637124, "frac_reward_zero_std": 0.2, "grad_norm": 6.405353546142578, "kl": 0.012380934425164015, "learning_rate": 4.4137532731167646e-06, "loss": 0.0007, "num_tokens": 37195914.0, "reward": 0.125, "reward_std": 0.722163724899292, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.722163736820221, "sampling/importance_sampling_ratio/max": 1.2695478081703186, "sampling/importance_sampling_ratio/mean": 0.9998631417751312, "sampling/importance_sampling_ratio/min": 0.8010401725769043, "sampling/sampling_logp_difference/max": 0.2532971382141113, "sampling/sampling_logp_difference/mean": 0.020455403067171574, "step": 8530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 309.7, "completions/max_terminated_length": 304.7, "completions/mean_length": 225.3375, "completions/mean_terminated_length": 220.25416870117186, "completions/min_length": 150.8, "completions/min_terminated_length": 150.8, "entropy": 1.0477460347115994, "epoch": 0.7851429622138457, "frac_reward_zero_std": 0.5, "grad_norm": 5.053669452667236, "kl": 0.01207971730036661, "learning_rate": 4.412122069805345e-06, "loss": 0.0009, "num_tokens": 37240797.0, "reward": -0.1125, "reward_std": 0.47635064721107484, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.4763506591320038, "sampling/importance_sampling_ratio/max": 1.2750996947288513, "sampling/importance_sampling_ratio/mean": 0.9998262286186218, "sampling/importance_sampling_ratio/min": 0.7781896710395813, "sampling/sampling_logp_difference/max": 0.295078432559967, "sampling/sampling_logp_difference/mean": 0.02079039067029953, "step": 8540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 216.0625, "completions/mean_terminated_length": 216.0625, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.0502664379775524, "epoch": 0.786062333363979, "frac_reward_zero_std": 0.4, "grad_norm": 0.09797542542219162, "kl": 0.012314386817160993, "learning_rate": 4.410488902591286e-06, "loss": 0.0, "num_tokens": 37285674.0, "reward": -0.5, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.2816360116004943, "sampling/importance_sampling_ratio/mean": 0.9998455047607422, "sampling/importance_sampling_ratio/min": 0.7757204294204711, "sampling/sampling_logp_difference/max": 0.2872185230255127, "sampling/sampling_logp_difference/mean": 0.02049608789384365, "step": 8550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.3, "completions/max_terminated_length": 314.3, "completions/mean_length": 201.6, "completions/mean_terminated_length": 201.6, "completions/min_length": 120.7, "completions/min_terminated_length": 120.7, "entropy": 1.0981248579919338, "epoch": 0.7869817045141123, "frac_reward_zero_std": 0.6, "grad_norm": 0.19411779940128326, "kl": 0.013684612110955641, "learning_rate": 4.408853773151977e-06, "loss": 0.0005, "num_tokens": 37326938.0, "reward": 0.175, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2825979709625244, "sampling/importance_sampling_ratio/mean": 1.0002241551876068, "sampling/importance_sampling_ratio/min": 0.7853127121925354, "sampling/sampling_logp_difference/max": 0.2776610732078552, "sampling/sampling_logp_difference/mean": 0.021080788783729078, "step": 8560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 204.3375, "completions/mean_terminated_length": 204.3375, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 1.073072849959135, "epoch": 0.7879010756642456, "frac_reward_zero_std": 0.3, "grad_norm": 4.65260124206543, "kl": 0.014246864267624915, "learning_rate": 4.407216683166832e-06, "loss": 0.0009, "num_tokens": 37369805.0, "reward": 0.0, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.258040726184845, "sampling/importance_sampling_ratio/mean": 1.000017422437668, "sampling/importance_sampling_ratio/min": 0.769109570980072, "sampling/sampling_logp_difference/max": 0.28928991556167605, "sampling/sampling_logp_difference/mean": 0.020745681412518025, "step": 8570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 315.5, "completions/max_terminated_length": 312.1, "completions/mean_length": 242.9375, "completions/mean_terminated_length": 235.9175018310547, "completions/min_length": 163.3, "completions/min_terminated_length": 163.3, "entropy": 1.0441000655293464, "epoch": 0.788820446814379, "frac_reward_zero_std": 0.6, "grad_norm": 0.9877323508262634, "kl": 0.013379216252360493, "learning_rate": 4.40557763431727e-06, "loss": 0.0009, "num_tokens": 37418408.0, "reward": -0.34375, "reward_std": 0.3768926799297333, "rewards/reward_fn/mean": -0.34375, "rewards/reward_fn/std": 0.3768926799297333, "sampling/importance_sampling_ratio/max": 1.2642944574356079, "sampling/importance_sampling_ratio/mean": 1.0005558550357818, "sampling/importance_sampling_ratio/min": 0.7921339631080627, "sampling/sampling_logp_difference/max": 0.2553990244865417, "sampling/sampling_logp_difference/mean": 0.020458700880408288, "step": 8580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 355.2, "completions/max_terminated_length": 355.2, "completions/mean_length": 233.4625, "completions/mean_terminated_length": 233.4625, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 1.1134083285927772, "epoch": 0.7897398179645123, "frac_reward_zero_std": 0.3, "grad_norm": 0.13154670596122742, "kl": 0.02115619127289392, "learning_rate": 4.403936628286727e-06, "loss": 0.0011, "num_tokens": 37463789.0, "reward": -0.08125, "reward_std": 0.645913690328598, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.6459136962890625, "sampling/importance_sampling_ratio/max": 1.2421431303024293, "sampling/importance_sampling_ratio/mean": 0.9999328076839447, "sampling/importance_sampling_ratio/min": 0.7584402024745941, "sampling/sampling_logp_difference/max": 0.29074472188949585, "sampling/sampling_logp_difference/mean": 0.02133639082312584, "step": 8590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.3, "completions/max_terminated_length": 260.3, "completions/mean_length": 191.425, "completions/mean_terminated_length": 191.425, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.1320825852453709, "epoch": 0.7906591891146456, "frac_reward_zero_std": 0.2, "grad_norm": 0.11070788651704788, "kl": 0.012626938044559211, "learning_rate": 4.402293666760648e-06, "loss": 0.0004, "num_tokens": 37505775.0, "reward": 0.175, "reward_std": 0.7036870539188385, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.7036870539188385, "sampling/importance_sampling_ratio/max": 1.2378549575805664, "sampling/importance_sampling_ratio/mean": 1.0002644181251525, "sampling/importance_sampling_ratio/min": 0.8008696913719178, "sampling/sampling_logp_difference/max": 0.23618134260177612, "sampling/sampling_logp_difference/mean": 0.021988094411790372, "step": 8600 }, { "epoch": 0.7906591891146456, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 295.0, "eval_completions/max_terminated_length": 292.93368237347295, "eval_completions/mean_length": 209.37303664921467, "eval_completions/mean_terminated_length": 208.86067492633174, "eval_completions/min_length": 137.08202443280976, "eval_completions/min_terminated_length": 137.08202443280976, "eval_entropy": 1.0782078065589253, "eval_frac_reward_zero_std": 0.4799301919720768, "eval_kl": 0.013331777814379121, "eval_loss": 0.0006423988961614668, "eval_num_tokens": 37505775.0, "eval_reward": -0.12030977312390925, "eval_reward_std": 0.452587479956279, "eval_rewards/reward_fn/mean": -0.12030977312390925, "eval_rewards/reward_fn/std": 0.45258749379121405, "eval_runtime": 1720.1189, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2639693589734782, "eval_sampling/importance_sampling_ratio/mean": 1.0000386109110782, "eval_sampling/importance_sampling_ratio/min": 0.7765839457199836, "eval_sampling/sampling_logp_difference/max": 0.28120821485552694, "eval_sampling/sampling_logp_difference/mean": 0.020494898699913142, "eval_steps_per_second": 0.042, "step": 8600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 223.6125, "completions/mean_terminated_length": 223.6125, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.1178068809211255, "epoch": 0.7915785602647789, "frac_reward_zero_std": 0.3, "grad_norm": 6.207152366638184, "kl": 0.01474191941670142, "learning_rate": 4.4006487514264855e-06, "loss": 0.0009, "num_tokens": 37551568.0, "reward": 0.125, "reward_std": 0.6699140250682831, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6699140727519989, "sampling/importance_sampling_ratio/max": 1.2637730240821838, "sampling/importance_sampling_ratio/mean": 1.0002938628196716, "sampling/importance_sampling_ratio/min": 0.7650082886219025, "sampling/sampling_logp_difference/max": 0.28136340379714964, "sampling/sampling_logp_difference/mean": 0.021245297417044638, "step": 8610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 311.1, "completions/max_terminated_length": 292.6, "completions/mean_length": 219.6375, "completions/mean_terminated_length": 213.32083435058593, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.0472187764942646, "epoch": 0.7924979314149122, "frac_reward_zero_std": 0.7, "grad_norm": 0.09594737738370895, "kl": 0.013252562715206296, "learning_rate": 4.399001883973699e-06, "loss": 0.0007, "num_tokens": 37600931.0, "reward": -0.2875, "reward_std": 0.21923734545707702, "rewards/reward_fn/mean": -0.2875, "rewards/reward_fn/std": 0.21923735737800598, "sampling/importance_sampling_ratio/max": 1.2677483201026916, "sampling/importance_sampling_ratio/mean": 0.9996668457984924, "sampling/importance_sampling_ratio/min": 0.779628449678421, "sampling/sampling_logp_difference/max": 0.27577433586120603, "sampling/sampling_logp_difference/mean": 0.02057949248701334, "step": 8620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 194.25, "completions/mean_terminated_length": 194.25, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.085795346647501, "epoch": 0.7934173025650455, "frac_reward_zero_std": 0.6, "grad_norm": 5.073726654052734, "kl": 0.013398844009498135, "learning_rate": 4.397353066093755e-06, "loss": 0.0004, "num_tokens": 37644151.0, "reward": -0.35, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.232960331439972, "sampling/importance_sampling_ratio/mean": 0.999810004234314, "sampling/importance_sampling_ratio/min": 0.7890442192554474, "sampling/sampling_logp_difference/max": 0.25934267044067383, "sampling/sampling_logp_difference/mean": 0.02118926253169775, "step": 8630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 201.6125, "completions/mean_terminated_length": 201.6125, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 0.997449991106987, "epoch": 0.7943366737151788, "frac_reward_zero_std": 0.4, "grad_norm": 5.738603115081787, "kl": 0.01605892198276706, "learning_rate": 4.39570229948012e-06, "loss": 0.001, "num_tokens": 37684032.0, "reward": -0.6, "reward_std": 0.5664041936397552, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.5664042294025421, "sampling/importance_sampling_ratio/max": 1.2117988467216492, "sampling/importance_sampling_ratio/mean": 0.9997898519039154, "sampling/importance_sampling_ratio/min": 0.7134903788566589, "sampling/sampling_logp_difference/max": 0.3588319182395935, "sampling/sampling_logp_difference/mean": 0.01988381501287222, "step": 8640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.4, "completions/max_terminated_length": 312.4, "completions/mean_length": 220.05, "completions/mean_terminated_length": 220.05, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.015807443857193, "epoch": 0.7952560448653121, "frac_reward_zero_std": 0.3, "grad_norm": 0.2305639088153839, "kl": 0.013385279721114784, "learning_rate": 4.394049585828265e-06, "loss": 0.0008, "num_tokens": 37727124.0, "reward": -0.075, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.3368706583976746, "sampling/importance_sampling_ratio/mean": 1.0001224517822265, "sampling/importance_sampling_ratio/min": 0.7933340787887573, "sampling/sampling_logp_difference/max": 0.32950369119644163, "sampling/sampling_logp_difference/mean": 0.02000252027064562, "step": 8650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.8, "completions/max_terminated_length": 303.8, "completions/mean_length": 217.8, "completions/mean_terminated_length": 217.8, "completions/min_length": 157.0, "completions/min_terminated_length": 157.0, "entropy": 0.99781683832407, "epoch": 0.7961754160154454, "frac_reward_zero_std": 0.7, "grad_norm": 0.12098076194524765, "kl": 0.013124394236365333, "learning_rate": 4.392394926835658e-06, "loss": 0.0006, "num_tokens": 37769964.0, "reward": -0.5, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.3230135083198546, "sampling/importance_sampling_ratio/mean": 1.0001308262348174, "sampling/importance_sampling_ratio/min": 0.7640765845775604, "sampling/sampling_logp_difference/max": 0.33219141960144044, "sampling/sampling_logp_difference/mean": 0.01973610110580921, "step": 8660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 215.125, "completions/mean_terminated_length": 215.125, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.067395707219839, "epoch": 0.7970947871655787, "frac_reward_zero_std": 0.2, "grad_norm": 0.12776310741901398, "kl": 0.011915515857981517, "learning_rate": 4.390738324201767e-06, "loss": 0.0002, "num_tokens": 37815478.0, "reward": 0.325, "reward_std": 0.6531550586223602, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.6531550824642182, "sampling/importance_sampling_ratio/max": 1.2950968742370605, "sampling/importance_sampling_ratio/mean": 0.9998649597167969, "sampling/importance_sampling_ratio/min": 0.7668637931346893, "sampling/sampling_logp_difference/max": 0.28392120003700255, "sampling/sampling_logp_difference/mean": 0.02106087440624833, "step": 8670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.1, "completions/max_terminated_length": 276.1, "completions/mean_length": 206.3625, "completions/mean_terminated_length": 206.3625, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 0.9877785183489323, "epoch": 0.7980141583157121, "frac_reward_zero_std": 0.4, "grad_norm": 5.728217124938965, "kl": 0.012440334673738108, "learning_rate": 4.389079779628057e-06, "loss": 0.0007, "num_tokens": 37861355.0, "reward": -0.275, "reward_std": 0.6237098217010498, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6237098336219787, "sampling/importance_sampling_ratio/max": 1.2531020641326904, "sampling/importance_sampling_ratio/mean": 0.9995154678821564, "sampling/importance_sampling_ratio/min": 0.79590203166008, "sampling/sampling_logp_difference/max": 0.24983651638031007, "sampling/sampling_logp_difference/mean": 0.019285763800144195, "step": 8680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.8, "completions/max_terminated_length": 307.8, "completions/mean_length": 213.0125, "completions/mean_terminated_length": 213.0125, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 1.0293732784688472, "epoch": 0.7989335294658454, "frac_reward_zero_std": 0.4, "grad_norm": 4.126009941101074, "kl": 0.011905543258762918, "learning_rate": 4.3874192948179836e-06, "loss": 0.0005, "num_tokens": 37905564.0, "reward": -0.1, "reward_std": 0.577347707748413, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.577347731590271, "sampling/importance_sampling_ratio/max": 1.296187162399292, "sampling/importance_sampling_ratio/mean": 1.0000042915344238, "sampling/importance_sampling_ratio/min": 0.7924862325191497, "sampling/sampling_logp_difference/max": 0.2844851493835449, "sampling/sampling_logp_difference/mean": 0.020524528063833713, "step": 8690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.1, "completions/max_terminated_length": 281.1, "completions/mean_length": 198.6375, "completions/mean_terminated_length": 198.6375, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.0830334290862083, "epoch": 0.7998529006159787, "frac_reward_zero_std": 0.6, "grad_norm": 5.114810466766357, "kl": 0.013196808815700933, "learning_rate": 4.385756871477e-06, "loss": 0.0005, "num_tokens": 37945951.0, "reward": 0.175, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.3142307877540589, "sampling/importance_sampling_ratio/mean": 0.9998034358024597, "sampling/importance_sampling_ratio/min": 0.7933276474475861, "sampling/sampling_logp_difference/max": 0.2809175491333008, "sampling/sampling_logp_difference/mean": 0.02139892429113388, "step": 8700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 197.775, "completions/mean_terminated_length": 197.775, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.054088556021452, "epoch": 0.800772271766112, "frac_reward_zero_std": 0.5, "grad_norm": 0.170479878783226, "kl": 0.013226498884614557, "learning_rate": 4.384092511312547e-06, "loss": 0.0007, "num_tokens": 37986565.0, "reward": -0.45, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.3006847262382508, "sampling/importance_sampling_ratio/mean": 0.9999036967754364, "sampling/importance_sampling_ratio/min": 0.795360004901886, "sampling/sampling_logp_difference/max": 0.274848198890686, "sampling/sampling_logp_difference/mean": 0.02088513635098934, "step": 8710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 296.0, "completions/max_terminated_length": 271.0, "completions/mean_length": 192.5875, "completions/mean_terminated_length": 188.64107208251954, "completions/min_length": 119.2, "completions/min_terminated_length": 119.2, "entropy": 0.9770347893238067, "epoch": 0.8016916429162453, "frac_reward_zero_std": 0.3, "grad_norm": 5.880671977996826, "kl": 0.011728064907947556, "learning_rate": 4.382426216034057e-06, "loss": 0.0006, "num_tokens": 38024796.0, "reward": -0.23125, "reward_std": 0.6089020669460297, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.608902096748352, "sampling/importance_sampling_ratio/max": 1.2556606650352478, "sampling/importance_sampling_ratio/mean": 0.9994518518447876, "sampling/importance_sampling_ratio/min": 0.7675889670848847, "sampling/sampling_logp_difference/max": 0.2778475761413574, "sampling/sampling_logp_difference/mean": 0.020682230964303015, "step": 8720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 341.1, "completions/max_terminated_length": 312.2, "completions/mean_length": 224.8375, "completions/mean_terminated_length": 220.95714416503907, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0363240242004395, "epoch": 0.8026110140663786, "frac_reward_zero_std": 0.2, "grad_norm": 6.539923667907715, "kl": 0.012879094044910744, "learning_rate": 4.380757987352948e-06, "loss": 0.0009, "num_tokens": 38069231.0, "reward": -0.3125, "reward_std": 0.6441861316561699, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.6441861316561699, "sampling/importance_sampling_ratio/max": 1.3064977288246156, "sampling/importance_sampling_ratio/mean": 1.0003900706768036, "sampling/importance_sampling_ratio/min": 0.7638601750135422, "sampling/sampling_logp_difference/max": 0.3517004370689392, "sampling/sampling_logp_difference/mean": 0.020342031493782997, "step": 8730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.0, "completions/max_terminated_length": 331.0, "completions/mean_length": 226.025, "completions/mean_terminated_length": 226.025, "completions/min_length": 151.1, "completions/min_terminated_length": 151.1, "entropy": 1.1086768448352813, "epoch": 0.8035303852165119, "frac_reward_zero_std": 0.6, "grad_norm": 6.454476356506348, "kl": 0.014282935357186943, "learning_rate": 4.379087826982625e-06, "loss": 0.0009, "num_tokens": 38115073.0, "reward": 0.2, "reward_std": 0.39897301197052004, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.39897301197052004, "sampling/importance_sampling_ratio/max": 1.2493292212486267, "sampling/importance_sampling_ratio/mean": 0.9994620382785797, "sampling/importance_sampling_ratio/min": 0.7958155155181885, "sampling/sampling_logp_difference/max": 0.23861565589904785, "sampling/sampling_logp_difference/mean": 0.021127542294561864, "step": 8740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.3, "completions/max_terminated_length": 330.3, "completions/mean_length": 227.5, "completions/mean_terminated_length": 227.5, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0533370144665242, "epoch": 0.8044497563666452, "frac_reward_zero_std": 0.2, "grad_norm": 5.858776092529297, "kl": 0.013886605406878517, "learning_rate": 4.377415736638476e-06, "loss": 0.0009, "num_tokens": 38158105.0, "reward": -0.1, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.2358952641487122, "sampling/importance_sampling_ratio/mean": 1.0004919052124024, "sampling/importance_sampling_ratio/min": 0.7900161445140839, "sampling/sampling_logp_difference/max": 0.25149001479148864, "sampling/sampling_logp_difference/mean": 0.02094944640994072, "step": 8750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.4, "completions/max_terminated_length": 267.4, "completions/mean_length": 204.675, "completions/mean_terminated_length": 204.675, "completions/min_length": 151.4, "completions/min_terminated_length": 151.4, "entropy": 1.0023468010127545, "epoch": 0.8053691275167785, "frac_reward_zero_std": 0.7, "grad_norm": 4.6049323081970215, "kl": 0.012533900793641806, "learning_rate": 4.375741718037873e-06, "loss": 0.0007, "num_tokens": 38198215.0, "reward": -0.375, "reward_std": 0.2886738538742065, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.2886738657951355, "sampling/importance_sampling_ratio/max": 1.270560097694397, "sampling/importance_sampling_ratio/mean": 1.0003179728984832, "sampling/importance_sampling_ratio/min": 0.7782504081726074, "sampling/sampling_logp_difference/max": 0.2787665009498596, "sampling/sampling_logp_difference/mean": 0.020182316191494465, "step": 8760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.3, "completions/max_terminated_length": 286.3, "completions/mean_length": 207.3375, "completions/mean_terminated_length": 207.3375, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0686169892549515, "epoch": 0.8062884986669119, "frac_reward_zero_std": 0.5, "grad_norm": 0.13368993997573853, "kl": 0.014455819001886994, "learning_rate": 4.374065772900165e-06, "loss": 0.0007, "num_tokens": 38243122.0, "reward": 0.3, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.2743713974952697, "sampling/importance_sampling_ratio/mean": 0.9998933494091033, "sampling/importance_sampling_ratio/min": 0.7936227262020111, "sampling/sampling_logp_difference/max": 0.2615763545036316, "sampling/sampling_logp_difference/mean": 0.02061789371073246, "step": 8770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 208.0625, "completions/mean_terminated_length": 208.0625, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.04651652649045, "epoch": 0.8072078698170452, "frac_reward_zero_std": 0.6, "grad_norm": 0.09922927618026733, "kl": 0.013400323572568595, "learning_rate": 4.372387902946684e-06, "loss": 0.0004, "num_tokens": 38288903.0, "reward": 0.025, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.2760984420776367, "sampling/importance_sampling_ratio/mean": 1.0001233637332916, "sampling/importance_sampling_ratio/min": 0.792185777425766, "sampling/sampling_logp_difference/max": 0.26076526641845704, "sampling/sampling_logp_difference/mean": 0.02038005869835615, "step": 8780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.3, "completions/max_terminated_length": 300.3, "completions/mean_length": 206.725, "completions/mean_terminated_length": 206.725, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.0389201514422894, "epoch": 0.8081272409671785, "frac_reward_zero_std": 0.5, "grad_norm": 0.0986640602350235, "kl": 0.013806187827140094, "learning_rate": 4.370708109900737e-06, "loss": 0.0013, "num_tokens": 38329113.0, "reward": -0.05, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2606154561042786, "sampling/importance_sampling_ratio/mean": 1.0006261765956879, "sampling/importance_sampling_ratio/min": 0.7690939396619797, "sampling/sampling_logp_difference/max": 0.3145637512207031, "sampling/sampling_logp_difference/mean": 0.020877469889819623, "step": 8790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 220.6375, "completions/mean_terminated_length": 220.6375, "completions/min_length": 160.1, "completions/min_terminated_length": 160.1, "entropy": 0.9513372279703617, "epoch": 0.8090466121173118, "frac_reward_zero_std": 0.5, "grad_norm": 6.304871559143066, "kl": 0.011985694116447121, "learning_rate": 4.369026395487605e-06, "loss": 0.0002, "num_tokens": 38372196.0, "reward": 0.0, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2387558817863464, "sampling/importance_sampling_ratio/mean": 0.9997300505638123, "sampling/importance_sampling_ratio/min": 0.8031254827976226, "sampling/sampling_logp_difference/max": 0.230490243434906, "sampling/sampling_logp_difference/mean": 0.019088551960885526, "step": 8800 }, { "epoch": 0.8090466121173118, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 306.1588132635253, "eval_completions/max_terminated_length": 304.5305410122164, "eval_completions/mean_length": 213.91186736474694, "eval_completions/mean_terminated_length": 213.51665234524125, "eval_completions/min_length": 140.5497382198953, "eval_completions/min_terminated_length": 140.5497382198953, "eval_entropy": 1.0669786999258903, "eval_frac_reward_zero_std": 0.4781849912739965, "eval_kl": 0.01355733668730703, "eval_loss": 0.0006920176674611866, "eval_num_tokens": 38372196.0, "eval_reward": -0.1261998254799302, "eval_reward_std": 0.4643589663099868, "eval_rewards/reward_fn/mean": -0.1261998254799302, "eval_rewards/reward_fn/std": 0.4643589827454735, "eval_runtime": 1775.8175, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2701281773066229, "eval_sampling/importance_sampling_ratio/mean": 1.0000257886309065, "eval_sampling/importance_sampling_ratio/min": 0.780046327263897, "eval_sampling/sampling_logp_difference/max": 0.2785873759479423, "eval_sampling/sampling_logp_difference/mean": 0.020360439199869754, "eval_steps_per_second": 0.041, "step": 8800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.4, "completions/max_terminated_length": 303.4, "completions/mean_length": 214.875, "completions/mean_terminated_length": 214.875, "completions/min_length": 129.7, "completions/min_terminated_length": 129.7, "entropy": 1.0360447973012925, "epoch": 0.8099659832674451, "frac_reward_zero_std": 0.1, "grad_norm": 8.163630485534668, "kl": 0.014608891669195146, "learning_rate": 4.367342761434544e-06, "loss": 0.0004, "num_tokens": 38417522.0, "reward": -0.08125, "reward_std": 0.7835526749491691, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.7835527345538139, "sampling/importance_sampling_ratio/max": 1.352431857585907, "sampling/importance_sampling_ratio/mean": 0.9999117016792297, "sampling/importance_sampling_ratio/min": 0.7414393663406372, "sampling/sampling_logp_difference/max": 0.39942505955696106, "sampling/sampling_logp_difference/mean": 0.02125026173889637, "step": 8810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 354.0, "completions/max_terminated_length": 354.0, "completions/mean_length": 245.675, "completions/mean_terminated_length": 245.675, "completions/min_length": 168.4, "completions/min_terminated_length": 168.4, "entropy": 1.0508483596146108, "epoch": 0.8108853544175784, "frac_reward_zero_std": 0.2, "grad_norm": 5.108138084411621, "kl": 0.01291628576000221, "learning_rate": 4.365657209470781e-06, "loss": 0.001, "num_tokens": 38464400.0, "reward": 0.075, "reward_std": 0.8239401578903198, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.8239402174949646, "sampling/importance_sampling_ratio/max": 1.2276472687721252, "sampling/importance_sampling_ratio/mean": 1.0002093613147736, "sampling/importance_sampling_ratio/min": 0.7313257157802582, "sampling/sampling_logp_difference/max": 0.3206257104873657, "sampling/sampling_logp_difference/mean": 0.020895576663315298, "step": 8820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 212.75, "completions/mean_terminated_length": 212.75, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 1.0252724669873714, "epoch": 0.8118047255677117, "frac_reward_zero_std": 0.6, "grad_norm": 4.22300386428833, "kl": 0.01240897552925162, "learning_rate": 4.363969741327514e-06, "loss": 0.0005, "num_tokens": 38504996.0, "reward": 0.3, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.27656809091568, "sampling/importance_sampling_ratio/mean": 1.0004706978797913, "sampling/importance_sampling_ratio/min": 0.7516753435134887, "sampling/sampling_logp_difference/max": 0.3101560115814209, "sampling/sampling_logp_difference/mean": 0.021022210642695428, "step": 8830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.6, "completions/max_terminated_length": 276.6, "completions/mean_length": 197.975, "completions/mean_terminated_length": 197.975, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.051889853924513, "epoch": 0.812724096717845, "frac_reward_zero_std": 0.3, "grad_norm": 0.17044700682163239, "kl": 0.01334261271986179, "learning_rate": 4.362280358737907e-06, "loss": 0.0001, "num_tokens": 38545474.0, "reward": -0.15, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2663458108901977, "sampling/importance_sampling_ratio/mean": 0.999366569519043, "sampling/importance_sampling_ratio/min": 0.7574613571166993, "sampling/sampling_logp_difference/max": 0.2864503264427185, "sampling/sampling_logp_difference/mean": 0.02084964234381914, "step": 8840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.0, "completions/max_terminated_length": 275.0, "completions/mean_length": 193.2, "completions/mean_terminated_length": 193.2, "completions/min_length": 121.8, "completions/min_terminated_length": 121.8, "entropy": 1.0511258374899626, "epoch": 0.8136434678679783, "frac_reward_zero_std": 0.6, "grad_norm": 6.085968017578125, "kl": 0.012661724345525727, "learning_rate": 4.360589063437091e-06, "loss": 0.0006, "num_tokens": 38586090.0, "reward": 0.225, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2459503054618835, "sampling/importance_sampling_ratio/mean": 0.99954913854599, "sampling/importance_sampling_ratio/min": 0.7793286204338074, "sampling/sampling_logp_difference/max": 0.2603767693042755, "sampling/sampling_logp_difference/mean": 0.021445014886558054, "step": 8850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.1, "completions/max_terminated_length": 275.1, "completions/mean_length": 218.4125, "completions/mean_terminated_length": 218.4125, "completions/min_length": 155.1, "completions/min_terminated_length": 155.1, "entropy": 1.00184967815876, "epoch": 0.8145628390181117, "frac_reward_zero_std": 0.3, "grad_norm": 5.44302225112915, "kl": 0.01325685556512326, "learning_rate": 4.358895857162163e-06, "loss": 0.0006, "num_tokens": 38629403.0, "reward": 0.21875, "reward_std": 0.611049872636795, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.6110499143600464, "sampling/importance_sampling_ratio/max": 1.2709824323654175, "sampling/importance_sampling_ratio/mean": 1.0004944443702697, "sampling/importance_sampling_ratio/min": 0.7775283813476562, "sampling/sampling_logp_difference/max": 0.27330842018127444, "sampling/sampling_logp_difference/mean": 0.020104968175292014, "step": 8860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 231.275, "completions/mean_terminated_length": 231.275, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.0975443571805954, "epoch": 0.815482210168245, "frac_reward_zero_std": 0.3, "grad_norm": 5.135892391204834, "kl": 0.01444791878457181, "learning_rate": 4.35720074165218e-06, "loss": 0.0008, "num_tokens": 38672265.0, "reward": -0.3, "reward_std": 0.6371305644512176, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.6371305763721467, "sampling/importance_sampling_ratio/max": 1.269204020500183, "sampling/importance_sampling_ratio/mean": 1.0003786444664002, "sampling/importance_sampling_ratio/min": 0.7893802344799041, "sampling/sampling_logp_difference/max": 0.24804942607879638, "sampling/sampling_logp_difference/mean": 0.021165334060788153, "step": 8870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.1, "completions/max_terminated_length": 308.1, "completions/mean_length": 216.5375, "completions/mean_terminated_length": 216.5375, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 0.9899786680936813, "epoch": 0.8164015813183783, "frac_reward_zero_std": 0.3, "grad_norm": 5.412006378173828, "kl": 0.01302625063690357, "learning_rate": 4.355503718648163e-06, "loss": 0.0007, "num_tokens": 38715020.0, "reward": -0.275, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.2774726510047913, "sampling/importance_sampling_ratio/mean": 1.0003698766231537, "sampling/importance_sampling_ratio/min": 0.7927900195121765, "sampling/sampling_logp_difference/max": 0.2668042480945587, "sampling/sampling_logp_difference/mean": 0.02002958841621876, "step": 8880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.7, "completions/max_terminated_length": 321.7, "completions/mean_length": 213.225, "completions/mean_terminated_length": 213.225, "completions/min_length": 151.4, "completions/min_terminated_length": 151.4, "entropy": 1.1042526111006736, "epoch": 0.8173209524685116, "frac_reward_zero_std": 0.6, "grad_norm": 0.09323861449956894, "kl": 0.012007428577635437, "learning_rate": 4.35380478989309e-06, "loss": 0.0007, "num_tokens": 38759454.0, "reward": 0.025, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.279676604270935, "sampling/importance_sampling_ratio/mean": 0.9997261226177215, "sampling/importance_sampling_ratio/min": 0.7901591837406159, "sampling/sampling_logp_difference/max": 0.2671209812164307, "sampling/sampling_logp_difference/mean": 0.021403517201542856, "step": 8890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 218.525, "completions/mean_terminated_length": 218.525, "completions/min_length": 150.2, "completions/min_terminated_length": 150.2, "entropy": 1.0217902146279811, "epoch": 0.8182403236186448, "frac_reward_zero_std": 0.3, "grad_norm": 0.20024260878562927, "kl": 0.015527741744881496, "learning_rate": 4.352103957131898e-06, "loss": 0.0007, "num_tokens": 38804152.0, "reward": -0.4, "reward_std": 0.6220328629016876, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6220328748226166, "sampling/importance_sampling_ratio/max": 1.304703164100647, "sampling/importance_sampling_ratio/mean": 1.0001261174678802, "sampling/importance_sampling_ratio/min": 0.8010365128517151, "sampling/sampling_logp_difference/max": 0.2709375977516174, "sampling/sampling_logp_difference/mean": 0.020492741093039512, "step": 8900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.7, "completions/max_terminated_length": 275.7, "completions/mean_length": 197.0, "completions/mean_terminated_length": 197.0, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 0.9798497945070267, "epoch": 0.8191596947687781, "frac_reward_zero_std": 0.3, "grad_norm": 0.14258703589439392, "kl": 0.01374350839178078, "learning_rate": 4.350401222111477e-06, "loss": 0.0007, "num_tokens": 38847464.0, "reward": 0.15, "reward_std": 0.6876311719417572, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6876312077045441, "sampling/importance_sampling_ratio/max": 1.3520187973976134, "sampling/importance_sampling_ratio/mean": 0.9995113253593445, "sampling/importance_sampling_ratio/min": 0.7725097596645355, "sampling/sampling_logp_difference/max": 0.34836832284927366, "sampling/sampling_logp_difference/mean": 0.01988589521497488, "step": 8910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 298.7, "completions/max_terminated_length": 293.4, "completions/mean_length": 226.65, "completions/mean_terminated_length": 219.31499938964845, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.0754505917429924, "epoch": 0.8200790659189114, "frac_reward_zero_std": 0.3, "grad_norm": 6.3675456047058105, "kl": 0.014070627558976413, "learning_rate": 4.348696586580673e-06, "loss": 0.0008, "num_tokens": 38894124.0, "reward": -0.26875, "reward_std": 0.6540716052055359, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.6540716052055359, "sampling/importance_sampling_ratio/max": 1.4324906945228577, "sampling/importance_sampling_ratio/mean": 1.0006136536598205, "sampling/importance_sampling_ratio/min": 0.7714730262756347, "sampling/sampling_logp_difference/max": 0.3719616115093231, "sampling/sampling_logp_difference/mean": 0.020882203057408334, "step": 8920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.3, "completions/max_terminated_length": 251.3, "completions/mean_length": 181.9375, "completions/mean_terminated_length": 181.9375, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.0295913755893706, "epoch": 0.8209984370690447, "frac_reward_zero_std": 0.6, "grad_norm": 0.12759514153003693, "kl": 0.014331286016386002, "learning_rate": 4.346990052290283e-06, "loss": 0.0007, "num_tokens": 38933231.0, "reward": -0.175, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2265989661216736, "sampling/importance_sampling_ratio/mean": 1.0001248836517334, "sampling/importance_sampling_ratio/min": 0.7822132647037506, "sampling/sampling_logp_difference/max": 0.26798402070999144, "sampling/sampling_logp_difference/mean": 0.02040304709225893, "step": 8930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.9, "completions/max_terminated_length": 266.9, "completions/mean_length": 191.6, "completions/mean_terminated_length": 191.6, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.0580658622086048, "epoch": 0.821917808219178, "frac_reward_zero_std": 0.5, "grad_norm": 6.240699768066406, "kl": 0.014172870485344902, "learning_rate": 4.345281620993056e-06, "loss": 0.0003, "num_tokens": 38975199.0, "reward": -0.5, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.3255136728286743, "sampling/importance_sampling_ratio/mean": 0.9999423444271087, "sampling/importance_sampling_ratio/min": 0.7773100018501282, "sampling/sampling_logp_difference/max": 0.31057080030441286, "sampling/sampling_logp_difference/mean": 0.02110734600573778, "step": 8940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.2, "completions/max_terminated_length": 311.2, "completions/mean_length": 206.7125, "completions/mean_terminated_length": 206.7125, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0140625901520253, "epoch": 0.8228371793693114, "frac_reward_zero_std": 0.1, "grad_norm": 5.323394298553467, "kl": 0.015286987269064412, "learning_rate": 4.343571294443686e-06, "loss": 0.0004, "num_tokens": 39016784.0, "reward": -0.15, "reward_std": 0.7600752472877502, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.7600752472877502, "sampling/importance_sampling_ratio/max": 1.2682265281677245, "sampling/importance_sampling_ratio/mean": 0.9999592900276184, "sampling/importance_sampling_ratio/min": 0.7936973810195923, "sampling/sampling_logp_difference/max": 0.2559078216552734, "sampling/sampling_logp_difference/mean": 0.02062276117503643, "step": 8950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.9, "completions/max_terminated_length": 264.9, "completions/mean_length": 189.9625, "completions/mean_terminated_length": 189.9625, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.0605313070118427, "epoch": 0.8237565505194447, "frac_reward_zero_std": 0.6, "grad_norm": 0.11600160598754883, "kl": 0.014354482275666668, "learning_rate": 4.3418590743988164e-06, "loss": 0.0007, "num_tokens": 39058429.0, "reward": -0.35, "reward_std": 0.3846505284309387, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3846505284309387, "sampling/importance_sampling_ratio/max": 1.27142174243927, "sampling/importance_sampling_ratio/mean": 0.9997960865497589, "sampling/importance_sampling_ratio/min": 0.7561356067657471, "sampling/sampling_logp_difference/max": 0.2956423044204712, "sampling/sampling_logp_difference/mean": 0.021681500785052776, "step": 8960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.4, "completions/max_terminated_length": 294.4, "completions/mean_length": 205.8875, "completions/mean_terminated_length": 205.8875, "completions/min_length": 126.3, "completions/min_terminated_length": 126.3, "entropy": 1.1220843508839606, "epoch": 0.824675921669578, "frac_reward_zero_std": 0.7, "grad_norm": 0.12449431419372559, "kl": 0.01721098316484131, "learning_rate": 4.340144962617034e-06, "loss": 0.0008, "num_tokens": 39106988.0, "reward": -0.275, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.27680561542511, "sampling/importance_sampling_ratio/mean": 0.9999529421329498, "sampling/importance_sampling_ratio/min": 0.7683217883110046, "sampling/sampling_logp_difference/max": 0.2912490278482437, "sampling/sampling_logp_difference/mean": 0.021633419208228588, "step": 8970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.2, "completions/max_terminated_length": 295.2, "completions/mean_length": 201.9875, "completions/mean_terminated_length": 201.9875, "completions/min_length": 117.1, "completions/min_terminated_length": 117.1, "entropy": 1.0488963827490807, "epoch": 0.8255952928197113, "frac_reward_zero_std": 0.6, "grad_norm": 0.14217568933963776, "kl": 0.014690255594905466, "learning_rate": 4.33842896085887e-06, "loss": 0.0004, "num_tokens": 39150363.0, "reward": -0.4, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2495683431625366, "sampling/importance_sampling_ratio/mean": 1.0003337383270263, "sampling/importance_sampling_ratio/min": 0.7603154063224793, "sampling/sampling_logp_difference/max": 0.29334421157836915, "sampling/sampling_logp_difference/mean": 0.02074357457458973, "step": 8980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.9, "completions/max_terminated_length": 301.9, "completions/mean_length": 211.7125, "completions/mean_terminated_length": 211.7125, "completions/min_length": 148.2, "completions/min_terminated_length": 148.2, "entropy": 1.0347774736583233, "epoch": 0.8265146639698446, "frac_reward_zero_std": 0.6, "grad_norm": 5.3192644119262695, "kl": 0.014245395571924746, "learning_rate": 4.336711070886795e-06, "loss": 0.0009, "num_tokens": 39197276.0, "reward": -0.025, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2591518044471741, "sampling/importance_sampling_ratio/mean": 1.0003688097000123, "sampling/importance_sampling_ratio/min": 0.7641286194324494, "sampling/sampling_logp_difference/max": 0.2892046719789505, "sampling/sampling_logp_difference/mean": 0.019831421598792077, "step": 8990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 201.675, "completions/mean_terminated_length": 201.675, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 1.0736404985189438, "epoch": 0.8274340351199779, "frac_reward_zero_std": 0.4, "grad_norm": 0.13172748684883118, "kl": 0.013162730977637694, "learning_rate": 4.33499129446522e-06, "loss": 0.0006, "num_tokens": 39241634.0, "reward": -0.05, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2501609444618225, "sampling/importance_sampling_ratio/mean": 0.9995034277439118, "sampling/importance_sampling_ratio/min": 0.7899911940097809, "sampling/sampling_logp_difference/max": 0.25230025947093965, "sampling/sampling_logp_difference/mean": 0.02099774908274412, "step": 9000 }, { "epoch": 0.8274340351199779, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 290.8760907504363, "eval_completions/max_terminated_length": 290.35602094240835, "eval_completions/mean_length": 204.80148342059337, "eval_completions/mean_terminated_length": 204.6026241442296, "eval_completions/min_length": 132.06457242582897, "eval_completions/min_terminated_length": 132.06457242582897, "eval_entropy": 1.0884233486902026, "eval_frac_reward_zero_std": 0.4973821989528796, "eval_kl": 0.014083638768578127, "eval_loss": 0.0006952761323191226, "eval_num_tokens": 39241634.0, "eval_reward": -0.12052792321116929, "eval_reward_std": 0.446931171401633, "eval_rewards/reward_fn/mean": -0.12052792321116929, "eval_rewards/reward_fn/std": 0.44693118471645765, "eval_runtime": 1700.756, "eval_samples_per_second": 0.337, "eval_sampling/importance_sampling_ratio/max": 1.276129672248534, "eval_sampling/importance_sampling_ratio/mean": 0.9999992776707621, "eval_sampling/importance_sampling_ratio/min": 0.7833430185575968, "eval_sampling/sampling_logp_difference/max": 0.2787516302255228, "eval_sampling/sampling_logp_difference/mean": 0.020847106105396054, "eval_steps_per_second": 0.042, "step": 9000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.9, "completions/max_terminated_length": 336.9, "completions/mean_length": 222.0125, "completions/mean_terminated_length": 222.0125, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.167338213324547, "epoch": 0.8283534062701112, "frac_reward_zero_std": 0.3, "grad_norm": 0.09281348437070847, "kl": 0.013699013454606756, "learning_rate": 4.333269633360493e-06, "loss": 0.0009, "num_tokens": 39286619.0, "reward": -0.25, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.2860737800598145, "sampling/importance_sampling_ratio/mean": 1.0004575550556183, "sampling/importance_sampling_ratio/min": 0.739242285490036, "sampling/sampling_logp_difference/max": 0.3334961533546448, "sampling/sampling_logp_difference/mean": 0.022037928737699986, "step": 9010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 210.125, "completions/mean_terminated_length": 210.125, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.082104642689228, "epoch": 0.8292727774202445, "frac_reward_zero_std": 0.3, "grad_norm": 6.376908779144287, "kl": 0.014378116937587037, "learning_rate": 4.3315460893408975e-06, "loss": 0.0003, "num_tokens": 39330965.0, "reward": 0.075, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.238431179523468, "sampling/importance_sampling_ratio/mean": 0.9992510080337524, "sampling/importance_sampling_ratio/min": 0.7362681478261948, "sampling/sampling_logp_difference/max": 0.34032883048057555, "sampling/sampling_logp_difference/mean": 0.021398686803877354, "step": 9020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 205.375, "completions/mean_terminated_length": 205.375, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.111964190006256, "epoch": 0.8301921485703778, "frac_reward_zero_std": 0.3, "grad_norm": 4.900437831878662, "kl": 0.015261386253405363, "learning_rate": 4.329820664176651e-06, "loss": 0.001, "num_tokens": 39374851.0, "reward": -0.25, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2499826669692993, "sampling/importance_sampling_ratio/mean": 0.9992579877376556, "sampling/importance_sampling_ratio/min": 0.7815191745758057, "sampling/sampling_logp_difference/max": 0.2575385093688965, "sampling/sampling_logp_difference/mean": 0.021621518954634666, "step": 9030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.5, "completions/max_terminated_length": 311.5, "completions/mean_length": 223.1875, "completions/mean_terminated_length": 223.1875, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0970640532672404, "epoch": 0.8311115197205111, "frac_reward_zero_std": 0.6, "grad_norm": 8.135531425476074, "kl": 0.01587138021714054, "learning_rate": 4.3280933596399066e-06, "loss": 0.0009, "num_tokens": 39425114.0, "reward": 0.125, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.3342608332633972, "sampling/importance_sampling_ratio/mean": 0.99973623752594, "sampling/importance_sampling_ratio/min": 0.7550672054290771, "sampling/sampling_logp_difference/max": 0.32931567430496217, "sampling/sampling_logp_difference/mean": 0.020646551996469496, "step": 9040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 218.4625, "completions/mean_terminated_length": 218.4625, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.0031850568950176, "epoch": 0.8320308908706445, "frac_reward_zero_std": 0.3, "grad_norm": 0.1056554764509201, "kl": 0.012379492950276471, "learning_rate": 4.326364177504741e-06, "loss": 0.0007, "num_tokens": 39471071.0, "reward": -0.08125, "reward_std": 0.5437338575720787, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5437338814139366, "sampling/importance_sampling_ratio/max": 1.2300634264945984, "sampling/importance_sampling_ratio/mean": 1.0004270315170287, "sampling/importance_sampling_ratio/min": 0.7933188140392303, "sampling/sampling_logp_difference/max": 0.24029686450958251, "sampling/sampling_logp_difference/mean": 0.020061824284493924, "step": 9050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.1, "completions/max_terminated_length": 257.1, "completions/mean_length": 190.1625, "completions/mean_terminated_length": 190.1625, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.0819395016878843, "epoch": 0.8329502620207778, "frac_reward_zero_std": 0.3, "grad_norm": 8.029595375061035, "kl": 0.013483137637376786, "learning_rate": 4.324633119547163e-06, "loss": 0.0005, "num_tokens": 39512276.0, "reward": -0.025, "reward_std": 0.6985589921474457, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6985590398311615, "sampling/importance_sampling_ratio/max": 1.2461308121681214, "sampling/importance_sampling_ratio/mean": 0.9995247185230255, "sampling/importance_sampling_ratio/min": 0.7347214132547378, "sampling/sampling_logp_difference/max": 0.33891324102878573, "sampling/sampling_logp_difference/mean": 0.022697543352842332, "step": 9060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.2, "completions/max_terminated_length": 256.2, "completions/mean_length": 177.3625, "completions/mean_terminated_length": 177.3625, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0015800952911378, "epoch": 0.8338696331709111, "frac_reward_zero_std": 0.6, "grad_norm": 0.12903080880641937, "kl": 0.013553792375023476, "learning_rate": 4.32290018754511e-06, "loss": 0.0003, "num_tokens": 39553529.0, "reward": 0.15, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.250769591331482, "sampling/importance_sampling_ratio/mean": 1.0005204617977141, "sampling/importance_sampling_ratio/min": 0.805527126789093, "sampling/sampling_logp_difference/max": 0.25156090259552, "sampling/sampling_logp_difference/mean": 0.020174132473766804, "step": 9070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 192.8875, "completions/mean_terminated_length": 192.8875, "completions/min_length": 114.5, "completions/min_terminated_length": 114.5, "entropy": 1.0314692974090576, "epoch": 0.8347890043210444, "frac_reward_zero_std": 0.3, "grad_norm": 4.108964443206787, "kl": 0.015804466599365696, "learning_rate": 4.3211653832784415e-06, "loss": 0.0007, "num_tokens": 39590920.0, "reward": 0.14375, "reward_std": 0.5542315781116486, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.554231595993042, "sampling/importance_sampling_ratio/max": 1.233027994632721, "sampling/importance_sampling_ratio/mean": 0.999692291021347, "sampling/importance_sampling_ratio/min": 0.7763713598251343, "sampling/sampling_logp_difference/max": 0.2633847236633301, "sampling/sampling_logp_difference/mean": 0.021515607833862305, "step": 9080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.9, "completions/max_terminated_length": 316.0, "completions/mean_length": 219.95, "completions/mean_terminated_length": 217.0, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.079778029769659, "epoch": 0.8357083754711777, "frac_reward_zero_std": 0.4, "grad_norm": 6.624993801116943, "kl": 0.01990006069536321, "learning_rate": 4.31942870852894e-06, "loss": 0.001, "num_tokens": 39637012.0, "reward": -0.65625, "reward_std": 0.45116754025220873, "rewards/reward_fn/mean": -0.65625, "rewards/reward_fn/std": 0.45116754025220873, "sampling/importance_sampling_ratio/max": 1.2935789823532104, "sampling/importance_sampling_ratio/mean": 1.000543999671936, "sampling/importance_sampling_ratio/min": 0.7381456702947616, "sampling/sampling_logp_difference/max": 0.3217781126499176, "sampling/sampling_logp_difference/mean": 0.020879948697984218, "step": 9090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 333.2, "completions/max_terminated_length": 308.7, "completions/mean_length": 222.025, "completions/mean_terminated_length": 218.60535888671876, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.0891269996762276, "epoch": 0.836627746621311, "frac_reward_zero_std": 0.6, "grad_norm": 0.11980469524860382, "kl": 0.013919207919389009, "learning_rate": 4.31769016508031e-06, "loss": 0.0009, "num_tokens": 39682374.0, "reward": -0.45625, "reward_std": 0.33160182684659956, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.3316018506884575, "sampling/importance_sampling_ratio/max": 1.2534599900245667, "sampling/importance_sampling_ratio/mean": 0.9995041251182556, "sampling/importance_sampling_ratio/min": 0.7733463704586029, "sampling/sampling_logp_difference/max": 0.27461144924163816, "sampling/sampling_logp_difference/mean": 0.020848235115408897, "step": 9100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.0, "completions/max_terminated_length": 280.0, "completions/mean_length": 202.1125, "completions/mean_terminated_length": 202.1125, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 0.9783101834356784, "epoch": 0.8375471177714443, "frac_reward_zero_std": 0.4, "grad_norm": 0.10178539901971817, "kl": 0.014253682590788231, "learning_rate": 4.315949754718174e-06, "loss": 0.0006, "num_tokens": 39724327.0, "reward": -0.175, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2349908232688904, "sampling/importance_sampling_ratio/mean": 1.0000597834587097, "sampling/importance_sampling_ratio/min": 0.7781132102012634, "sampling/sampling_logp_difference/max": 0.25194196701049804, "sampling/sampling_logp_difference/mean": 0.02025037109851837, "step": 9110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 212.725, "completions/mean_terminated_length": 212.725, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0247681461274625, "epoch": 0.8384664889215776, "frac_reward_zero_std": 0.5, "grad_norm": 6.23419713973999, "kl": 0.013253672223072499, "learning_rate": 4.3142074792300735e-06, "loss": 0.0011, "num_tokens": 39766353.0, "reward": -0.075, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2943772554397583, "sampling/importance_sampling_ratio/mean": 1.000042301416397, "sampling/importance_sampling_ratio/min": 0.7744916021823883, "sampling/sampling_logp_difference/max": 0.2750296056270599, "sampling/sampling_logp_difference/mean": 0.020512605458498, "step": 9120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.9, "completions/max_terminated_length": 299.9, "completions/mean_length": 219.1375, "completions/mean_terminated_length": 219.1375, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0563831850886345, "epoch": 0.8393858600717109, "frac_reward_zero_std": 0.5, "grad_norm": 0.12490265816450119, "kl": 0.014660876081325113, "learning_rate": 4.312463340405466e-06, "loss": 0.0011, "num_tokens": 39810244.0, "reward": 0.075, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.232581889629364, "sampling/importance_sampling_ratio/mean": 0.9997854590415954, "sampling/importance_sampling_ratio/min": 0.7938170850276947, "sampling/sampling_logp_difference/max": 0.23857340812683106, "sampling/sampling_logp_difference/mean": 0.02054567039012909, "step": 9130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.7, "completions/max_terminated_length": 321.7, "completions/mean_length": 221.75, "completions/mean_terminated_length": 221.75, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0767484508454799, "epoch": 0.8403052312218443, "frac_reward_zero_std": 0.4, "grad_norm": 0.13321304321289062, "kl": 0.016059817327186466, "learning_rate": 4.310717340035721e-06, "loss": 0.001, "num_tokens": 39853816.0, "reward": -0.5, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.2731367945671082, "sampling/importance_sampling_ratio/mean": 0.9998704493045807, "sampling/importance_sampling_ratio/min": 0.7707279980182647, "sampling/sampling_logp_difference/max": 0.2709728956222534, "sampling/sampling_logp_difference/mean": 0.02110467106103897, "step": 9140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.0, "completions/max_terminated_length": 339.0, "completions/mean_length": 231.2375, "completions/mean_terminated_length": 231.2375, "completions/min_length": 159.5, "completions/min_terminated_length": 159.5, "entropy": 1.1491557240486145, "epoch": 0.8412246023719776, "frac_reward_zero_std": 0.4, "grad_norm": 0.13711419701576233, "kl": 0.014815918926615268, "learning_rate": 4.308969479914121e-06, "loss": 0.0004, "num_tokens": 39900035.0, "reward": -0.6, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.2555963039398192, "sampling/importance_sampling_ratio/mean": 0.9999122262001038, "sampling/importance_sampling_ratio/min": 0.7683758199214935, "sampling/sampling_logp_difference/max": 0.2754080295562744, "sampling/sampling_logp_difference/mean": 0.02226655576378107, "step": 9150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.7, "completions/max_terminated_length": 307.7, "completions/mean_length": 229.95, "completions/mean_terminated_length": 229.95, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.0626921452581883, "epoch": 0.8421439735221109, "frac_reward_zero_std": 0.5, "grad_norm": 4.959325313568115, "kl": 0.012543244345579296, "learning_rate": 4.307219761835859e-06, "loss": 0.0011, "num_tokens": 39948807.0, "reward": 0.4, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.3199602603912353, "sampling/importance_sampling_ratio/mean": 0.9998042523860932, "sampling/importance_sampling_ratio/min": 0.7607799351215363, "sampling/sampling_logp_difference/max": 0.3005982041358948, "sampling/sampling_logp_difference/mean": 0.021209580823779106, "step": 9160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.7, "completions/max_terminated_length": 319.7, "completions/mean_length": 228.05, "completions/mean_terminated_length": 228.05, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.0768062099814415, "epoch": 0.8430633446722442, "frac_reward_zero_std": 0.3, "grad_norm": 4.708240509033203, "kl": 0.014057989662978798, "learning_rate": 4.3054681875980345e-06, "loss": 0.0006, "num_tokens": 39994179.0, "reward": -0.075, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.258805787563324, "sampling/importance_sampling_ratio/mean": 0.9997485280036926, "sampling/importance_sampling_ratio/min": 0.7407498121261596, "sampling/sampling_logp_difference/max": 0.31606017202138903, "sampling/sampling_logp_difference/mean": 0.021179242059588432, "step": 9170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 208.3125, "completions/mean_terminated_length": 208.3125, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.0739698484539986, "epoch": 0.8439827158223775, "frac_reward_zero_std": 0.6, "grad_norm": 0.154567152261734, "kl": 0.01466771587729454, "learning_rate": 4.303714758999655e-06, "loss": 0.0009, "num_tokens": 40037604.0, "reward": 0.125, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2703548312187194, "sampling/importance_sampling_ratio/mean": 0.9997775554656982, "sampling/importance_sampling_ratio/min": 0.7874185383319855, "sampling/sampling_logp_difference/max": 0.2725841164588928, "sampling/sampling_logp_difference/mean": 0.020871107280254365, "step": 9180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.7, "completions/max_terminated_length": 293.7, "completions/mean_length": 197.3125, "completions/mean_terminated_length": 197.3125, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0348471753299235, "epoch": 0.8449020869725108, "frac_reward_zero_std": 0.4, "grad_norm": 0.12504403293132782, "kl": 0.037911325960885735, "learning_rate": 4.301959477841634e-06, "loss": 0.0021, "num_tokens": 40079269.0, "reward": 0.45, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.2465128779411316, "sampling/importance_sampling_ratio/mean": 0.9999492168426514, "sampling/importance_sampling_ratio/min": 0.7693849444389343, "sampling/sampling_logp_difference/max": 0.2726785659790039, "sampling/sampling_logp_difference/mean": 0.021268236637115478, "step": 9190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 288.9, "completions/max_terminated_length": 280.4, "completions/mean_length": 204.875, "completions/mean_terminated_length": 201.99642944335938, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.0135632636025549, "epoch": 0.8458214581226441, "frac_reward_zero_std": 0.5, "grad_norm": 0.11063108593225479, "kl": 0.02418628037557937, "learning_rate": 4.300202345926783e-06, "loss": 0.0009, "num_tokens": 40123355.0, "reward": 0.16875, "reward_std": 0.4616333365440369, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.46163336038589475, "sampling/importance_sampling_ratio/max": 1.291407585144043, "sampling/importance_sampling_ratio/mean": 1.0001366555690765, "sampling/importance_sampling_ratio/min": 0.7350589573383332, "sampling/sampling_logp_difference/max": 0.32391552329063417, "sampling/sampling_logp_difference/mean": 0.02087221574038267, "step": 9200 }, { "epoch": 0.8458214581226441, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 282.57940663176265, "eval_completions/max_terminated_length": 281.10122164048863, "eval_completions/mean_length": 196.70135253054102, "eval_completions/mean_terminated_length": 196.2131846962175, "eval_completions/min_length": 125.40837696335079, "eval_completions/min_terminated_length": 125.40837696335079, "eval_entropy": 1.0501805007665777, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.01427465006969154, "eval_loss": 0.0006881013396196067, "eval_num_tokens": 40123355.0, "eval_reward": -0.10536649214659685, "eval_reward_std": 0.4497832890736495, "eval_rewards/reward_fn/mean": -0.10536649214659685, "eval_rewards/reward_fn/std": 0.4497833028045624, "eval_runtime": 1654.3853, "eval_samples_per_second": 0.346, "eval_sampling/importance_sampling_ratio/max": 1.2750771322383514, "eval_sampling/importance_sampling_ratio/mean": 1.0000070504077025, "eval_sampling/importance_sampling_ratio/min": 0.7809220805218082, "eval_sampling/sampling_logp_difference/max": 0.285108559054646, "eval_sampling/sampling_logp_difference/mean": 0.020493712770325145, "eval_steps_per_second": 0.044, "step": 9200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 200.1625, "completions/mean_terminated_length": 200.1625, "completions/min_length": 141.4, "completions/min_terminated_length": 141.4, "entropy": 1.0149377666413784, "epoch": 0.8467408292727774, "frac_reward_zero_std": 0.6, "grad_norm": 6.181873798370361, "kl": 0.012101669079856946, "learning_rate": 4.29844336505982e-06, "loss": 0.0009, "num_tokens": 40166496.0, "reward": -0.2, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.2578066349029542, "sampling/importance_sampling_ratio/mean": 1.0000592112541198, "sampling/importance_sampling_ratio/min": 0.7751826047897339, "sampling/sampling_logp_difference/max": 0.2725083589553833, "sampling/sampling_logp_difference/mean": 0.02047896720468998, "step": 9210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.3, "completions/max_terminated_length": 279.3, "completions/mean_length": 194.8375, "completions/mean_terminated_length": 194.8375, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0877435892820357, "epoch": 0.8476602004229107, "frac_reward_zero_std": 0.7, "grad_norm": 0.09860740602016449, "kl": 0.014681422291323543, "learning_rate": 4.296682537047357e-06, "loss": 0.0007, "num_tokens": 40210123.0, "reward": -0.45, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2785296797752381, "sampling/importance_sampling_ratio/mean": 1.000680184364319, "sampling/importance_sampling_ratio/min": 0.6770463168621064, "sampling/sampling_logp_difference/max": 0.4251534461975098, "sampling/sampling_logp_difference/mean": 0.021336807496845724, "step": 9220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 200.125, "completions/mean_terminated_length": 200.125, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.051857339590788, "epoch": 0.8485795715730441, "frac_reward_zero_std": 0.4, "grad_norm": 5.961211204528809, "kl": 0.012547589174937456, "learning_rate": 4.294919863697908e-06, "loss": 0.0008, "num_tokens": 40252789.0, "reward": 0.325, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2687966704368592, "sampling/importance_sampling_ratio/mean": 1.000016212463379, "sampling/importance_sampling_ratio/min": 0.7707673192024231, "sampling/sampling_logp_difference/max": 0.29917728900909424, "sampling/sampling_logp_difference/mean": 0.021338864229619503, "step": 9230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 181.775, "completions/mean_terminated_length": 181.775, "completions/min_length": 107.9, "completions/min_terminated_length": 107.9, "entropy": 1.1333774331957103, "epoch": 0.8494989427231774, "frac_reward_zero_std": 0.3, "grad_norm": 0.14399151504039764, "kl": 0.014548925778944977, "learning_rate": 4.293155346821877e-06, "loss": 0.0002, "num_tokens": 40292059.0, "reward": -0.175, "reward_std": 0.6834926784038544, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6834926784038544, "sampling/importance_sampling_ratio/max": 1.2513783693313598, "sampling/importance_sampling_ratio/mean": 1.000236266851425, "sampling/importance_sampling_ratio/min": 0.7833053290843963, "sampling/sampling_logp_difference/max": 0.26112923622131345, "sampling/sampling_logp_difference/mean": 0.0228878166526556, "step": 9240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 205.5875, "completions/mean_terminated_length": 205.5875, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0250325586646796, "epoch": 0.8504183138733107, "frac_reward_zero_std": 0.3, "grad_norm": 7.277683258056641, "kl": 0.012372186040738597, "learning_rate": 4.291388988231567e-06, "loss": 0.0009, "num_tokens": 40335666.0, "reward": -0.075, "reward_std": 0.69102583527565, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6910258591175079, "sampling/importance_sampling_ratio/max": 1.2929649233818055, "sampling/importance_sampling_ratio/mean": 1.0000785410404205, "sampling/importance_sampling_ratio/min": 0.8050344824790955, "sampling/sampling_logp_difference/max": 0.2619426131248474, "sampling/sampling_logp_difference/mean": 0.020836175605654716, "step": 9250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 290.3, "completions/max_terminated_length": 289.0, "completions/mean_length": 216.925, "completions/mean_terminated_length": 205.9875, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 0.9945106759667397, "epoch": 0.851337685023444, "frac_reward_zero_std": 0.6, "grad_norm": 6.407793045043945, "kl": 0.013124381442321465, "learning_rate": 4.289620789741169e-06, "loss": 0.0005, "num_tokens": 40379100.0, "reward": -0.18125, "reward_std": 0.2984764397144318, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.2984764397144318, "sampling/importance_sampling_ratio/max": 1.3819982647895812, "sampling/importance_sampling_ratio/mean": 1.0003758192062377, "sampling/importance_sampling_ratio/min": 0.7749913692474365, "sampling/sampling_logp_difference/max": 0.33480271100997927, "sampling/sampling_logp_difference/mean": 0.019821017049252987, "step": 9260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 311.6, "completions/max_terminated_length": 297.6, "completions/mean_length": 204.2625, "completions/mean_terminated_length": 201.04107208251952, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 0.9963992416858674, "epoch": 0.8522570561735773, "frac_reward_zero_std": 0.5, "grad_norm": 4.822330474853516, "kl": 0.01267253804835491, "learning_rate": 4.2878507531667635e-06, "loss": 0.0008, "num_tokens": 40420217.0, "reward": 0.14375, "reward_std": 0.44560937881469725, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.44560940861701964, "sampling/importance_sampling_ratio/max": 1.2284598708152772, "sampling/importance_sampling_ratio/mean": 0.999797260761261, "sampling/importance_sampling_ratio/min": 0.7841561198234558, "sampling/sampling_logp_difference/max": 0.24948132038116455, "sampling/sampling_logp_difference/mean": 0.0200586493127048, "step": 9270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.6, "completions/max_terminated_length": 318.6, "completions/mean_length": 220.3, "completions/mean_terminated_length": 220.3, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.0040163815021514, "epoch": 0.8531764273237106, "frac_reward_zero_std": 0.4, "grad_norm": 5.669116020202637, "kl": 0.012397695961408318, "learning_rate": 4.286078880326321e-06, "loss": 0.0006, "num_tokens": 40469465.0, "reward": -0.375, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.339774978160858, "sampling/importance_sampling_ratio/mean": 1.0003621935844422, "sampling/importance_sampling_ratio/min": 0.796686726808548, "sampling/sampling_logp_difference/max": 0.30299014449119566, "sampling/sampling_logp_difference/mean": 0.020472215954214334, "step": 9280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 206.125, "completions/mean_terminated_length": 206.125, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.0859864443540572, "epoch": 0.8540957984738439, "frac_reward_zero_std": 0.5, "grad_norm": 0.09730260074138641, "kl": 0.012205937394173815, "learning_rate": 4.284305173039697e-06, "loss": 0.0011, "num_tokens": 40513491.0, "reward": -0.475, "reward_std": 0.4956935167312622, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.49569355249404906, "sampling/importance_sampling_ratio/max": 1.2358090043067933, "sampling/importance_sampling_ratio/mean": 0.99974684715271, "sampling/importance_sampling_ratio/min": 0.786816543340683, "sampling/sampling_logp_difference/max": 0.2538063764572144, "sampling/sampling_logp_difference/mean": 0.02168538048863411, "step": 9290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.6, "completions/max_terminated_length": 273.6, "completions/mean_length": 190.0125, "completions/mean_terminated_length": 190.0125, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 0.9810887835919857, "epoch": 0.8550151696239772, "frac_reward_zero_std": 0.6, "grad_norm": 4.653404712677002, "kl": 0.011882903607329354, "learning_rate": 4.28252963312863e-06, "loss": -0.0, "num_tokens": 40554524.0, "reward": 0.2, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2662530422210694, "sampling/importance_sampling_ratio/mean": 0.9999871790409088, "sampling/importance_sampling_ratio/min": 0.7802515506744385, "sampling/sampling_logp_difference/max": 0.2725172400474548, "sampling/sampling_logp_difference/mean": 0.02011385690420866, "step": 9300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.2, "completions/max_terminated_length": 305.2, "completions/mean_length": 224.95, "completions/mean_terminated_length": 224.95, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.0662229508161545, "epoch": 0.8559345407741105, "frac_reward_zero_std": 0.5, "grad_norm": 5.997750759124756, "kl": 0.012497552571585403, "learning_rate": 4.280752262416743e-06, "loss": 0.0006, "num_tokens": 40596200.0, "reward": 0.1, "reward_std": 0.5058775067329406, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5058775067329406, "sampling/importance_sampling_ratio/max": 1.253900980949402, "sampling/importance_sampling_ratio/mean": 0.999490088224411, "sampling/importance_sampling_ratio/min": 0.7740092873573303, "sampling/sampling_logp_difference/max": 0.26535206437110903, "sampling/sampling_logp_difference/mean": 0.021304864436388016, "step": 9310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0875, "completions/max_length": 358.0, "completions/max_terminated_length": 353.9, "completions/mean_length": 266.0125, "completions/mean_terminated_length": 262.425, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 1.0258894417434932, "epoch": 0.8568539119242439, "frac_reward_zero_std": 0.4, "grad_norm": 2.601646900177002, "kl": 0.012547625275328756, "learning_rate": 4.2789730627295375e-06, "loss": 0.0007, "num_tokens": 40643769.0, "reward": -0.65, "reward_std": 0.45575389117002485, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.45575389713048936, "sampling/importance_sampling_ratio/max": 1.3197279930114747, "sampling/importance_sampling_ratio/mean": 0.9998461008071899, "sampling/importance_sampling_ratio/min": 0.7493658185005188, "sampling/sampling_logp_difference/max": 0.343079662322998, "sampling/sampling_logp_difference/mean": 0.019720896240323783, "step": 9320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.6, "completions/max_terminated_length": 320.6, "completions/mean_length": 206.8375, "completions/mean_terminated_length": 206.8375, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.1017692893743516, "epoch": 0.8577732830743772, "frac_reward_zero_std": 0.5, "grad_norm": 7.260909557342529, "kl": 0.012049544736510143, "learning_rate": 4.277192035894394e-06, "loss": 0.0008, "num_tokens": 40690604.0, "reward": 0.1, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.248613965511322, "sampling/importance_sampling_ratio/mean": 1.0000350415706634, "sampling/importance_sampling_ratio/min": 0.7468100249767303, "sampling/sampling_logp_difference/max": 0.2992702484130859, "sampling/sampling_logp_difference/mean": 0.021339874900877475, "step": 9330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 223.6, "completions/mean_terminated_length": 223.6, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.0679803416132927, "epoch": 0.8586926542245105, "frac_reward_zero_std": 0.3, "grad_norm": 5.616764068603516, "kl": 0.014230901229893789, "learning_rate": 4.275409183740571e-06, "loss": 0.0003, "num_tokens": 40735660.0, "reward": 0.06875, "reward_std": 0.6736144065856934, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.6736144423484802, "sampling/importance_sampling_ratio/max": 1.2792980551719666, "sampling/importance_sampling_ratio/mean": 1.00025731921196, "sampling/importance_sampling_ratio/min": 0.8023109674453736, "sampling/sampling_logp_difference/max": 0.2552531987428665, "sampling/sampling_logp_difference/mean": 0.021008102968335152, "step": 9340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.5, "completions/max_terminated_length": 332.5, "completions/mean_length": 230.05, "completions/mean_terminated_length": 230.05, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.0947725862264632, "epoch": 0.8596120253746438, "frac_reward_zero_std": 0.4, "grad_norm": 9.477141380310059, "kl": 0.01353483431157656, "learning_rate": 4.2736245080992e-06, "loss": 0.0004, "num_tokens": 40781776.0, "reward": -0.125, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.4086276769638062, "sampling/importance_sampling_ratio/mean": 0.9999428153038025, "sampling/importance_sampling_ratio/min": 0.7742278456687928, "sampling/sampling_logp_difference/max": 0.35217082500457764, "sampling/sampling_logp_difference/mean": 0.021426423452794552, "step": 9350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.7, "completions/max_terminated_length": 266.7, "completions/mean_length": 194.8375, "completions/mean_terminated_length": 194.8375, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.035982808470726, "epoch": 0.8605313965247771, "frac_reward_zero_std": 0.4, "grad_norm": 0.10680683702230453, "kl": 0.011760793637949973, "learning_rate": 4.271838010803285e-06, "loss": 0.0002, "num_tokens": 40822227.0, "reward": -0.2, "reward_std": 0.5875160038471222, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5875160157680511, "sampling/importance_sampling_ratio/max": 1.2574408292770385, "sampling/importance_sampling_ratio/mean": 1.0000138878822327, "sampling/importance_sampling_ratio/min": 0.7874206244945526, "sampling/sampling_logp_difference/max": 0.2666949987411499, "sampling/sampling_logp_difference/mean": 0.020099215023219586, "step": 9360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.0, "completions/max_terminated_length": 265.0, "completions/mean_length": 179.1, "completions/mean_terminated_length": 179.1, "completions/min_length": 114.6, "completions/min_terminated_length": 114.6, "entropy": 0.9882428172975779, "epoch": 0.8614507676749104, "frac_reward_zero_std": 0.7, "grad_norm": 0.1051802784204483, "kl": 0.012894728328683413, "learning_rate": 4.270049693687703e-06, "loss": 0.0007, "num_tokens": 40861179.0, "reward": -0.2, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.224350380897522, "sampling/importance_sampling_ratio/mean": 1.0002098321914672, "sampling/importance_sampling_ratio/min": 0.8084352314472198, "sampling/sampling_logp_difference/max": 0.22565038204193116, "sampling/sampling_logp_difference/mean": 0.020096934400498867, "step": 9370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.4, "completions/max_terminated_length": 299.4, "completions/mean_length": 220.15, "completions/mean_terminated_length": 220.15, "completions/min_length": 155.6, "completions/min_terminated_length": 155.6, "entropy": 1.0685037840157747, "epoch": 0.8623701388250437, "frac_reward_zero_std": 0.6, "grad_norm": 6.1336588859558105, "kl": 0.01244941539480351, "learning_rate": 4.268259558589199e-06, "loss": 0.0005, "num_tokens": 40903895.0, "reward": 0.075, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2915653228759765, "sampling/importance_sampling_ratio/mean": 0.9998454868793487, "sampling/importance_sampling_ratio/min": 0.8025399506092071, "sampling/sampling_logp_difference/max": 0.2671700954437256, "sampling/sampling_logp_difference/mean": 0.020693313144147397, "step": 9380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.9, "completions/max_terminated_length": 337.9, "completions/mean_length": 212.825, "completions/mean_terminated_length": 212.825, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 0.9851359441876412, "epoch": 0.863289509975177, "frac_reward_zero_std": 0.2, "grad_norm": 0.12484292685985565, "kl": 0.011633832799270749, "learning_rate": 4.266467607346385e-06, "loss": 0.0006, "num_tokens": 40943841.0, "reward": -0.075, "reward_std": 0.7330758512020111, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.7330758988857269, "sampling/importance_sampling_ratio/max": 1.2857784152030944, "sampling/importance_sampling_ratio/mean": 1.0003718912601471, "sampling/importance_sampling_ratio/min": 0.6936464488506318, "sampling/sampling_logp_difference/max": 0.3920463502407074, "sampling/sampling_logp_difference/mean": 0.020058882422745228, "step": 9390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 207.85, "completions/mean_terminated_length": 207.85, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.1397768624126912, "epoch": 0.8642088811253102, "frac_reward_zero_std": 0.3, "grad_norm": 5.894076824188232, "kl": 0.018785274855326863, "learning_rate": 4.264673841799738e-06, "loss": 0.0008, "num_tokens": 40986861.0, "reward": 0.225, "reward_std": 0.538717406988144, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.538717406988144, "sampling/importance_sampling_ratio/max": 1.341950821876526, "sampling/importance_sampling_ratio/mean": 0.9999549746513366, "sampling/importance_sampling_ratio/min": 0.7946701586246491, "sampling/sampling_logp_difference/max": 0.2974968671798706, "sampling/sampling_logp_difference/mean": 0.022362768836319446, "step": 9400 }, { "epoch": 0.8642088811253102, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 304.16230366492147, "eval_completions/max_terminated_length": 301.10296684118674, "eval_completions/mean_length": 208.2783595113438, "eval_completions/mean_terminated_length": 207.5465991700923, "eval_completions/min_length": 135.3996509598604, "eval_completions/min_terminated_length": 135.3996509598604, "eval_entropy": 1.1178450576609966, "eval_frac_reward_zero_std": 0.4712041884816754, "eval_kl": 0.01378022451246284, "eval_loss": 0.0006542524206452072, "eval_num_tokens": 40986861.0, "eval_reward": -0.1199825479930192, "eval_reward_std": 0.45652407288135244, "eval_rewards/reward_fn/mean": -0.1199825479930192, "eval_rewards/reward_fn/std": 0.45652408765248603, "eval_runtime": 1765.4973, "eval_samples_per_second": 0.325, "eval_sampling/importance_sampling_ratio/max": 1.2708078344038851, "eval_sampling/importance_sampling_ratio/mean": 0.9999822626771311, "eval_sampling/importance_sampling_ratio/min": 0.7692912818301618, "eval_sampling/sampling_logp_difference/max": 0.2929871873393733, "eval_sampling/sampling_logp_difference/mean": 0.021439828090759783, "eval_steps_per_second": 0.041, "step": 9400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 288.0, "completions/max_terminated_length": 277.7, "completions/mean_length": 191.775, "completions/mean_terminated_length": 189.0625, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.0214477904140948, "epoch": 0.8651282522754435, "frac_reward_zero_std": 0.7, "grad_norm": 0.16902276873588562, "kl": 0.012258631087024695, "learning_rate": 4.2628782637916e-06, "loss": 0.0005, "num_tokens": 41027947.0, "reward": -0.08125, "reward_std": 0.2612631618976593, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.26126316785812376, "sampling/importance_sampling_ratio/max": 1.2709553360939025, "sampling/importance_sampling_ratio/mean": 1.0001488983631135, "sampling/importance_sampling_ratio/min": 0.8061449110507966, "sampling/sampling_logp_difference/max": 0.2511605262756348, "sampling/sampling_logp_difference/mean": 0.020405074674636127, "step": 9410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 190.225, "completions/mean_terminated_length": 190.225, "completions/min_length": 102.2, "completions/min_terminated_length": 102.2, "entropy": 1.1250043250620365, "epoch": 0.866047623425577, "frac_reward_zero_std": 0.5, "grad_norm": 0.13066133856773376, "kl": 0.014909390255343168, "learning_rate": 4.261080875166173e-06, "loss": 0.0007, "num_tokens": 41070325.0, "reward": -0.125, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2398820519447327, "sampling/importance_sampling_ratio/mean": 0.9998952507972717, "sampling/importance_sampling_ratio/min": 0.7565064251422882, "sampling/sampling_logp_difference/max": 0.2979163765907288, "sampling/sampling_logp_difference/mean": 0.022244729846715928, "step": 9420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 216.275, "completions/mean_terminated_length": 216.275, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0000505484640598, "epoch": 0.8669669945757102, "frac_reward_zero_std": 0.4, "grad_norm": 0.1027241200208664, "kl": 0.012128865055274218, "learning_rate": 4.25928167776952e-06, "loss": 0.0006, "num_tokens": 41121811.0, "reward": -0.425, "reward_std": 0.46045787930488585, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.46045787930488585, "sampling/importance_sampling_ratio/max": 1.2835095286369325, "sampling/importance_sampling_ratio/mean": 1.0002802729606628, "sampling/importance_sampling_ratio/min": 0.7690848052501679, "sampling/sampling_logp_difference/max": 0.2817251682281494, "sampling/sampling_logp_difference/mean": 0.02007162496447563, "step": 9430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 286.5, "completions/max_terminated_length": 269.8, "completions/mean_length": 201.7875, "completions/mean_terminated_length": 198.91964416503907, "completions/min_length": 134.1, "completions/min_terminated_length": 134.1, "entropy": 1.1248772270977496, "epoch": 0.8678863657258435, "frac_reward_zero_std": 0.5, "grad_norm": 4.7596821784973145, "kl": 0.013726324849994854, "learning_rate": 4.2574806734495585e-06, "loss": 0.0008, "num_tokens": 41164946.0, "reward": -0.63125, "reward_std": 0.3879900202155113, "rewards/reward_fn/mean": -0.63125, "rewards/reward_fn/std": 0.38799004405736925, "sampling/importance_sampling_ratio/max": 1.2861182689666748, "sampling/importance_sampling_ratio/mean": 1.0001450419425963, "sampling/importance_sampling_ratio/min": 0.7563543498516083, "sampling/sampling_logp_difference/max": 0.3209580719470978, "sampling/sampling_logp_difference/mean": 0.021943685412406922, "step": 9440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 202.8, "completions/mean_terminated_length": 202.8, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.1349318958818913, "epoch": 0.8688057368759768, "frac_reward_zero_std": 0.7, "grad_norm": 0.12704035639762878, "kl": 0.01278592292801477, "learning_rate": 4.2556778640560684e-06, "loss": 0.0006, "num_tokens": 41210530.0, "reward": -0.3, "reward_std": 0.29206851720809934, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.29206851720809934, "sampling/importance_sampling_ratio/max": 1.2494293332099915, "sampling/importance_sampling_ratio/mean": 1.0004690349102021, "sampling/importance_sampling_ratio/min": 0.7734472572803497, "sampling/sampling_logp_difference/max": 0.2814633548259735, "sampling/sampling_logp_difference/mean": 0.02121050041168928, "step": 9450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 201.3875, "completions/mean_terminated_length": 201.3875, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.087150366604328, "epoch": 0.8697251080261101, "frac_reward_zero_std": 0.4, "grad_norm": 0.08864292502403259, "kl": 0.013705696223769336, "learning_rate": 4.253873251440675e-06, "loss": 0.0011, "num_tokens": 41253257.0, "reward": -0.30625, "reward_std": 0.5061943471431732, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5061943650245666, "sampling/importance_sampling_ratio/max": 1.2796146512031554, "sampling/importance_sampling_ratio/mean": 1.000315248966217, "sampling/importance_sampling_ratio/min": 0.7781864166259765, "sampling/sampling_logp_difference/max": 0.26923600435256956, "sampling/sampling_logp_difference/mean": 0.021445483900606632, "step": 9460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.7, "completions/max_terminated_length": 276.7, "completions/mean_length": 209.1375, "completions/mean_terminated_length": 209.1375, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.0442793197929858, "epoch": 0.8706444791762434, "frac_reward_zero_std": 0.7, "grad_norm": 5.67975378036499, "kl": 0.011219354334753007, "learning_rate": 4.252066837456863e-06, "loss": 0.0004, "num_tokens": 41298876.0, "reward": -0.0375, "reward_std": 0.2899480164051056, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.28994802832603456, "sampling/importance_sampling_ratio/max": 1.2525194525718688, "sampling/importance_sampling_ratio/mean": 0.9997061550617218, "sampling/importance_sampling_ratio/min": 0.7690974354743958, "sampling/sampling_logp_difference/max": 0.27766828536987304, "sampling/sampling_logp_difference/mean": 0.020051709190011025, "step": 9470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.7, "completions/max_terminated_length": 263.7, "completions/mean_length": 177.65, "completions/mean_terminated_length": 177.65, "completions/min_length": 114.8, "completions/min_terminated_length": 114.8, "entropy": 1.0293123967945577, "epoch": 0.8715638503263767, "frac_reward_zero_std": 0.4, "grad_norm": 6.224239826202393, "kl": 0.012973755650455132, "learning_rate": 4.250258623959964e-06, "loss": 0.0004, "num_tokens": 41338272.0, "reward": 0.0, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2313329100608825, "sampling/importance_sampling_ratio/mean": 0.9992869019508361, "sampling/importance_sampling_ratio/min": 0.7636358082294464, "sampling/sampling_logp_difference/max": 0.28920406103134155, "sampling/sampling_logp_difference/mean": 0.02119403313845396, "step": 9480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.4, "completions/max_terminated_length": 290.4, "completions/mean_length": 203.875, "completions/mean_terminated_length": 203.875, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0829306326806545, "epoch": 0.87248322147651, "frac_reward_zero_std": 0.7, "grad_norm": 6.267871856689453, "kl": 0.014898564520990476, "learning_rate": 4.248448612807158e-06, "loss": 0.0008, "num_tokens": 41381350.0, "reward": -0.2, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.2889012098312378, "sampling/importance_sampling_ratio/mean": 0.9998408317565918, "sampling/importance_sampling_ratio/min": 0.7678257882595062, "sampling/sampling_logp_difference/max": 0.3009048938751221, "sampling/sampling_logp_difference/mean": 0.021698074787855147, "step": 9490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 360.1, "completions/max_terminated_length": 360.1, "completions/mean_length": 222.175, "completions/mean_terminated_length": 222.175, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.1091829352080822, "epoch": 0.8734025926266433, "frac_reward_zero_std": 0.4, "grad_norm": 4.812692165374756, "kl": 0.014656672516139224, "learning_rate": 4.246636805857471e-06, "loss": 0.0003, "num_tokens": 41428708.0, "reward": -0.40625, "reward_std": 0.4869155645370483, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.48691557049751283, "sampling/importance_sampling_ratio/max": 1.2748552203178405, "sampling/importance_sampling_ratio/mean": 1.0005533754825593, "sampling/importance_sampling_ratio/min": 0.7714539170265198, "sampling/sampling_logp_difference/max": 0.292732834815979, "sampling/sampling_logp_difference/mean": 0.02146294340491295, "step": 9500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.4, "completions/max_terminated_length": 266.4, "completions/mean_length": 192.0, "completions/mean_terminated_length": 192.0, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.1175913028419018, "epoch": 0.8743219637767767, "frac_reward_zero_std": 0.3, "grad_norm": 5.591628551483154, "kl": 0.013466640951810405, "learning_rate": 4.244823204971773e-06, "loss": 0.0006, "num_tokens": 41470388.0, "reward": -0.075, "reward_std": 0.6480583846569061, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6480584084987641, "sampling/importance_sampling_ratio/max": 1.279826855659485, "sampling/importance_sampling_ratio/mean": 0.9999175369739532, "sampling/importance_sampling_ratio/min": 0.7253692507743835, "sampling/sampling_logp_difference/max": 0.37929381132125856, "sampling/sampling_logp_difference/mean": 0.021798209473490714, "step": 9510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.5, "completions/max_terminated_length": 320.5, "completions/mean_length": 209.7875, "completions/mean_terminated_length": 209.7875, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.0254456117749213, "epoch": 0.87524133492691, "frac_reward_zero_std": 0.3, "grad_norm": 4.897063732147217, "kl": 0.013817855034722016, "learning_rate": 4.24300781201278e-06, "loss": 0.0006, "num_tokens": 41511411.0, "reward": 0.09375, "reward_std": 0.5878328442573547, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.5878328740596771, "sampling/importance_sampling_ratio/max": 1.3004331946372987, "sampling/importance_sampling_ratio/mean": 1.0007537961006165, "sampling/importance_sampling_ratio/min": 0.8015347778797149, "sampling/sampling_logp_difference/max": 0.27206729650497435, "sampling/sampling_logp_difference/mean": 0.020880899019539358, "step": 9520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 220.6, "completions/mean_terminated_length": 220.6, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 1.1239751264452935, "epoch": 0.8761607060770433, "frac_reward_zero_std": 0.9, "grad_norm": 0.10313112288713455, "kl": 0.012625037803081796, "learning_rate": 4.2411906288450446e-06, "loss": 0.0006, "num_tokens": 41558899.0, "reward": 0.25, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.2585035920143128, "sampling/importance_sampling_ratio/mean": 0.9997806251049042, "sampling/importance_sampling_ratio/min": 0.759350574016571, "sampling/sampling_logp_difference/max": 0.2849051356315613, "sampling/sampling_logp_difference/mean": 0.021224210783839226, "step": 9530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.6, "completions/max_terminated_length": 282.6, "completions/mean_length": 197.375, "completions/mean_terminated_length": 197.375, "completions/min_length": 121.2, "completions/min_terminated_length": 121.2, "entropy": 1.0184436745941639, "epoch": 0.8770800772271766, "frac_reward_zero_std": 0.4, "grad_norm": 5.85416841506958, "kl": 0.012690801458666101, "learning_rate": 4.23937165733496e-06, "loss": 0.0006, "num_tokens": 41598353.0, "reward": 0.025, "reward_std": 0.5841213405132294, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5841213643550873, "sampling/importance_sampling_ratio/max": 1.2786506056785583, "sampling/importance_sampling_ratio/mean": 1.0008072555065155, "sampling/importance_sampling_ratio/min": 0.7840060293674469, "sampling/sampling_logp_difference/max": 0.27755733132362365, "sampling/sampling_logp_difference/mean": 0.020149990729987622, "step": 9540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 199.525, "completions/mean_terminated_length": 199.525, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.1305587790906428, "epoch": 0.8779994483773099, "frac_reward_zero_std": 0.5, "grad_norm": 0.14015160501003265, "kl": 0.014332179387565702, "learning_rate": 4.237550899350755e-06, "loss": 0.0008, "num_tokens": 41641555.0, "reward": -0.28125, "reward_std": 0.3879900202155113, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.38799004405736925, "sampling/importance_sampling_ratio/max": 1.2467270374298096, "sampling/importance_sampling_ratio/mean": 1.000416213274002, "sampling/importance_sampling_ratio/min": 0.7685997307300567, "sampling/sampling_logp_difference/max": 0.2680966377258301, "sampling/sampling_logp_difference/mean": 0.022024156339466573, "step": 9550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 199.275, "completions/mean_terminated_length": 199.275, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.022459152340889, "epoch": 0.8789188195274432, "frac_reward_zero_std": 0.2, "grad_norm": 5.585063457489014, "kl": 0.012183271886897273, "learning_rate": 4.235728356762496e-06, "loss": 0.0008, "num_tokens": 41682849.0, "reward": -0.325, "reward_std": 0.664098572731018, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.664098584651947, "sampling/importance_sampling_ratio/max": 1.2485016942024232, "sampling/importance_sampling_ratio/mean": 0.9998503208160401, "sampling/importance_sampling_ratio/min": 0.7473591148853302, "sampling/sampling_logp_difference/max": 0.30840687155723573, "sampling/sampling_logp_difference/mean": 0.020419451408088207, "step": 9560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 184.15, "completions/mean_terminated_length": 184.15, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.0129288025200367, "epoch": 0.8798381906775765, "frac_reward_zero_std": 0.9, "grad_norm": 0.16601814329624176, "kl": 0.012892916263081134, "learning_rate": 4.2339040314420785e-06, "loss": 0.0006, "num_tokens": 41720981.0, "reward": 0.125, "reward_std": 0.10350983142852783, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.10350984334945679, "sampling/importance_sampling_ratio/max": 1.2907332897186279, "sampling/importance_sampling_ratio/mean": 0.9997830748558044, "sampling/importance_sampling_ratio/min": 0.715716329216957, "sampling/sampling_logp_difference/max": 0.3621420621871948, "sampling/sampling_logp_difference/mean": 0.020247426815330982, "step": 9570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 203.2125, "completions/mean_terminated_length": 203.2125, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0720893174409867, "epoch": 0.8807575618277098, "frac_reward_zero_std": 0.6, "grad_norm": 0.11259850114583969, "kl": 0.013155275018652901, "learning_rate": 4.232077925263231e-06, "loss": 0.0006, "num_tokens": 41764542.0, "reward": -0.625, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2568911552429198, "sampling/importance_sampling_ratio/mean": 1.0002195596694947, "sampling/importance_sampling_ratio/min": 0.7920534014701843, "sampling/sampling_logp_difference/max": 0.24945040345191954, "sampling/sampling_logp_difference/mean": 0.020420567505061625, "step": 9580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 216.825, "completions/mean_terminated_length": 216.825, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.1203306689858437, "epoch": 0.8816769329778431, "frac_reward_zero_std": 0.4, "grad_norm": 5.35356330871582, "kl": 0.013244663062505425, "learning_rate": 4.230250040101511e-06, "loss": 0.0003, "num_tokens": 41809336.0, "reward": -0.025, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.310008203983307, "sampling/importance_sampling_ratio/mean": 1.0004303455352783, "sampling/importance_sampling_ratio/min": 0.7932311296463013, "sampling/sampling_logp_difference/max": 0.28424692153930664, "sampling/sampling_logp_difference/mean": 0.02151360586285591, "step": 9590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 208.075, "completions/mean_terminated_length": 208.075, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.0415862828493119, "epoch": 0.8825963041279765, "frac_reward_zero_std": 0.6, "grad_norm": 0.11437294632196426, "kl": 0.011833914188900962, "learning_rate": 4.2284203778343035e-06, "loss": 0.0009, "num_tokens": 41853614.0, "reward": 0.375, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.3295745253562927, "sampling/importance_sampling_ratio/mean": 0.9994451403617859, "sampling/importance_sampling_ratio/min": 0.8004082441329956, "sampling/sampling_logp_difference/max": 0.28709366321563723, "sampling/sampling_logp_difference/mean": 0.02034113146364689, "step": 9600 }, { "epoch": 0.8825963041279765, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 302.2286212914485, "eval_completions/max_terminated_length": 299.6614310645724, "eval_completions/mean_length": 209.489092495637, "eval_completions/mean_terminated_length": 208.64636866917368, "eval_completions/min_length": 135.1221640488656, "eval_completions/min_terminated_length": 135.1221640488656, "eval_entropy": 1.0988744190954205, "eval_frac_reward_zero_std": 0.4572425828970332, "eval_kl": 0.012775648690851155, "eval_loss": 0.000649746973067522, "eval_num_tokens": 41853614.0, "eval_reward": -0.12969022687609075, "eval_reward_std": 0.4833485728315034, "eval_rewards/reward_fn/mean": -0.12969022687609075, "eval_rewards/reward_fn/std": 0.48334858947503423, "eval_runtime": 1754.8544, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2730154134008065, "eval_sampling/importance_sampling_ratio/mean": 0.999966192828007, "eval_sampling/importance_sampling_ratio/min": 0.7780854427377591, "eval_sampling/sampling_logp_difference/max": 0.28386186653614875, "eval_sampling/sampling_logp_difference/mean": 0.02094105439972503, "eval_steps_per_second": 0.041, "step": 9600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 347.8, "completions/max_terminated_length": 321.5, "completions/mean_length": 213.3625, "completions/mean_terminated_length": 209.09464416503906, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0937266290187835, "epoch": 0.8835156752781098, "frac_reward_zero_std": 0.5, "grad_norm": 4.586399555206299, "kl": 0.013600053958361969, "learning_rate": 4.226588940340818e-06, "loss": 0.001, "num_tokens": 41896019.0, "reward": -0.45625, "reward_std": 0.4643930375576019, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.4643930494785309, "sampling/importance_sampling_ratio/max": 1.23620947599411, "sampling/importance_sampling_ratio/mean": 1.000060886144638, "sampling/importance_sampling_ratio/min": 0.779224282503128, "sampling/sampling_logp_difference/max": 0.25545597076416016, "sampling/sampling_logp_difference/mean": 0.021192287653684617, "step": 9610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.0, "completions/max_terminated_length": 271.0, "completions/mean_length": 200.9625, "completions/mean_terminated_length": 200.9625, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.0503280691802501, "epoch": 0.8844350464282431, "frac_reward_zero_std": 0.8, "grad_norm": 5.859262943267822, "kl": 0.011484472051961347, "learning_rate": 4.224755729502088e-06, "loss": 0.0007, "num_tokens": 41940192.0, "reward": -0.15, "reward_std": 0.20701966285705567, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.20701968669891357, "sampling/importance_sampling_ratio/max": 1.2784010291099548, "sampling/importance_sampling_ratio/mean": 1.0000763714313508, "sampling/importance_sampling_ratio/min": 0.7774456679821015, "sampling/sampling_logp_difference/max": 0.27693238854408264, "sampling/sampling_logp_difference/mean": 0.02082743253558874, "step": 9620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.4, "completions/max_terminated_length": 321.4, "completions/mean_length": 219.1125, "completions/mean_terminated_length": 219.1125, "completions/min_length": 152.3, "completions/min_terminated_length": 152.3, "entropy": 1.0275631800293923, "epoch": 0.8853544175783764, "frac_reward_zero_std": 0.6, "grad_norm": 4.696669578552246, "kl": 0.01372640259214677, "learning_rate": 4.222920747200967e-06, "loss": 0.0005, "num_tokens": 41985889.0, "reward": 0.15, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.255465042591095, "sampling/importance_sampling_ratio/mean": 1.0001073002815246, "sampling/importance_sampling_ratio/min": 0.7948961734771729, "sampling/sampling_logp_difference/max": 0.2462047815322876, "sampling/sampling_logp_difference/mean": 0.02045886144042015, "step": 9630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 301.6, "completions/max_terminated_length": 282.0, "completions/mean_length": 204.3125, "completions/mean_terminated_length": 201.12857360839843, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.1203458964824677, "epoch": 0.8862737887285097, "frac_reward_zero_std": 0.1, "grad_norm": 5.51729679107666, "kl": 0.011950867052655667, "learning_rate": 4.221083995322131e-06, "loss": 0.0009, "num_tokens": 42026954.0, "reward": -0.25625, "reward_std": 0.7729968667030335, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.7729968965053559, "sampling/importance_sampling_ratio/max": 1.2780627489089966, "sampling/importance_sampling_ratio/mean": 1.0000020205974578, "sampling/importance_sampling_ratio/min": 0.7992276847362518, "sampling/sampling_logp_difference/max": 0.26778675317764283, "sampling/sampling_logp_difference/mean": 0.021610469184815885, "step": 9640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 223.7875, "completions/mean_terminated_length": 223.7875, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.1012702770531178, "epoch": 0.887193159878643, "frac_reward_zero_std": 0.3, "grad_norm": 0.1031562089920044, "kl": 0.012227150338003413, "learning_rate": 4.219245475752069e-06, "loss": 0.0004, "num_tokens": 42073257.0, "reward": -0.175, "reward_std": 0.6152592301368713, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6152592420578002, "sampling/importance_sampling_ratio/max": 1.250788974761963, "sampling/importance_sampling_ratio/mean": 0.9994693100452423, "sampling/importance_sampling_ratio/min": 0.7511912703514099, "sampling/sampling_logp_difference/max": 0.31445457339286803, "sampling/sampling_logp_difference/mean": 0.02158159911632538, "step": 9650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 218.05, "completions/mean_terminated_length": 218.05, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.0608809441328049, "epoch": 0.8881125310287763, "frac_reward_zero_std": 0.4, "grad_norm": 5.9746294021606445, "kl": 0.011982587684178725, "learning_rate": 4.217405190379089e-06, "loss": 0.0006, "num_tokens": 42116877.0, "reward": -0.375, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.2380985140800476, "sampling/importance_sampling_ratio/mean": 1.0002872407436372, "sampling/importance_sampling_ratio/min": 0.7663156151771545, "sampling/sampling_logp_difference/max": 0.26736300587654116, "sampling/sampling_logp_difference/mean": 0.020570875331759452, "step": 9660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.5, "completions/max_terminated_length": 333.5, "completions/mean_length": 239.65, "completions/mean_terminated_length": 239.65, "completions/min_length": 155.4, "completions/min_terminated_length": 155.4, "entropy": 1.144056434184313, "epoch": 0.8890319021789096, "frac_reward_zero_std": 0.6, "grad_norm": 5.332046985626221, "kl": 0.01368372081196867, "learning_rate": 4.215563141093312e-06, "loss": 0.0009, "num_tokens": 42165441.0, "reward": -0.3, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2724552750587463, "sampling/importance_sampling_ratio/mean": 0.9997852504253387, "sampling/importance_sampling_ratio/min": 0.7828254759311676, "sampling/sampling_logp_difference/max": 0.27671523243188856, "sampling/sampling_logp_difference/mean": 0.021492741629481317, "step": 9670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.1, "completions/max_terminated_length": 272.1, "completions/mean_length": 202.475, "completions/mean_terminated_length": 202.475, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.0348761811852456, "epoch": 0.8899512733290429, "frac_reward_zero_std": 0.6, "grad_norm": 5.441434860229492, "kl": 0.0122612448874861, "learning_rate": 4.213719329786669e-06, "loss": 0.0005, "num_tokens": 42206631.0, "reward": 0.175, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2383183717727662, "sampling/importance_sampling_ratio/mean": 1.0001991927623748, "sampling/importance_sampling_ratio/min": 0.7942554652690887, "sampling/sampling_logp_difference/max": 0.2458705186843872, "sampling/sampling_logp_difference/mean": 0.020978409983217716, "step": 9680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 215.15, "completions/mean_terminated_length": 215.15, "completions/min_length": 159.4, "completions/min_terminated_length": 159.4, "entropy": 1.0491121105849743, "epoch": 0.8908706444791762, "frac_reward_zero_std": 0.5, "grad_norm": 8.878660202026367, "kl": 0.013528053706977516, "learning_rate": 4.211873758352905e-06, "loss": 0.0008, "num_tokens": 42253171.0, "reward": -0.175, "reward_std": 0.48475000262260437, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4847500503063202, "sampling/importance_sampling_ratio/max": 1.3063908815383911, "sampling/importance_sampling_ratio/mean": 1.0002971947193147, "sampling/importance_sampling_ratio/min": 0.8052680850028991, "sampling/sampling_logp_difference/max": 0.2757500171661377, "sampling/sampling_logp_difference/mean": 0.02025048416107893, "step": 9690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 212.05, "completions/mean_terminated_length": 212.05, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.1145124427974225, "epoch": 0.8917900156293096, "frac_reward_zero_std": 0.5, "grad_norm": 9.041208267211914, "kl": 0.013038315193261952, "learning_rate": 4.2100264286875655e-06, "loss": 0.0002, "num_tokens": 42298927.0, "reward": -0.35625, "reward_std": 0.3770622000098228, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.3770622119307518, "sampling/importance_sampling_ratio/max": 1.3083243727684022, "sampling/importance_sampling_ratio/mean": 0.9999693334102631, "sampling/importance_sampling_ratio/min": 0.760411936044693, "sampling/sampling_logp_difference/max": 0.3154757857322693, "sampling/sampling_logp_difference/mean": 0.021753753535449505, "step": 9700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.4, "completions/max_terminated_length": 306.4, "completions/mean_length": 219.9125, "completions/mean_terminated_length": 219.9125, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 0.9792264573276043, "epoch": 0.8927093867794429, "frac_reward_zero_std": 0.8, "grad_norm": 0.13263283669948578, "kl": 0.012379392544971779, "learning_rate": 4.208177342688009e-06, "loss": 0.0005, "num_tokens": 42345608.0, "reward": -0.15, "reward_std": 0.19948650598526002, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.19948650598526002, "sampling/importance_sampling_ratio/max": 1.2635570764541626, "sampling/importance_sampling_ratio/mean": 1.0000475287437438, "sampling/importance_sampling_ratio/min": 0.7395309031009674, "sampling/sampling_logp_difference/max": 0.31907384991645815, "sampling/sampling_logp_difference/mean": 0.019541067630052568, "step": 9710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.6, "completions/max_terminated_length": 315.6, "completions/mean_length": 223.6875, "completions/mean_terminated_length": 223.6875, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.1294842675328254, "epoch": 0.8936287579295762, "frac_reward_zero_std": 0.5, "grad_norm": 0.1231222003698349, "kl": 0.013621494133258239, "learning_rate": 4.206326502253395e-06, "loss": 0.0006, "num_tokens": 42391871.0, "reward": -0.075, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2503790616989137, "sampling/importance_sampling_ratio/mean": 0.9997543692588806, "sampling/importance_sampling_ratio/min": 0.7689642369747162, "sampling/sampling_logp_difference/max": 0.2808941245079041, "sampling/sampling_logp_difference/mean": 0.021539823338389396, "step": 9720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 202.6, "completions/mean_terminated_length": 202.6, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.0644588619470596, "epoch": 0.8945481290797095, "frac_reward_zero_std": 0.7, "grad_norm": 6.22776460647583, "kl": 0.012958214484388008, "learning_rate": 4.204473909284683e-06, "loss": 0.0003, "num_tokens": 42432375.0, "reward": -0.225, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.3088894009590148, "sampling/importance_sampling_ratio/mean": 1.000408935546875, "sampling/importance_sampling_ratio/min": 0.7405220627784729, "sampling/sampling_logp_difference/max": 0.3421663880348206, "sampling/sampling_logp_difference/mean": 0.021268545649945736, "step": 9730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.3, "completions/max_terminated_length": 310.3, "completions/mean_length": 216.0625, "completions/mean_terminated_length": 216.0625, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.121429083496332, "epoch": 0.8954675002298428, "frac_reward_zero_std": 0.4, "grad_norm": 0.1185220330953598, "kl": 0.012687655206536873, "learning_rate": 4.202619565684637e-06, "loss": 0.0006, "num_tokens": 42478756.0, "reward": -0.05, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2556814551353455, "sampling/importance_sampling_ratio/mean": 0.9997085869312287, "sampling/importance_sampling_ratio/min": 0.7144069790840148, "sampling/sampling_logp_difference/max": 0.3506688058376312, "sampling/sampling_logp_difference/mean": 0.02191872037947178, "step": 9740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.5, "completions/max_terminated_length": 285.5, "completions/mean_length": 193.0, "completions/mean_terminated_length": 193.0, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.0082319222390652, "epoch": 0.8963868713799761, "frac_reward_zero_std": 0.5, "grad_norm": 0.15611813962459564, "kl": 0.013091642066137865, "learning_rate": 4.200763473357815e-06, "loss": 0.0007, "num_tokens": 42520436.0, "reward": -0.1625, "reward_std": 0.4165701389312744, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.4165701508522034, "sampling/importance_sampling_ratio/max": 1.2566653728485107, "sampling/importance_sampling_ratio/mean": 1.0009639620780946, "sampling/importance_sampling_ratio/min": 0.8061083495616913, "sampling/sampling_logp_difference/max": 0.2510338068008423, "sampling/sampling_logp_difference/mean": 0.020529049262404443, "step": 9750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.9, "completions/max_terminated_length": 292.9, "completions/mean_length": 211.3875, "completions/mean_terminated_length": 211.3875, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0765687823295593, "epoch": 0.8973062425301094, "frac_reward_zero_std": 0.6, "grad_norm": 4.571231365203857, "kl": 0.013524802320171147, "learning_rate": 4.198905634210574e-06, "loss": 0.001, "num_tokens": 42565067.0, "reward": -0.125, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2781299948692322, "sampling/importance_sampling_ratio/mean": 1.0000158727169037, "sampling/importance_sampling_ratio/min": 0.7878073632717133, "sampling/sampling_logp_difference/max": 0.25305579900741576, "sampling/sampling_logp_difference/mean": 0.021288704685866834, "step": 9760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.0, "completions/max_terminated_length": 281.0, "completions/mean_length": 195.6375, "completions/mean_terminated_length": 195.6375, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0503267630934716, "epoch": 0.8982256136802427, "frac_reward_zero_std": 0.6, "grad_norm": 0.11161709576845169, "kl": 0.012849222536897287, "learning_rate": 4.197046050151061e-06, "loss": 0.0003, "num_tokens": 42604182.0, "reward": -0.35, "reward_std": 0.2828427076339722, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.2828427076339722, "sampling/importance_sampling_ratio/max": 1.3016541957855225, "sampling/importance_sampling_ratio/mean": 1.0003121495246887, "sampling/importance_sampling_ratio/min": 0.7665340006351471, "sampling/sampling_logp_difference/max": 0.30095781981945036, "sampling/sampling_logp_difference/mean": 0.02132515013217926, "step": 9770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 196.8875, "completions/mean_terminated_length": 196.8875, "completions/min_length": 114.5, "completions/min_terminated_length": 114.5, "entropy": 1.0708158880472183, "epoch": 0.899144984830376, "frac_reward_zero_std": 0.6, "grad_norm": 0.1354639232158661, "kl": 0.013832597702275962, "learning_rate": 4.19518472308922e-06, "loss": 0.0007, "num_tokens": 42644149.0, "reward": 0.425, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.2889424443244935, "sampling/importance_sampling_ratio/mean": 0.9998552441596985, "sampling/importance_sampling_ratio/min": 0.7741113662719726, "sampling/sampling_logp_difference/max": 0.2997929841279984, "sampling/sampling_logp_difference/mean": 0.021681608632206917, "step": 9780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 204.2375, "completions/mean_terminated_length": 204.2375, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0588064894080162, "epoch": 0.9000643559805094, "frac_reward_zero_std": 0.5, "grad_norm": 9.118574142456055, "kl": 0.014294803218217567, "learning_rate": 4.193321654936784e-06, "loss": 0.0007, "num_tokens": 42684104.0, "reward": -0.425, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2602597951889039, "sampling/importance_sampling_ratio/mean": 1.0002468883991242, "sampling/importance_sampling_ratio/min": 0.7909176647663116, "sampling/sampling_logp_difference/max": 0.24242277145385743, "sampling/sampling_logp_difference/mean": 0.021819408982992172, "step": 9790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.3, "completions/max_terminated_length": 314.3, "completions/mean_length": 211.8, "completions/mean_terminated_length": 211.8, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.0993777729570866, "epoch": 0.9009837271306427, "frac_reward_zero_std": 0.9, "grad_norm": 0.10068485140800476, "kl": 0.014061538490932435, "learning_rate": 4.191456847607271e-06, "loss": 0.0009, "num_tokens": 42726104.0, "reward": -0.55, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.2719435811042785, "sampling/importance_sampling_ratio/mean": 1.0002134203910829, "sampling/importance_sampling_ratio/min": 0.7852752864360809, "sampling/sampling_logp_difference/max": 0.2689140856266022, "sampling/sampling_logp_difference/mean": 0.022439641505479814, "step": 9800 }, { "epoch": 0.9009837271306427, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 294.6020942408377, "eval_completions/max_terminated_length": 292.5410122164049, "eval_completions/mean_length": 205.26657940663176, "eval_completions/mean_terminated_length": 204.78424969684806, "eval_completions/min_length": 130.90924956369983, "eval_completions/min_terminated_length": 130.90924956369983, "eval_entropy": 1.0658454161143844, "eval_frac_reward_zero_std": 0.46596858638743455, "eval_kl": 0.012391853963803455, "eval_loss": 0.0006414640229195356, "eval_num_tokens": 42726104.0, "eval_reward": -0.12718150087260036, "eval_reward_std": 0.4705223036218064, "eval_rewards/reward_fn/mean": -0.12718150087260036, "eval_rewards/reward_fn/std": 0.4705223177688076, "eval_runtime": 1716.1421, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2718837652322925, "eval_sampling/importance_sampling_ratio/mean": 1.0000518246262902, "eval_sampling/importance_sampling_ratio/min": 0.7777804651445536, "eval_sampling/sampling_logp_difference/max": 0.28259797854573315, "eval_sampling/sampling_logp_difference/mean": 0.020613478408902104, "eval_steps_per_second": 0.042, "step": 9800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 213.825, "completions/mean_terminated_length": 213.825, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0444575045257807, "epoch": 0.901903098280776, "frac_reward_zero_std": 0.5, "grad_norm": 0.11462917178869247, "kl": 0.01323149292729795, "learning_rate": 4.189590303015989e-06, "loss": 0.0007, "num_tokens": 42772514.0, "reward": -0.35, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2432184219360352, "sampling/importance_sampling_ratio/mean": 0.9998831748962402, "sampling/importance_sampling_ratio/min": 0.7639705777168274, "sampling/sampling_logp_difference/max": 0.2880393922328949, "sampling/sampling_logp_difference/mean": 0.02058042334392667, "step": 9810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.6, "completions/max_terminated_length": 319.6, "completions/mean_length": 210.6875, "completions/mean_terminated_length": 210.6875, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 0.9514480959624052, "epoch": 0.9028224694309093, "frac_reward_zero_std": 0.5, "grad_norm": 0.09970615059137344, "kl": 0.01132732086116448, "learning_rate": 4.187722023080028e-06, "loss": 0.0007, "num_tokens": 42813809.0, "reward": 0.0, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.264959728717804, "sampling/importance_sampling_ratio/mean": 0.999555516242981, "sampling/importance_sampling_ratio/min": 0.7805728733539581, "sampling/sampling_logp_difference/max": 0.26745531558990476, "sampling/sampling_logp_difference/mean": 0.019769147783517838, "step": 9820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.9, "completions/max_terminated_length": 304.9, "completions/mean_length": 212.65, "completions/mean_terminated_length": 212.65, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.1224188640713693, "epoch": 0.9037418405810426, "frac_reward_zero_std": 0.3, "grad_norm": 0.11106351017951965, "kl": 0.012111852323869243, "learning_rate": 4.1858520097182635e-06, "loss": 0.0006, "num_tokens": 42858397.0, "reward": -0.1, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2450867295265198, "sampling/importance_sampling_ratio/mean": 0.9999409794807435, "sampling/importance_sampling_ratio/min": 0.7635382831096649, "sampling/sampling_logp_difference/max": 0.2786913812160492, "sampling/sampling_logp_difference/mean": 0.02184486836194992, "step": 9830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.4, "completions/max_terminated_length": 304.4, "completions/mean_length": 208.25, "completions/mean_terminated_length": 208.25, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 0.994593895971775, "epoch": 0.9046612117311759, "frac_reward_zero_std": 0.6, "grad_norm": 6.653997421264648, "kl": 0.012556693982332945, "learning_rate": 4.183980264851345e-06, "loss": 0.0006, "num_tokens": 42901433.0, "reward": -0.35, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2709911823272706, "sampling/importance_sampling_ratio/mean": 0.9997059643268585, "sampling/importance_sampling_ratio/min": 0.7827220678329467, "sampling/sampling_logp_difference/max": 0.277280730009079, "sampling/sampling_logp_difference/mean": 0.020361056365072728, "step": 9840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.7, "completions/max_terminated_length": 306.7, "completions/mean_length": 216.025, "completions/mean_terminated_length": 216.025, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "entropy": 1.141061294078827, "epoch": 0.9055805828813092, "frac_reward_zero_std": 0.4, "grad_norm": 0.10001188516616821, "kl": 0.014090858225245029, "learning_rate": 4.182106790401708e-06, "loss": 0.0005, "num_tokens": 42947163.0, "reward": -0.325, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.258895456790924, "sampling/importance_sampling_ratio/mean": 1.000214058160782, "sampling/importance_sampling_ratio/min": 0.7883702754974365, "sampling/sampling_logp_difference/max": 0.2593182325363159, "sampling/sampling_logp_difference/mean": 0.02213544175028801, "step": 9850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 195.3125, "completions/mean_terminated_length": 195.3125, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0623713336884975, "epoch": 0.9064999540314425, "frac_reward_zero_std": 0.6, "grad_norm": 0.12905265390872955, "kl": 0.011333338438998908, "learning_rate": 4.1802315882935605e-06, "loss": 0.0007, "num_tokens": 42990100.0, "reward": -0.225, "reward_std": 0.4099008321762085, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.40990084409713745, "sampling/importance_sampling_ratio/max": 1.2736912488937377, "sampling/importance_sampling_ratio/mean": 1.0001018643379211, "sampling/importance_sampling_ratio/min": 0.8005401253700256, "sampling/sampling_logp_difference/max": 0.2463274836540222, "sampling/sampling_logp_difference/mean": 0.021272101253271104, "step": 9860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.7, "completions/max_terminated_length": 318.7, "completions/mean_length": 232.0375, "completions/mean_terminated_length": 232.0375, "completions/min_length": 154.3, "completions/min_terminated_length": 154.3, "entropy": 1.0941888131201267, "epoch": 0.9074193251815758, "frac_reward_zero_std": 0.2, "grad_norm": 4.709369659423828, "kl": 0.011914685886586085, "learning_rate": 4.178354660452885e-06, "loss": 0.0004, "num_tokens": 43039663.0, "reward": 0.175, "reward_std": 0.7624960362911224, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.7624960839748383, "sampling/importance_sampling_ratio/max": 1.2737467288970947, "sampling/importance_sampling_ratio/mean": 1.0000319063663483, "sampling/importance_sampling_ratio/min": 0.7594300627708435, "sampling/sampling_logp_difference/max": 0.2909351706504822, "sampling/sampling_logp_difference/mean": 0.021561730466783047, "step": 9870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.6, "completions/max_terminated_length": 267.6, "completions/mean_length": 191.6125, "completions/mean_terminated_length": 191.6125, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.018044298887253, "epoch": 0.9083386963317092, "frac_reward_zero_std": 0.5, "grad_norm": 0.18088793754577637, "kl": 0.01100357886753045, "learning_rate": 4.176476008807435e-06, "loss": 0.0005, "num_tokens": 43079984.0, "reward": -0.175, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2976246356964112, "sampling/importance_sampling_ratio/mean": 1.000465887784958, "sampling/importance_sampling_ratio/min": 0.7479133546352387, "sampling/sampling_logp_difference/max": 0.33148305416107177, "sampling/sampling_logp_difference/mean": 0.020699850656092166, "step": 9880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.9, "completions/max_terminated_length": 281.9, "completions/mean_length": 209.2625, "completions/mean_terminated_length": 209.2625, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.158181793242693, "epoch": 0.9092580674818425, "frac_reward_zero_std": 0.4, "grad_norm": 6.159550666809082, "kl": 0.012121203407878057, "learning_rate": 4.174595635286739e-06, "loss": 0.0005, "num_tokens": 43126797.0, "reward": -0.025, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.249333918094635, "sampling/importance_sampling_ratio/mean": 1.0001055359840394, "sampling/importance_sampling_ratio/min": 0.7764723420143127, "sampling/sampling_logp_difference/max": 0.25966774225234984, "sampling/sampling_logp_difference/mean": 0.022444486804306506, "step": 9890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.2, "completions/max_terminated_length": 303.2, "completions/mean_length": 212.55, "completions/mean_terminated_length": 212.55, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.084275595843792, "epoch": 0.9101774386319758, "frac_reward_zero_std": 0.4, "grad_norm": 7.679750442504883, "kl": 0.014772210892988368, "learning_rate": 4.17271354182209e-06, "loss": 0.0005, "num_tokens": 43171825.0, "reward": -0.125, "reward_std": 0.5773320138454437, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5773320615291595, "sampling/importance_sampling_ratio/max": 1.2418003320693969, "sampling/importance_sampling_ratio/mean": 0.9998679757118225, "sampling/importance_sampling_ratio/min": 0.7864194452762604, "sampling/sampling_logp_difference/max": 0.25828573703765867, "sampling/sampling_logp_difference/mean": 0.020814811810851097, "step": 9900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.4, "completions/max_terminated_length": 257.4, "completions/mean_length": 190.0125, "completions/mean_terminated_length": 190.0125, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0402702633291483, "epoch": 0.911096809782109, "frac_reward_zero_std": 0.5, "grad_norm": 0.10965149104595184, "kl": 0.01178727098449599, "learning_rate": 4.170829730346549e-06, "loss": 0.0003, "num_tokens": 43210746.0, "reward": 0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2478034138679504, "sampling/importance_sampling_ratio/mean": 0.9992958724498748, "sampling/importance_sampling_ratio/min": 0.7731986105442047, "sampling/sampling_logp_difference/max": 0.27292357087135316, "sampling/sampling_logp_difference/mean": 0.0211125111207366, "step": 9910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 206.2625, "completions/mean_terminated_length": 206.2625, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.0419825196266175, "epoch": 0.9120161809322423, "frac_reward_zero_std": 0.3, "grad_norm": 4.548367977142334, "kl": 0.012233923820895143, "learning_rate": 4.168944202794942e-06, "loss": 0.0004, "num_tokens": 43253111.0, "reward": 0.475, "reward_std": 0.5277738928794861, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.5277739048004151, "sampling/importance_sampling_ratio/max": 1.25494704246521, "sampling/importance_sampling_ratio/mean": 1.0001584112644195, "sampling/importance_sampling_ratio/min": 0.7737285912036895, "sampling/sampling_logp_difference/max": 0.2682157516479492, "sampling/sampling_logp_difference/mean": 0.020374906342476608, "step": 9920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.0, "completions/max_terminated_length": 278.0, "completions/mean_length": 185.125, "completions/mean_terminated_length": 185.125, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.0577528722584248, "epoch": 0.9129355520823756, "frac_reward_zero_std": 0.2, "grad_norm": 6.536627292633057, "kl": 0.013295302464393898, "learning_rate": 4.167056961103857e-06, "loss": 0.0004, "num_tokens": 43295097.0, "reward": -0.35, "reward_std": 0.7255583882331849, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.7255583882331849, "sampling/importance_sampling_ratio/max": 1.2637118220329284, "sampling/importance_sampling_ratio/mean": 0.9997943699359894, "sampling/importance_sampling_ratio/min": 0.7911862373352051, "sampling/sampling_logp_difference/max": 0.2615497350692749, "sampling/sampling_logp_difference/mean": 0.021569762378931046, "step": 9930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 339.3, "completions/max_terminated_length": 323.2, "completions/mean_length": 219.175, "completions/mean_terminated_length": 216.31964416503905, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.1012148641049861, "epoch": 0.9138549232325089, "frac_reward_zero_std": 0.5, "grad_norm": 0.10916528105735779, "kl": 0.01237546139745973, "learning_rate": 4.165168007211641e-06, "loss": 0.0008, "num_tokens": 43339191.0, "reward": 0.01875, "reward_std": 0.36951334923505785, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.36951336115598676, "sampling/importance_sampling_ratio/max": 1.310545516014099, "sampling/importance_sampling_ratio/mean": 1.0005014717578888, "sampling/importance_sampling_ratio/min": 0.7439672410488128, "sampling/sampling_logp_difference/max": 0.33183404207229616, "sampling/sampling_logp_difference/mean": 0.021340764686465265, "step": 9940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.3, "completions/max_terminated_length": 282.3, "completions/mean_length": 203.725, "completions/mean_terminated_length": 203.725, "completions/min_length": 141.7, "completions/min_terminated_length": 141.7, "entropy": 1.0810140691697598, "epoch": 0.9147742943826422, "frac_reward_zero_std": 0.8, "grad_norm": 0.11440661549568176, "kl": 0.013508238637587055, "learning_rate": 4.163277343058402e-06, "loss": 0.0006, "num_tokens": 43382449.0, "reward": 0.125, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2269197702407837, "sampling/importance_sampling_ratio/mean": 0.9999945819377899, "sampling/importance_sampling_ratio/min": 0.793588000535965, "sampling/sampling_logp_difference/max": 0.24035245180130005, "sampling/sampling_logp_difference/mean": 0.02049623690545559, "step": 9950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.9, "completions/max_terminated_length": 327.9, "completions/mean_length": 230.7625, "completions/mean_terminated_length": 230.7625, "completions/min_length": 153.8, "completions/min_terminated_length": 153.8, "entropy": 1.084105795621872, "epoch": 0.9156936655327755, "frac_reward_zero_std": 0.4, "grad_norm": 0.12843193113803864, "kl": 0.012788108299719169, "learning_rate": 4.1613849705860045e-06, "loss": 0.0009, "num_tokens": 43426150.0, "reward": -0.15, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2241904616355896, "sampling/importance_sampling_ratio/mean": 0.9998680174350738, "sampling/importance_sampling_ratio/min": 0.7727454632520676, "sampling/sampling_logp_difference/max": 0.28132748901844024, "sampling/sampling_logp_difference/mean": 0.020420204289257525, "step": 9960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.9, "completions/max_terminated_length": 302.2, "completions/mean_length": 221.7, "completions/mean_terminated_length": 219.09107360839843, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.1231134947389365, "epoch": 0.9166130366829089, "frac_reward_zero_std": 0.3, "grad_norm": 5.437591075897217, "kl": 0.012637374206678941, "learning_rate": 4.159490891738065e-06, "loss": 0.0005, "num_tokens": 43469342.0, "reward": -0.25625, "reward_std": 0.5471285209059715, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.5471285328269004, "sampling/importance_sampling_ratio/max": 1.272013771533966, "sampling/importance_sampling_ratio/mean": 1.0002080202102661, "sampling/importance_sampling_ratio/min": 0.7928065955638885, "sampling/sampling_logp_difference/max": 0.25600200295448305, "sampling/sampling_logp_difference/mean": 0.021474472247064114, "step": 9970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.2, "completions/max_terminated_length": 298.2, "completions/mean_length": 206.125, "completions/mean_terminated_length": 206.125, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.025414904206991, "epoch": 0.9175324078330422, "frac_reward_zero_std": 0.3, "grad_norm": 6.39093017578125, "kl": 0.011710855801356956, "learning_rate": 4.157595108459956e-06, "loss": 0.0006, "num_tokens": 43512608.0, "reward": -0.125, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.3112743854522706, "sampling/importance_sampling_ratio/mean": 0.9998523175716401, "sampling/importance_sampling_ratio/min": 0.7735490381717682, "sampling/sampling_logp_difference/max": 0.3039805948734283, "sampling/sampling_logp_difference/mean": 0.02088075168430805, "step": 9980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 335.7, "completions/max_terminated_length": 318.5, "completions/mean_length": 226.6875, "completions/mean_terminated_length": 223.23214416503907, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.0605177782475947, "epoch": 0.9184517789831755, "frac_reward_zero_std": 0.6, "grad_norm": 3.9470953941345215, "kl": 0.012939522019587457, "learning_rate": 4.155697622698798e-06, "loss": 0.0007, "num_tokens": 43556127.0, "reward": -0.03125, "reward_std": 0.26260885447263715, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.2626088663935661, "sampling/importance_sampling_ratio/max": 1.2675914883613586, "sampling/importance_sampling_ratio/mean": 0.9999189972877502, "sampling/importance_sampling_ratio/min": 0.7676353096961975, "sampling/sampling_logp_difference/max": 0.2747470498085022, "sampling/sampling_logp_difference/mean": 0.020195367746055126, "step": 9990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.4, "completions/max_terminated_length": 291.4, "completions/mean_length": 207.6375, "completions/mean_terminated_length": 207.6375, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0923216745257378, "epoch": 0.9193711501333088, "frac_reward_zero_std": 0.2, "grad_norm": 8.452990531921387, "kl": 0.013404901279136539, "learning_rate": 4.153798436403461e-06, "loss": 0.0008, "num_tokens": 43598322.0, "reward": -0.6, "reward_std": 0.6640828788280487, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.6640829145908356, "sampling/importance_sampling_ratio/max": 1.2647338032722473, "sampling/importance_sampling_ratio/mean": 0.9997815251350403, "sampling/importance_sampling_ratio/min": 0.7440345078706742, "sampling/sampling_logp_difference/max": 0.3365832090377808, "sampling/sampling_logp_difference/mean": 0.021151659823954106, "step": 10000 }, { "epoch": 0.9193711501333088, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 307.6247818499127, "eval_completions/max_terminated_length": 305.72076788830714, "eval_completions/mean_length": 215.83791448516578, "eval_completions/mean_terminated_length": 215.10829399601536, "eval_completions/min_length": 140.79406631762652, "eval_completions/min_terminated_length": 140.79406631762652, "eval_entropy": 1.0878906078363588, "eval_frac_reward_zero_std": 0.45898778359511344, "eval_kl": 0.012694555550105024, "eval_loss": 0.0006528117228299379, "eval_num_tokens": 43598322.0, "eval_reward": -0.11212914485165794, "eval_reward_std": 0.4808378133006121, "eval_rewards/reward_fn/mean": -0.11212914485165794, "eval_rewards/reward_fn/std": 0.4808378264073926, "eval_runtime": 1782.6022, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.270275457367223, "eval_sampling/importance_sampling_ratio/mean": 1.0000046306463644, "eval_sampling/importance_sampling_ratio/min": 0.776869342238179, "eval_sampling/sampling_logp_difference/max": 0.28534317422287625, "eval_sampling/sampling_logp_difference/mean": 0.02064037541769458, "eval_steps_per_second": 0.04, "step": 10000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 305.0, "completions/max_terminated_length": 302.5, "completions/mean_length": 209.6625, "completions/mean_terminated_length": 207.1714294433594, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.0862784489989281, "epoch": 0.9202905212834421, "frac_reward_zero_std": 0.7, "grad_norm": 5.9774489402771, "kl": 0.012121929199201987, "learning_rate": 4.151897551524562e-06, "loss": 0.0007, "num_tokens": 43638399.0, "reward": -0.20625, "reward_std": 0.30608372688293456, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.3060837388038635, "sampling/importance_sampling_ratio/max": 1.3041836261749267, "sampling/importance_sampling_ratio/mean": 1.0002559542655944, "sampling/importance_sampling_ratio/min": 0.7627354383468627, "sampling/sampling_logp_difference/max": 0.312178897857666, "sampling/sampling_logp_difference/mean": 0.02101072557270527, "step": 10010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 221.85, "completions/mean_terminated_length": 221.85, "completions/min_length": 152.9, "completions/min_terminated_length": 152.9, "entropy": 1.0584623493254184, "epoch": 0.9212098924335754, "frac_reward_zero_std": 0.7, "grad_norm": 0.11924711614847183, "kl": 0.0124340983631555, "learning_rate": 4.149994970014461e-06, "loss": 0.0006, "num_tokens": 43682379.0, "reward": -0.25, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.240449559688568, "sampling/importance_sampling_ratio/mean": 0.9999407827854156, "sampling/importance_sampling_ratio/min": 0.7438097625970841, "sampling/sampling_logp_difference/max": 0.31846465468406676, "sampling/sampling_logp_difference/mean": 0.02087667416781187, "step": 10020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.8, "completions/max_terminated_length": 277.8, "completions/mean_length": 201.9625, "completions/mean_terminated_length": 201.9625, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.0715405866503716, "epoch": 0.9221292635837087, "frac_reward_zero_std": 0.8, "grad_norm": 7.42564058303833, "kl": 0.011505114351166412, "learning_rate": 4.1480906938272625e-06, "loss": 0.0006, "num_tokens": 43724256.0, "reward": 0.05, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2294539213180542, "sampling/importance_sampling_ratio/mean": 1.000331747531891, "sampling/importance_sampling_ratio/min": 0.7762697517871857, "sampling/sampling_logp_difference/max": 0.27048930525779724, "sampling/sampling_logp_difference/mean": 0.02077507060021162, "step": 10030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.3, "completions/max_terminated_length": 329.3, "completions/mean_length": 222.675, "completions/mean_terminated_length": 222.675, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0443955652415753, "epoch": 0.923048634733842, "frac_reward_zero_std": 0.5, "grad_norm": 5.73855447769165, "kl": 0.012398993881652132, "learning_rate": 4.14618472491881e-06, "loss": 0.0009, "num_tokens": 43769134.0, "reward": -0.15, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2669079542160033, "sampling/importance_sampling_ratio/mean": 0.9995689272880555, "sampling/importance_sampling_ratio/min": 0.7819952487945556, "sampling/sampling_logp_difference/max": 0.2762083649635315, "sampling/sampling_logp_difference/mean": 0.020977215096354484, "step": 10040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.7, "completions/max_terminated_length": 283.7, "completions/mean_length": 195.0375, "completions/mean_terminated_length": 195.0375, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.0189052432775498, "epoch": 0.9239680058839753, "frac_reward_zero_std": 0.5, "grad_norm": 5.630411148071289, "kl": 0.010687343965400942, "learning_rate": 4.1442770652466894e-06, "loss": 0.0007, "num_tokens": 43810009.0, "reward": -0.525, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3260497808456422, "sampling/importance_sampling_ratio/mean": 0.9999111652374267, "sampling/importance_sampling_ratio/min": 0.7872267067432404, "sampling/sampling_logp_difference/max": 0.29278160333633424, "sampling/sampling_logp_difference/mean": 0.0206361161544919, "step": 10050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 219.4, "completions/mean_terminated_length": 219.4, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1371871024370193, "epoch": 0.9248873770341086, "frac_reward_zero_std": 0.5, "grad_norm": 4.975265026092529, "kl": 0.012838270131032914, "learning_rate": 4.142367716770216e-06, "loss": 0.0007, "num_tokens": 43854185.0, "reward": -0.3, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2900489568710327, "sampling/importance_sampling_ratio/mean": 0.9998832821846009, "sampling/importance_sampling_ratio/min": 0.769539189338684, "sampling/sampling_logp_difference/max": 0.3090594053268433, "sampling/sampling_logp_difference/mean": 0.02179559916257858, "step": 10060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.3, "completions/max_terminated_length": 325.3, "completions/mean_length": 229.25, "completions/mean_terminated_length": 229.25, "completions/min_length": 154.2, "completions/min_terminated_length": 154.2, "entropy": 1.115637905150652, "epoch": 0.925806748184242, "frac_reward_zero_std": 0.4, "grad_norm": 0.11488236486911774, "kl": 0.012637785455444828, "learning_rate": 4.1404566814504465e-06, "loss": 0.0008, "num_tokens": 43899333.0, "reward": 0.125, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2780714511871338, "sampling/importance_sampling_ratio/mean": 1.000387394428253, "sampling/importance_sampling_ratio/min": 0.7233813628554344, "sampling/sampling_logp_difference/max": 0.4683168888092041, "sampling/sampling_logp_difference/mean": 0.021387055702507495, "step": 10070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.3, "completions/max_terminated_length": 292.3, "completions/mean_length": 215.9, "completions/mean_terminated_length": 215.9, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.103851781785488, "epoch": 0.9267261193343753, "frac_reward_zero_std": 0.5, "grad_norm": 5.025798797607422, "kl": 0.01169613670790568, "learning_rate": 4.138543961250167e-06, "loss": 0.0007, "num_tokens": 43944245.0, "reward": -0.28125, "reward_std": 0.40403020977973936, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.40403022766113283, "sampling/importance_sampling_ratio/max": 1.2466405510902405, "sampling/importance_sampling_ratio/mean": 0.9992899179458619, "sampling/importance_sampling_ratio/min": 0.7840254664421081, "sampling/sampling_logp_difference/max": 0.2633986949920654, "sampling/sampling_logp_difference/mean": 0.021729676239192485, "step": 10080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.6, "completions/max_terminated_length": 327.6, "completions/mean_length": 217.6125, "completions/mean_terminated_length": 217.6125, "completions/min_length": 114.4, "completions/min_terminated_length": 114.4, "entropy": 1.102070540189743, "epoch": 0.9276454904845086, "frac_reward_zero_std": 0.6, "grad_norm": 0.11581776291131973, "kl": 0.012422125902958214, "learning_rate": 4.136629558133892e-06, "loss": 0.0007, "num_tokens": 43990846.0, "reward": -0.25, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.285259234905243, "sampling/importance_sampling_ratio/mean": 0.9996606230735778, "sampling/importance_sampling_ratio/min": 0.7792568624019622, "sampling/sampling_logp_difference/max": 0.2775253474712372, "sampling/sampling_logp_difference/mean": 0.021807271987199783, "step": 10090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 309.1, "completions/max_terminated_length": 304.4, "completions/mean_length": 228.4125, "completions/mean_terminated_length": 221.625, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0026326715946197, "epoch": 0.9285648616346419, "frac_reward_zero_std": 0.7, "grad_norm": 5.633270740509033, "kl": 0.011834928521420807, "learning_rate": 4.134713474067871e-06, "loss": 0.001, "num_tokens": 44036671.0, "reward": -0.15, "reward_std": 0.3206089615821838, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3206089735031128, "sampling/importance_sampling_ratio/max": 1.279895794391632, "sampling/importance_sampling_ratio/mean": 0.9999902844429016, "sampling/importance_sampling_ratio/min": 0.7957706928253174, "sampling/sampling_logp_difference/max": 0.2670531511306763, "sampling/sampling_logp_difference/mean": 0.019834168814122678, "step": 10100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 373.4, "completions/max_terminated_length": 328.1, "completions/mean_length": 229.2625, "completions/mean_terminated_length": 221.80535888671875, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.038137100636959, "epoch": 0.9294842327847752, "frac_reward_zero_std": 0.3, "grad_norm": 5.561835765838623, "kl": 0.012804041616618633, "learning_rate": 4.132795711020072e-06, "loss": 0.0009, "num_tokens": 44081940.0, "reward": -0.3125, "reward_std": 0.597175246477127, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.5971752583980561, "sampling/importance_sampling_ratio/max": 1.2729414701461792, "sampling/importance_sampling_ratio/mean": 1.0001795649528504, "sampling/importance_sampling_ratio/min": 0.7103420525789261, "sampling/sampling_logp_difference/max": 0.3813006401062012, "sampling/sampling_logp_difference/mean": 0.020219190791249276, "step": 10110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 200.875, "completions/mean_terminated_length": 200.875, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.0636507153511048, "epoch": 0.9304036039349085, "frac_reward_zero_std": 0.5, "grad_norm": 0.11457975953817368, "kl": 0.011503635783446953, "learning_rate": 4.130876270960194e-06, "loss": 0.0005, "num_tokens": 44124818.0, "reward": 0.2, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.249902105331421, "sampling/importance_sampling_ratio/mean": 1.0006509125232697, "sampling/importance_sampling_ratio/min": 0.7936123430728912, "sampling/sampling_logp_difference/max": 0.2437128484249115, "sampling/sampling_logp_difference/mean": 0.02092997757717967, "step": 10120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.7, "completions/max_terminated_length": 243.7, "completions/mean_length": 179.4875, "completions/mean_terminated_length": 179.4875, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 1.0238811157643795, "epoch": 0.9313229750850418, "frac_reward_zero_std": 0.6, "grad_norm": 6.087275505065918, "kl": 0.011851310968631878, "learning_rate": 4.128955155859655e-06, "loss": 0.0004, "num_tokens": 44166569.0, "reward": -0.1, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2734082818031311, "sampling/importance_sampling_ratio/mean": 0.9998384892940522, "sampling/importance_sampling_ratio/min": 0.7657072305679321, "sampling/sampling_logp_difference/max": 0.27218451499938967, "sampling/sampling_logp_difference/mean": 0.020982794091105463, "step": 10130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.4, "completions/max_terminated_length": 322.4, "completions/mean_length": 228.0875, "completions/mean_terminated_length": 228.0875, "completions/min_length": 155.3, "completions/min_terminated_length": 155.3, "entropy": 1.0872258216142654, "epoch": 0.9322423462351751, "frac_reward_zero_std": 0.7, "grad_norm": 4.898396015167236, "kl": 0.012243143655359745, "learning_rate": 4.127032367691595e-06, "loss": 0.0006, "num_tokens": 44213800.0, "reward": 0.375, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2649528622627257, "sampling/importance_sampling_ratio/mean": 1.0000635743141175, "sampling/importance_sampling_ratio/min": 0.7821682393550873, "sampling/sampling_logp_difference/max": 0.26632280349731446, "sampling/sampling_logp_difference/mean": 0.021630979143083096, "step": 10140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.5, "completions/max_terminated_length": 307.5, "completions/mean_length": 197.35, "completions/mean_terminated_length": 197.35, "completions/min_length": 111.2, "completions/min_terminated_length": 111.2, "entropy": 1.117677503079176, "epoch": 0.9331617173853084, "frac_reward_zero_std": 0.6, "grad_norm": 0.10713942348957062, "kl": 0.012321652245009317, "learning_rate": 4.1251079084308696e-06, "loss": 0.0005, "num_tokens": 44253884.0, "reward": -0.05, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2579238533973693, "sampling/importance_sampling_ratio/mean": 1.0002871453762054, "sampling/importance_sampling_ratio/min": 0.7812039792537689, "sampling/sampling_logp_difference/max": 0.28034592866897584, "sampling/sampling_logp_difference/mean": 0.022083471529185772, "step": 10150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 211.9625, "completions/mean_terminated_length": 211.9625, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.1346209309995174, "epoch": 0.9340810885354418, "frac_reward_zero_std": 0.5, "grad_norm": 4.825003147125244, "kl": 0.013396174972876906, "learning_rate": 4.123181780054054e-06, "loss": 0.0005, "num_tokens": 44297153.0, "reward": -0.625, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.252335262298584, "sampling/importance_sampling_ratio/mean": 0.9992134630680084, "sampling/importance_sampling_ratio/min": 0.7430922031402588, "sampling/sampling_logp_difference/max": 0.3399652123451233, "sampling/sampling_logp_difference/mean": 0.02306628841906786, "step": 10160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 190.9, "completions/mean_terminated_length": 190.9, "completions/min_length": 109.7, "completions/min_terminated_length": 109.7, "entropy": 1.0772713780403138, "epoch": 0.9350004596855751, "frac_reward_zero_std": 0.3, "grad_norm": 0.13464578986167908, "kl": 0.012470044620567933, "learning_rate": 4.121253984539437e-06, "loss": 0.0005, "num_tokens": 44339841.0, "reward": 0.05, "reward_std": 0.5605887413024903, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5605887413024903, "sampling/importance_sampling_ratio/max": 1.2467645764350892, "sampling/importance_sampling_ratio/mean": 1.0002282857894897, "sampling/importance_sampling_ratio/min": 0.7040220260620117, "sampling/sampling_logp_difference/max": 0.39617461562156675, "sampling/sampling_logp_difference/mean": 0.022177689522504807, "step": 10170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.6, "completions/max_terminated_length": 261.6, "completions/mean_length": 185.1125, "completions/mean_terminated_length": 185.1125, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.0606713362038136, "epoch": 0.9359198308357084, "frac_reward_zero_std": 0.7, "grad_norm": 4.9123358726501465, "kl": 0.011477985786041246, "learning_rate": 4.1193245238670185e-06, "loss": 0.0003, "num_tokens": 44379834.0, "reward": -0.075, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2313984155654907, "sampling/importance_sampling_ratio/mean": 1.0001949906349181, "sampling/importance_sampling_ratio/min": 0.7667060196399689, "sampling/sampling_logp_difference/max": 0.2685143709182739, "sampling/sampling_logp_difference/mean": 0.021125704795122147, "step": 10180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 199.6, "completions/mean_terminated_length": 199.6, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 0.9970380067825317, "epoch": 0.9368392019858417, "frac_reward_zero_std": 0.5, "grad_norm": 0.1254442185163498, "kl": 0.011453713034279644, "learning_rate": 4.117393400018508e-06, "loss": 0.0007, "num_tokens": 44422002.0, "reward": 0.025, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2510828733444215, "sampling/importance_sampling_ratio/mean": 0.999369102716446, "sampling/importance_sampling_ratio/min": 0.7481465220451355, "sampling/sampling_logp_difference/max": 0.3017898201942444, "sampling/sampling_logp_difference/mean": 0.02071061860769987, "step": 10190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.2, "completions/max_terminated_length": 272.2, "completions/mean_length": 191.3625, "completions/mean_terminated_length": 191.3625, "completions/min_length": 129.7, "completions/min_terminated_length": 129.7, "entropy": 1.1218883745372294, "epoch": 0.937758573135975, "frac_reward_zero_std": 0.5, "grad_norm": 0.10124137997627258, "kl": 0.012542158045107499, "learning_rate": 4.115460614977327e-06, "loss": 0.0004, "num_tokens": 44465591.0, "reward": -0.025, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2346439480781555, "sampling/importance_sampling_ratio/mean": 0.9997369170188903, "sampling/importance_sampling_ratio/min": 0.8017941594123841, "sampling/sampling_logp_difference/max": 0.23429613709449768, "sampling/sampling_logp_difference/mean": 0.0216204397380352, "step": 10200 }, { "epoch": 0.937758573135975, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 301.97033158813264, "eval_completions/max_terminated_length": 298.1064572425829, "eval_completions/mean_length": 208.8778359511344, "eval_completions/mean_terminated_length": 207.9615123808696, "eval_completions/min_length": 132.81849912739966, "eval_completions/min_terminated_length": 132.81849912739966, "eval_entropy": 1.0803693891819859, "eval_frac_reward_zero_std": 0.4694589877835951, "eval_kl": 0.01212196178038655, "eval_loss": 0.0005831699236296117, "eval_num_tokens": 44465591.0, "eval_reward": -0.11681937172774869, "eval_reward_std": 0.46716400939340574, "eval_rewards/reward_fn/mean": -0.11681937172774869, "eval_rewards/reward_fn/std": 0.4671640228122525, "eval_runtime": 1755.6927, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.2689617035900735, "eval_sampling/importance_sampling_ratio/mean": 1.000034414348802, "eval_sampling/importance_sampling_ratio/min": 0.7785505206563086, "eval_sampling/sampling_logp_difference/max": 0.2833192922682038, "eval_sampling/sampling_logp_difference/mean": 0.020877718936853487, "eval_steps_per_second": 0.041, "step": 10200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 344.1, "completions/max_terminated_length": 333.6, "completions/mean_length": 230.725, "completions/mean_terminated_length": 228.09107360839843, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.0862012781202792, "epoch": 0.9386779442861083, "frac_reward_zero_std": 0.5, "grad_norm": 0.1443544179201126, "kl": 0.011698687722673639, "learning_rate": 4.113526170728599e-06, "loss": 0.0009, "num_tokens": 44507025.0, "reward": 0.00625, "reward_std": 0.4123764753341675, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.4123764932155609, "sampling/importance_sampling_ratio/max": 1.2385761380195617, "sampling/importance_sampling_ratio/mean": 1.0003546714782714, "sampling/importance_sampling_ratio/min": 0.7840991258621216, "sampling/sampling_logp_difference/max": 0.24853100776672363, "sampling/sampling_logp_difference/mean": 0.021160734444856645, "step": 10210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.7, "completions/max_terminated_length": 308.4, "completions/mean_length": 213.7875, "completions/mean_terminated_length": 210.83214416503907, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.1726679146289825, "epoch": 0.9395973154362416, "frac_reward_zero_std": 0.6, "grad_norm": 0.09240900725126266, "kl": 0.011946985794929787, "learning_rate": 4.111590069259154e-06, "loss": 0.0005, "num_tokens": 44554496.0, "reward": -0.65625, "reward_std": 0.34549421072006226, "rewards/reward_fn/mean": -0.65625, "rewards/reward_fn/std": 0.3454942166805267, "sampling/importance_sampling_ratio/max": 1.23777996301651, "sampling/importance_sampling_ratio/mean": 0.9995286524295807, "sampling/importance_sampling_ratio/min": 0.7468135952949524, "sampling/sampling_logp_difference/max": 0.3061178684234619, "sampling/sampling_logp_difference/mean": 0.022370877489447594, "step": 10220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.8, "completions/max_terminated_length": 294.8, "completions/mean_length": 210.9875, "completions/mean_terminated_length": 210.9875, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0804511032998563, "epoch": 0.9405166865863749, "frac_reward_zero_std": 0.4, "grad_norm": 0.08680616319179535, "kl": 0.011094671086175368, "learning_rate": 4.109652312557523e-06, "loss": 0.0009, "num_tokens": 44598247.0, "reward": 0.19375, "reward_std": 0.5569750487804412, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.5569750607013703, "sampling/importance_sampling_ratio/max": 1.2653362154960632, "sampling/importance_sampling_ratio/mean": 0.9989010751247406, "sampling/importance_sampling_ratio/min": 0.7568799138069153, "sampling/sampling_logp_difference/max": 0.31052327156066895, "sampling/sampling_logp_difference/mean": 0.0208892535418272, "step": 10230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 286.9, "completions/max_terminated_length": 269.9, "completions/mean_length": 208.7375, "completions/mean_terminated_length": 205.81964416503905, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.029560673236847, "epoch": 0.9414360577365082, "frac_reward_zero_std": 0.4, "grad_norm": 0.10422623157501221, "kl": 0.010972953969030641, "learning_rate": 4.107712902613936e-06, "loss": 0.0003, "num_tokens": 44640962.0, "reward": 0.34375, "reward_std": 0.518483555316925, "rewards/reward_fn/mean": 0.34375, "rewards/reward_fn/std": 0.5184835731983185, "sampling/importance_sampling_ratio/max": 1.3312625646591187, "sampling/importance_sampling_ratio/mean": 0.9997054874897003, "sampling/importance_sampling_ratio/min": 0.7700465261936188, "sampling/sampling_logp_difference/max": 0.3303660064935684, "sampling/sampling_logp_difference/mean": 0.020313326828181745, "step": 10240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 309.2, "completions/max_terminated_length": 294.6, "completions/mean_length": 210.3875, "completions/mean_terminated_length": 206.96964416503906, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 1.0763397060334683, "epoch": 0.9423554288866416, "frac_reward_zero_std": 0.6, "grad_norm": 0.13309457898139954, "kl": 0.012036188796628267, "learning_rate": 4.105771841420326e-06, "loss": 0.0006, "num_tokens": 44684545.0, "reward": -0.08125, "reward_std": 0.3706337630748749, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.3706337630748749, "sampling/importance_sampling_ratio/max": 1.2699186325073242, "sampling/importance_sampling_ratio/mean": 1.000418734550476, "sampling/importance_sampling_ratio/min": 0.7517093122005463, "sampling/sampling_logp_difference/max": 0.30125845670700074, "sampling/sampling_logp_difference/mean": 0.021397302486002447, "step": 10250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.1, "completions/max_terminated_length": 284.1, "completions/mean_length": 196.4875, "completions/mean_terminated_length": 196.4875, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.0934504769742488, "epoch": 0.9432748000367749, "frac_reward_zero_std": 0.5, "grad_norm": 0.17028969526290894, "kl": 0.012450052046915516, "learning_rate": 4.103829130970316e-06, "loss": 0.0003, "num_tokens": 44726608.0, "reward": -0.175, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.253408670425415, "sampling/importance_sampling_ratio/mean": 1.0008140921592712, "sampling/importance_sampling_ratio/min": 0.8045485317707062, "sampling/sampling_logp_difference/max": 0.24627667665481567, "sampling/sampling_logp_difference/mean": 0.021818987838923932, "step": 10260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.4, "completions/max_terminated_length": 274.4, "completions/mean_length": 198.0125, "completions/mean_terminated_length": 198.0125, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.0959128715097903, "epoch": 0.9441941711869082, "frac_reward_zero_std": 0.7, "grad_norm": 6.276423454284668, "kl": 0.01241445767809637, "learning_rate": 4.101884773259226e-06, "loss": 0.0006, "num_tokens": 44768857.0, "reward": 0.05, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.2999594926834106, "sampling/importance_sampling_ratio/mean": 1.0001509726047515, "sampling/importance_sampling_ratio/min": 0.7914525508880615, "sampling/sampling_logp_difference/max": 0.2714351236820221, "sampling/sampling_logp_difference/mean": 0.021563909389078616, "step": 10270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.1, "completions/max_terminated_length": 327.1, "completions/mean_length": 223.45, "completions/mean_terminated_length": 223.45, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1305181242525577, "epoch": 0.9451135423370415, "frac_reward_zero_std": 0.4, "grad_norm": 0.14003349840641022, "kl": 0.012995348905678838, "learning_rate": 4.099938770284066e-06, "loss": 0.0008, "num_tokens": 44815685.0, "reward": 0.175, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.3027095437049865, "sampling/importance_sampling_ratio/mean": 0.9996786713600159, "sampling/importance_sampling_ratio/min": 0.7672662794589996, "sampling/sampling_logp_difference/max": 0.3015668034553528, "sampling/sampling_logp_difference/mean": 0.021869585476815702, "step": 10280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.5, "completions/max_terminated_length": 321.5, "completions/mean_length": 218.3625, "completions/mean_terminated_length": 218.3625, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0821509510278702, "epoch": 0.9460329134871748, "frac_reward_zero_std": 0.4, "grad_norm": 6.053228378295898, "kl": 0.012100315373390912, "learning_rate": 4.09799112404354e-06, "loss": 0.0007, "num_tokens": 44860218.0, "reward": -0.175, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.2931585550308227, "sampling/importance_sampling_ratio/mean": 0.999995905160904, "sampling/importance_sampling_ratio/min": 0.758847963809967, "sampling/sampling_logp_difference/max": 0.33407409191131593, "sampling/sampling_logp_difference/mean": 0.021278361789882183, "step": 10290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.7, "completions/max_terminated_length": 322.7, "completions/mean_length": 218.0375, "completions/mean_terminated_length": 218.0375, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.1207055203616618, "epoch": 0.9469522846373081, "frac_reward_zero_std": 0.3, "grad_norm": 0.1234496608376503, "kl": 0.012555064749903977, "learning_rate": 4.096041836538034e-06, "loss": 0.0004, "num_tokens": 44909229.0, "reward": 0.125, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.2336604356765748, "sampling/importance_sampling_ratio/mean": 0.9997068583965302, "sampling/importance_sampling_ratio/min": 0.7919573426246643, "sampling/sampling_logp_difference/max": 0.24188706874847413, "sampling/sampling_logp_difference/mean": 0.021434499882161617, "step": 10300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 365.9, "completions/max_terminated_length": 365.9, "completions/mean_length": 241.7625, "completions/mean_terminated_length": 241.7625, "completions/min_length": 153.0, "completions/min_terminated_length": 153.0, "entropy": 1.0469856701791287, "epoch": 0.9478716557874414, "frac_reward_zero_std": 0.5, "grad_norm": 0.1245109811425209, "kl": 0.012146510538877919, "learning_rate": 4.094090909769624e-06, "loss": 0.0008, "num_tokens": 44955458.0, "reward": -0.35, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.3004451751708985, "sampling/importance_sampling_ratio/mean": 1.0003809988498689, "sampling/importance_sampling_ratio/min": 0.7111880511045456, "sampling/sampling_logp_difference/max": 0.3873276710510254, "sampling/sampling_logp_difference/mean": 0.02062074262648821, "step": 10310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.1, "completions/max_terminated_length": 328.1, "completions/mean_length": 219.2125, "completions/mean_terminated_length": 219.2125, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.0849432490766049, "epoch": 0.9487910269375747, "frac_reward_zero_std": 0.2, "grad_norm": 5.17555570602417, "kl": 0.013941249338677153, "learning_rate": 4.092138345742066e-06, "loss": 0.0009, "num_tokens": 44999011.0, "reward": 0.025, "reward_std": 0.722163724899292, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.722163736820221, "sampling/importance_sampling_ratio/max": 1.2737768650054933, "sampling/importance_sampling_ratio/mean": 0.9998674988746643, "sampling/importance_sampling_ratio/min": 0.7963681876659393, "sampling/sampling_logp_difference/max": 0.26053595542907715, "sampling/sampling_logp_difference/mean": 0.02148672826588154, "step": 10320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.0, "completions/max_terminated_length": 275.0, "completions/mean_length": 198.8875, "completions/mean_terminated_length": 198.8875, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 0.9883600525557995, "epoch": 0.949710398087708, "frac_reward_zero_std": 0.6, "grad_norm": 0.12324035167694092, "kl": 0.011732803890481591, "learning_rate": 4.090184146460803e-06, "loss": 0.001, "num_tokens": 45039674.0, "reward": -0.175, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2945401072502136, "sampling/importance_sampling_ratio/mean": 1.0001711010932923, "sampling/importance_sampling_ratio/min": 0.7919064342975617, "sampling/sampling_logp_difference/max": 0.27038298845291137, "sampling/sampling_logp_difference/mean": 0.02044001594185829, "step": 10330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.5, "completions/max_terminated_length": 285.5, "completions/mean_length": 204.925, "completions/mean_terminated_length": 204.925, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.0684061922132968, "epoch": 0.9506297692378414, "frac_reward_zero_std": 0.6, "grad_norm": 0.13442599773406982, "kl": 0.012978469621157274, "learning_rate": 4.088228313932952e-06, "loss": 0.0007, "num_tokens": 45088292.0, "reward": -0.575, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.249456799030304, "sampling/importance_sampling_ratio/mean": 0.9998907029628754, "sampling/importance_sampling_ratio/min": 0.7811747372150422, "sampling/sampling_logp_difference/max": 0.26323936879634857, "sampling/sampling_logp_difference/mean": 0.02084452658891678, "step": 10340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 310.7, "completions/max_terminated_length": 307.8, "completions/mean_length": 223.05, "completions/mean_terminated_length": 217.08333435058594, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 0.9963147506117821, "epoch": 0.9515491403879747, "frac_reward_zero_std": 0.3, "grad_norm": 5.986005783081055, "kl": 0.010773250641068443, "learning_rate": 4.08627085016731e-06, "loss": 0.0007, "num_tokens": 45133456.0, "reward": -0.1875, "reward_std": 0.6417836964130401, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.641783720254898, "sampling/importance_sampling_ratio/max": 1.2499692916870118, "sampling/importance_sampling_ratio/mean": 0.999278974533081, "sampling/importance_sampling_ratio/min": 0.7789934873580933, "sampling/sampling_logp_difference/max": 0.2658161520957947, "sampling/sampling_logp_difference/mean": 0.020526618324220182, "step": 10350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 225.7625, "completions/mean_terminated_length": 225.7625, "completions/min_length": 153.2, "completions/min_terminated_length": 153.2, "entropy": 1.0932547181844712, "epoch": 0.952468511538108, "frac_reward_zero_std": 0.5, "grad_norm": 6.037631988525391, "kl": 0.011984587949700653, "learning_rate": 4.084311757174351e-06, "loss": 0.0006, "num_tokens": 45178645.0, "reward": 0.2, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2891338467597961, "sampling/importance_sampling_ratio/mean": 0.9996941566467286, "sampling/importance_sampling_ratio/min": 0.7790184020996094, "sampling/sampling_logp_difference/max": 0.27410773038864134, "sampling/sampling_logp_difference/mean": 0.021622342802584172, "step": 10360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.1, "completions/max_terminated_length": 304.1, "completions/mean_length": 217.7875, "completions/mean_terminated_length": 217.7875, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.0760998100042343, "epoch": 0.9533878826882413, "frac_reward_zero_std": 0.7, "grad_norm": 5.30362606048584, "kl": 0.0119273939228151, "learning_rate": 4.082351036966219e-06, "loss": 0.0005, "num_tokens": 45227132.0, "reward": -0.3, "reward_std": 0.29206851720809934, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.29206851720809934, "sampling/importance_sampling_ratio/max": 1.2349516987800597, "sampling/importance_sampling_ratio/mean": 0.9994374632835388, "sampling/importance_sampling_ratio/min": 0.7644939184188843, "sampling/sampling_logp_difference/max": 0.27041531205177305, "sampling/sampling_logp_difference/mean": 0.021473823860287667, "step": 10370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.9, "completions/max_terminated_length": 306.9, "completions/mean_length": 202.625, "completions/mean_terminated_length": 202.625, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.0970313034951686, "epoch": 0.9543072538383746, "frac_reward_zero_std": 0.5, "grad_norm": 0.10596173256635666, "kl": 0.011290543002542109, "learning_rate": 4.080388691556731e-06, "loss": 0.0005, "num_tokens": 45270926.0, "reward": -0.3, "reward_std": 0.4772325396537781, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.4772325396537781, "sampling/importance_sampling_ratio/max": 1.2590313315391541, "sampling/importance_sampling_ratio/mean": 0.9998628616333007, "sampling/importance_sampling_ratio/min": 0.7525496363639832, "sampling/sampling_logp_difference/max": 0.29455026388168337, "sampling/sampling_logp_difference/mean": 0.021889030188322067, "step": 10380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 363.4, "completions/max_terminated_length": 363.4, "completions/mean_length": 233.65, "completions/mean_terminated_length": 233.65, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.0481990166008472, "epoch": 0.9552266249885079, "frac_reward_zero_std": 0.3, "grad_norm": 0.14682136476039886, "kl": 0.011960020160768182, "learning_rate": 4.0784247229613725e-06, "loss": 0.0008, "num_tokens": 45316762.0, "reward": -0.05, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.2645080924034118, "sampling/importance_sampling_ratio/mean": 0.9997771143913269, "sampling/importance_sampling_ratio/min": 0.782551783323288, "sampling/sampling_logp_difference/max": 0.2770486533641815, "sampling/sampling_logp_difference/mean": 0.02064321469515562, "step": 10390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.2, "completions/max_terminated_length": 319.2, "completions/mean_length": 234.525, "completions/mean_terminated_length": 234.525, "completions/min_length": 157.5, "completions/min_terminated_length": 157.5, "entropy": 1.1062172412872315, "epoch": 0.9561459961386412, "frac_reward_zero_std": 0.4, "grad_norm": 0.21614931523799896, "kl": 0.013751373841660097, "learning_rate": 4.076459133197297e-06, "loss": 0.001, "num_tokens": 45364028.0, "reward": -0.375, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.2508392333984375, "sampling/importance_sampling_ratio/mean": 1.0001893877983092, "sampling/importance_sampling_ratio/min": 0.764997124671936, "sampling/sampling_logp_difference/max": 0.2755335807800293, "sampling/sampling_logp_difference/mean": 0.021274834498763086, "step": 10400 }, { "epoch": 0.9561459961386412, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 313.1256544502618, "eval_completions/max_terminated_length": 310.56893542757416, "eval_completions/mean_length": 219.33987783595114, "eval_completions/mean_terminated_length": 218.67948583366567, "eval_completions/min_length": 145.28970331588133, "eval_completions/min_terminated_length": 145.28970331588133, "eval_entropy": 1.105688792352693, "eval_frac_reward_zero_std": 0.4607329842931937, "eval_kl": 0.012114258587886994, "eval_loss": 0.0005862778052687645, "eval_num_tokens": 45364028.0, "eval_reward": -0.11594677137870855, "eval_reward_std": 0.47542667297896707, "eval_rewards/reward_fn/mean": -0.11594677137870855, "eval_rewards/reward_fn/std": 0.47542668681390204, "eval_runtime": 1811.2426, "eval_samples_per_second": 0.316, "eval_sampling/importance_sampling_ratio/max": 1.2852353492124335, "eval_sampling/importance_sampling_ratio/mean": 0.999932608471283, "eval_sampling/importance_sampling_ratio/min": 0.7741092507439759, "eval_sampling/sampling_logp_difference/max": 0.2926337091710555, "eval_sampling/sampling_logp_difference/mean": 0.021567777941819365, "eval_steps_per_second": 0.04, "step": 10400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.6, "completions/max_terminated_length": 286.6, "completions/mean_length": 195.2875, "completions/mean_terminated_length": 195.2875, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.0127125672996045, "epoch": 0.9570653672887744, "frac_reward_zero_std": 0.4, "grad_norm": 0.10437576472759247, "kl": 0.01140181570663117, "learning_rate": 4.0744919242833245e-06, "loss": 0.0002, "num_tokens": 45403315.0, "reward": 0.45, "reward_std": 0.45706321597099303, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.457063227891922, "sampling/importance_sampling_ratio/max": 1.273975908756256, "sampling/importance_sampling_ratio/mean": 1.0004236042499541, "sampling/importance_sampling_ratio/min": 0.7795145869255066, "sampling/sampling_logp_difference/max": 0.28554829955101013, "sampling/sampling_logp_difference/mean": 0.02144970279186964, "step": 10410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.8, "completions/max_terminated_length": 317.8, "completions/mean_length": 235.4875, "completions/mean_terminated_length": 235.4875, "completions/min_length": 156.1, "completions/min_terminated_length": 156.1, "entropy": 1.0301893040537835, "epoch": 0.9579847384389077, "frac_reward_zero_std": 0.6, "grad_norm": 4.177628993988037, "kl": 0.01085810798103921, "learning_rate": 4.072523098239934e-06, "loss": 0.0007, "num_tokens": 45452234.0, "reward": -0.375, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.290365219116211, "sampling/importance_sampling_ratio/mean": 0.9998989582061768, "sampling/importance_sampling_ratio/min": 0.791630494594574, "sampling/sampling_logp_difference/max": 0.2677578330039978, "sampling/sampling_logp_difference/mean": 0.02097775824368, "step": 10420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.3, "completions/max_terminated_length": 296.3, "completions/mean_length": 205.375, "completions/mean_terminated_length": 205.375, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.0855352245271206, "epoch": 0.958904109589041, "frac_reward_zero_std": 0.6, "grad_norm": 0.06958826631307602, "kl": 0.012600268621463328, "learning_rate": 4.070552657089268e-06, "loss": 0.0002, "num_tokens": 45496128.0, "reward": 0.075, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2498738646507264, "sampling/importance_sampling_ratio/mean": 1.0001031160354614, "sampling/importance_sampling_ratio/min": 0.718328707665205, "sampling/sampling_logp_difference/max": 0.43838202953338623, "sampling/sampling_logp_difference/mean": 0.021739591285586356, "step": 10430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.0, "completions/max_terminated_length": 309.0, "completions/mean_length": 209.6625, "completions/mean_terminated_length": 209.6625, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.020475872606039, "epoch": 0.9598234807391745, "frac_reward_zero_std": 0.3, "grad_norm": 5.337012767791748, "kl": 0.011584178893826902, "learning_rate": 4.068580602855128e-06, "loss": 0.0003, "num_tokens": 45535341.0, "reward": -0.025, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.248820722103119, "sampling/importance_sampling_ratio/mean": 0.9998334109783172, "sampling/importance_sampling_ratio/min": 0.7920613169670105, "sampling/sampling_logp_difference/max": 0.24264122247695924, "sampling/sampling_logp_difference/mean": 0.020543941669166087, "step": 10440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.3, "completions/max_terminated_length": 281.3, "completions/mean_length": 199.7625, "completions/mean_terminated_length": 199.7625, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.0434882607311011, "epoch": 0.9607428518893077, "frac_reward_zero_std": 0.2, "grad_norm": 6.235615253448486, "kl": 0.011350975203095004, "learning_rate": 4.066606937562972e-06, "loss": 0.0002, "num_tokens": 45575682.0, "reward": -0.175, "reward_std": 0.6784210562705993, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6784210681915284, "sampling/importance_sampling_ratio/max": 1.2825585007667542, "sampling/importance_sampling_ratio/mean": 0.9997688472270966, "sampling/importance_sampling_ratio/min": 0.7676453232765198, "sampling/sampling_logp_difference/max": 0.3005779325962067, "sampling/sampling_logp_difference/mean": 0.021067434549331666, "step": 10450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 196.1125, "completions/mean_terminated_length": 196.1125, "completions/min_length": 123.6, "completions/min_terminated_length": 123.6, "entropy": 1.029785092175007, "epoch": 0.961662223039441, "frac_reward_zero_std": 0.5, "grad_norm": 10.549694061279297, "kl": 0.012248043646104633, "learning_rate": 4.064631663239913e-06, "loss": 0.0001, "num_tokens": 45615747.0, "reward": 0.325, "reward_std": 0.48400617241859434, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.48400617241859434, "sampling/importance_sampling_ratio/max": 1.3396892786026, "sampling/importance_sampling_ratio/mean": 0.9997697710990906, "sampling/importance_sampling_ratio/min": 0.7832044661045074, "sampling/sampling_logp_difference/max": 0.3111557126045227, "sampling/sampling_logp_difference/mean": 0.021834480948746203, "step": 10460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.6, "completions/max_terminated_length": 319.6, "completions/mean_length": 206.425, "completions/mean_terminated_length": 206.425, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0707479596138, "epoch": 0.9625815941895743, "frac_reward_zero_std": 0.7, "grad_norm": 0.11894063651561737, "kl": 0.012403414433356375, "learning_rate": 4.0626547819147145e-06, "loss": 0.0006, "num_tokens": 45662989.0, "reward": -0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2448806166648865, "sampling/importance_sampling_ratio/mean": 1.000095045566559, "sampling/importance_sampling_ratio/min": 0.7950270354747773, "sampling/sampling_logp_difference/max": 0.23437100648880005, "sampling/sampling_logp_difference/mean": 0.0215628981590271, "step": 10470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.3, "completions/max_terminated_length": 302.5, "completions/mean_length": 232.025, "completions/mean_terminated_length": 229.20357360839844, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.1766895592212676, "epoch": 0.9635009653397076, "frac_reward_zero_std": 0.5, "grad_norm": 0.14321599900722504, "kl": 0.01329638710594736, "learning_rate": 4.060676295617793e-06, "loss": 0.0005, "num_tokens": 45710047.0, "reward": -0.33125, "reward_std": 0.43548367023468015, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.43548368811607363, "sampling/importance_sampling_ratio/max": 1.249999499320984, "sampling/importance_sampling_ratio/mean": 1.0001276731491089, "sampling/importance_sampling_ratio/min": 0.7459398329257965, "sampling/sampling_logp_difference/max": 0.32543532848358153, "sampling/sampling_logp_difference/mean": 0.022406399622559546, "step": 10480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.3, "completions/max_terminated_length": 308.3, "completions/mean_length": 201.4625, "completions/mean_terminated_length": 201.4625, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0636261396110058, "epoch": 0.9644203364898409, "frac_reward_zero_std": 0.3, "grad_norm": 4.557554721832275, "kl": 0.011500283403438515, "learning_rate": 4.058696206381212e-06, "loss": 0.001, "num_tokens": 45752836.0, "reward": -0.25, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.256778359413147, "sampling/importance_sampling_ratio/mean": 0.9999978721141816, "sampling/importance_sampling_ratio/min": 0.8044473111629487, "sampling/sampling_logp_difference/max": 0.2542697548866272, "sampling/sampling_logp_difference/mean": 0.021296991780400277, "step": 10490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 213.225, "completions/mean_terminated_length": 213.225, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.1436607293784617, "epoch": 0.9653397076399742, "frac_reward_zero_std": 0.6, "grad_norm": 5.346660137176514, "kl": 0.0111072966537904, "learning_rate": 4.056714516238683e-06, "loss": 0.0007, "num_tokens": 45798414.0, "reward": -0.2, "reward_std": 0.38124017119407655, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3812402069568634, "sampling/importance_sampling_ratio/max": 1.225525188446045, "sampling/importance_sampling_ratio/mean": 1.0003278136253357, "sampling/importance_sampling_ratio/min": 0.7997310638427735, "sampling/sampling_logp_difference/max": 0.23075408935546876, "sampling/sampling_logp_difference/mean": 0.021824534237384795, "step": 10500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.7, "completions/max_terminated_length": 306.7, "completions/mean_length": 215.8, "completions/mean_terminated_length": 215.8, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.0551449432969093, "epoch": 0.9662590787901075, "frac_reward_zero_std": 0.7, "grad_norm": 0.12798041105270386, "kl": 0.011671011138241738, "learning_rate": 4.054731227225559e-06, "loss": 0.0006, "num_tokens": 45840846.0, "reward": -0.05, "reward_std": 0.2777460336685181, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.2777460336685181, "sampling/importance_sampling_ratio/max": 1.3200905203819275, "sampling/importance_sampling_ratio/mean": 0.9999128878116608, "sampling/importance_sampling_ratio/min": 0.7464821040630341, "sampling/sampling_logp_difference/max": 0.3346898078918457, "sampling/sampling_logp_difference/mean": 0.02133995033800602, "step": 10510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.6, "completions/max_terminated_length": 325.6, "completions/mean_length": 231.3625, "completions/mean_terminated_length": 231.3625, "completions/min_length": 147.9, "completions/min_terminated_length": 147.9, "entropy": 1.0473475318402052, "epoch": 0.9671784499402408, "frac_reward_zero_std": 0.4, "grad_norm": 0.1262798309326172, "kl": 0.011416665063006804, "learning_rate": 4.0527463413788385e-06, "loss": 0.0006, "num_tokens": 45886123.0, "reward": -0.15625, "reward_std": 0.5709905445575714, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.5709905862808228, "sampling/importance_sampling_ratio/max": 1.278911828994751, "sampling/importance_sampling_ratio/mean": 1.0001543283462524, "sampling/importance_sampling_ratio/min": 0.7883116424083709, "sampling/sampling_logp_difference/max": 0.26336551904678346, "sampling/sampling_logp_difference/mean": 0.020774608198553323, "step": 10520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.9, "completions/max_terminated_length": 318.9, "completions/mean_length": 221.325, "completions/mean_terminated_length": 221.325, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.1253571450710296, "epoch": 0.9680978210903742, "frac_reward_zero_std": 0.4, "grad_norm": 5.822620868682861, "kl": 0.012644177151378245, "learning_rate": 4.050759860737158e-06, "loss": 0.0008, "num_tokens": 45930413.0, "reward": -0.15, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.2382335662841797, "sampling/importance_sampling_ratio/mean": 1.000080019235611, "sampling/importance_sampling_ratio/min": 0.7598256230354309, "sampling/sampling_logp_difference/max": 0.28314597606658937, "sampling/sampling_logp_difference/mean": 0.021708792820572854, "step": 10530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.5, "completions/max_terminated_length": 287.5, "completions/mean_length": 211.4625, "completions/mean_terminated_length": 211.4625, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.0859676823019981, "epoch": 0.9690171922405075, "frac_reward_zero_std": 0.4, "grad_norm": 0.11180011928081512, "kl": 0.011860002327011898, "learning_rate": 4.0487717873407905e-06, "loss": 0.0005, "num_tokens": 45975402.0, "reward": 0.2, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.2991538405418397, "sampling/importance_sampling_ratio/mean": 1.000290310382843, "sampling/importance_sampling_ratio/min": 0.7626415014266967, "sampling/sampling_logp_difference/max": 0.3174164414405823, "sampling/sampling_logp_difference/mean": 0.021566261537373066, "step": 10540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 218.0375, "completions/mean_terminated_length": 218.0375, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.0779301442205906, "epoch": 0.9699365633906408, "frac_reward_zero_std": 0.5, "grad_norm": 5.548909664154053, "kl": 0.011621744866715744, "learning_rate": 4.046782123231649e-06, "loss": 0.0009, "num_tokens": 46017981.0, "reward": -0.275, "reward_std": 0.4587401747703552, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4587401866912842, "sampling/importance_sampling_ratio/max": 1.2580926299095154, "sampling/importance_sampling_ratio/mean": 1.0000939846038819, "sampling/importance_sampling_ratio/min": 0.7904458999633789, "sampling/sampling_logp_difference/max": 0.25487940311431884, "sampling/sampling_logp_difference/mean": 0.021544848382472993, "step": 10550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 202.5, "completions/mean_terminated_length": 202.5, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.1069591827690601, "epoch": 0.9708559345407741, "frac_reward_zero_std": 0.4, "grad_norm": 6.161771774291992, "kl": 0.011589636065764353, "learning_rate": 4.044790870453277e-06, "loss": 0.0008, "num_tokens": 46061629.0, "reward": 0.125, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2610793113708496, "sampling/importance_sampling_ratio/mean": 1.0001426935195923, "sampling/importance_sampling_ratio/min": 0.7209874749183655, "sampling/sampling_logp_difference/max": 0.3384708285331726, "sampling/sampling_logp_difference/mean": 0.022074504196643828, "step": 10560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.0, "completions/max_terminated_length": 286.0, "completions/mean_length": 193.025, "completions/mean_terminated_length": 193.025, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.073929849267006, "epoch": 0.9717753056909074, "frac_reward_zero_std": 0.5, "grad_norm": 7.265277862548828, "kl": 0.01212570206844248, "learning_rate": 4.042798031050851e-06, "loss": 0.0006, "num_tokens": 46102887.0, "reward": 0.21875, "reward_std": 0.4149737238883972, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.4149737298488617, "sampling/importance_sampling_ratio/max": 1.2928330898284912, "sampling/importance_sampling_ratio/mean": 0.9998846113681793, "sampling/importance_sampling_ratio/min": 0.7823190093040466, "sampling/sampling_logp_difference/max": 0.28492101430892947, "sampling/sampling_logp_difference/mean": 0.021818788908421993, "step": 10570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.2, "completions/max_terminated_length": 293.2, "completions/mean_length": 207.5875, "completions/mean_terminated_length": 207.5875, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.118571413308382, "epoch": 0.9726946768410407, "frac_reward_zero_std": 0.3, "grad_norm": 6.936199188232422, "kl": 0.012011450593126938, "learning_rate": 4.040803607071177e-06, "loss": 0.0003, "num_tokens": 46147894.0, "reward": -0.15, "reward_std": 0.5967668652534485, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5967668890953064, "sampling/importance_sampling_ratio/max": 1.2423072814941407, "sampling/importance_sampling_ratio/mean": 1.000246423482895, "sampling/importance_sampling_ratio/min": 0.7878649413585663, "sampling/sampling_logp_difference/max": 0.24613490104675292, "sampling/sampling_logp_difference/mean": 0.021873552352190018, "step": 10580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 194.925, "completions/mean_terminated_length": 194.925, "completions/min_length": 115.7, "completions/min_terminated_length": 115.7, "entropy": 1.0310880564153195, "epoch": 0.973614047991174, "frac_reward_zero_std": 0.6, "grad_norm": 0.11639176309108734, "kl": 0.011106506478972733, "learning_rate": 4.038807600562686e-06, "loss": 0.0007, "num_tokens": 46193952.0, "reward": 0.075, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.3118448495864867, "sampling/importance_sampling_ratio/mean": 0.9998110771179199, "sampling/importance_sampling_ratio/min": 0.7712965965270996, "sampling/sampling_logp_difference/max": 0.3088846802711487, "sampling/sampling_logp_difference/mean": 0.020934041403234006, "step": 10590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.1, "completions/max_terminated_length": 304.1, "completions/mean_length": 202.2, "completions/mean_terminated_length": 202.2, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.106232649832964, "epoch": 0.9745334191413073, "frac_reward_zero_std": 0.6, "grad_norm": 0.0831189751625061, "kl": 0.01306624935241416, "learning_rate": 4.036810013575441e-06, "loss": 0.0008, "num_tokens": 46235520.0, "reward": 0.075, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2465574502944947, "sampling/importance_sampling_ratio/mean": 0.999852865934372, "sampling/importance_sampling_ratio/min": 0.7872740864753723, "sampling/sampling_logp_difference/max": 0.24698171615600586, "sampling/sampling_logp_difference/mean": 0.02304402571171522, "step": 10600 }, { "epoch": 0.9745334191413073, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 287.16230366492147, "eval_completions/max_terminated_length": 285.7137870855148, "eval_completions/mean_length": 202.353184991274, "eval_completions/mean_terminated_length": 201.9850516593893, "eval_completions/min_length": 132.63001745200697, "eval_completions/min_terminated_length": 132.63001745200697, "eval_entropy": 1.0724842459951605, "eval_frac_reward_zero_std": 0.4694589877835951, "eval_kl": 0.012340867355543905, "eval_loss": 0.000629495014436543, "eval_num_tokens": 46235520.0, "eval_reward": -0.10416666666666667, "eval_reward_std": 0.45963017665902983, "eval_rewards/reward_fn/mean": -0.10416666666666667, "eval_rewards/reward_fn/std": 0.4596301897658103, "eval_runtime": 1680.1868, "eval_samples_per_second": 0.341, "eval_sampling/importance_sampling_ratio/max": 1.2706357956348706, "eval_sampling/importance_sampling_ratio/mean": 1.0000003525307872, "eval_sampling/importance_sampling_ratio/min": 0.7745894178306038, "eval_sampling/sampling_logp_difference/max": 0.2884454738615279, "eval_sampling/sampling_logp_difference/mean": 0.020839204023515783, "eval_steps_per_second": 0.043, "step": 10600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 184.6375, "completions/mean_terminated_length": 184.6375, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.053043469786644, "epoch": 0.9754527902914406, "frac_reward_zero_std": 0.2, "grad_norm": 6.17620325088501, "kl": 0.012032003083731979, "learning_rate": 4.034810848161121e-06, "loss": 0.0008, "num_tokens": 46277027.0, "reward": -0.125, "reward_std": 0.664098572731018, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.664098584651947, "sampling/importance_sampling_ratio/max": 1.3186286211013794, "sampling/importance_sampling_ratio/mean": 1.0001787960529327, "sampling/importance_sampling_ratio/min": 0.7960096895694733, "sampling/sampling_logp_difference/max": 0.28953124284744264, "sampling/sampling_logp_difference/mean": 0.021297085098922253, "step": 10610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.8, "completions/max_terminated_length": 275.8, "completions/mean_length": 196.3875, "completions/mean_terminated_length": 196.3875, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.0626906849443913, "epoch": 0.976372161441574, "frac_reward_zero_std": 0.6, "grad_norm": 0.16859760880470276, "kl": 0.012061810144223273, "learning_rate": 4.03281010637303e-06, "loss": 0.0006, "num_tokens": 46318978.0, "reward": -0.775, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.775, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.3289263248443604, "sampling/importance_sampling_ratio/mean": 0.9999575555324555, "sampling/importance_sampling_ratio/min": 0.7912368595600128, "sampling/sampling_logp_difference/max": 0.31029605865478516, "sampling/sampling_logp_difference/mean": 0.020812244154512882, "step": 10620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.3, "completions/max_terminated_length": 310.3, "completions/mean_length": 210.875, "completions/mean_terminated_length": 210.875, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0917795330286026, "epoch": 0.9772915325917073, "frac_reward_zero_std": 0.5, "grad_norm": 5.864053249359131, "kl": 0.012830328493146225, "learning_rate": 4.030807790266089e-06, "loss": 0.0003, "num_tokens": 46363736.0, "reward": -0.05, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2548107981681824, "sampling/importance_sampling_ratio/mean": 0.9997477114200592, "sampling/importance_sampling_ratio/min": 0.7736791849136353, "sampling/sampling_logp_difference/max": 0.27289886474609376, "sampling/sampling_logp_difference/mean": 0.02122162710875273, "step": 10630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.3, "completions/max_terminated_length": 327.3, "completions/mean_length": 221.75, "completions/mean_terminated_length": 221.75, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.068437810242176, "epoch": 0.9782109037418406, "frac_reward_zero_std": 0.4, "grad_norm": 0.12272761762142181, "kl": 0.012631441734265536, "learning_rate": 4.028803901896839e-06, "loss": 0.0005, "num_tokens": 46408036.0, "reward": -0.05, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.3968976020812989, "sampling/importance_sampling_ratio/mean": 1.000179409980774, "sampling/importance_sampling_ratio/min": 0.7675132155418396, "sampling/sampling_logp_difference/max": 0.35868425369262696, "sampling/sampling_logp_difference/mean": 0.02128507699817419, "step": 10640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.5, "completions/max_terminated_length": 243.5, "completions/mean_length": 184.35, "completions/mean_terminated_length": 184.35, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 0.9428809782490134, "epoch": 0.9791302748919739, "frac_reward_zero_std": 0.7, "grad_norm": 4.360230922698975, "kl": 0.010870372073259204, "learning_rate": 4.026798443323433e-06, "loss": 0.0005, "num_tokens": 46445960.0, "reward": 0.175, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2528712391853332, "sampling/importance_sampling_ratio/mean": 0.9997745811939239, "sampling/importance_sampling_ratio/min": 0.7859901905059814, "sampling/sampling_logp_difference/max": 0.26341627836227416, "sampling/sampling_logp_difference/mean": 0.019972100481390954, "step": 10650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.5, "completions/max_terminated_length": 303.5, "completions/mean_length": 227.2125, "completions/mean_terminated_length": 227.2125, "completions/min_length": 154.3, "completions/min_terminated_length": 154.3, "entropy": 1.1281873263418674, "epoch": 0.9800496460421072, "frac_reward_zero_std": 0.3, "grad_norm": 5.716594696044922, "kl": 0.012848854687763377, "learning_rate": 4.024791416605638e-06, "loss": 0.0002, "num_tokens": 46490449.0, "reward": -0.225, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.3063491463661194, "sampling/importance_sampling_ratio/mean": 1.0001194834709168, "sampling/importance_sampling_ratio/min": 0.761669272184372, "sampling/sampling_logp_difference/max": 0.3015973210334778, "sampling/sampling_logp_difference/mean": 0.022025337629020213, "step": 10660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.5, "completions/max_terminated_length": 302.5, "completions/mean_length": 219.4125, "completions/mean_terminated_length": 219.4125, "completions/min_length": 150.1, "completions/min_terminated_length": 150.1, "entropy": 1.0468624982982875, "epoch": 0.9809690171922405, "frac_reward_zero_std": 0.6, "grad_norm": 5.165857315063477, "kl": 0.011563016258878633, "learning_rate": 4.022782823804832e-06, "loss": 0.0007, "num_tokens": 46533058.0, "reward": -0.475, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.3615684151649474, "sampling/importance_sampling_ratio/mean": 1.0003543496131897, "sampling/importance_sampling_ratio/min": 0.7702685356140136, "sampling/sampling_logp_difference/max": 0.3616400122642517, "sampling/sampling_logp_difference/mean": 0.02093289392068982, "step": 10670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.1, "completions/max_terminated_length": 299.8, "completions/mean_length": 213.1625, "completions/mean_terminated_length": 210.22678833007814, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "entropy": 1.0434798903763294, "epoch": 0.9818883883423738, "frac_reward_zero_std": 0.6, "grad_norm": 0.08004210144281387, "kl": 0.010540537041379139, "learning_rate": 4.0207726669839984e-06, "loss": 0.0006, "num_tokens": 46582095.0, "reward": 0.01875, "reward_std": 0.3223917126655579, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.32239171862602234, "sampling/importance_sampling_ratio/max": 1.2532089710235597, "sampling/importance_sampling_ratio/mean": 1.0001936256885529, "sampling/importance_sampling_ratio/min": 0.8021430969238281, "sampling/sampling_logp_difference/max": 0.24016132354736328, "sampling/sampling_logp_difference/mean": 0.020376601070165635, "step": 10680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 214.9, "completions/mean_terminated_length": 214.9, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.0605176731944084, "epoch": 0.9828077594925071, "frac_reward_zero_std": 0.1, "grad_norm": 0.08953667432069778, "kl": 0.013633295457111672, "learning_rate": 4.018760948207731e-06, "loss": 0.001, "num_tokens": 46625295.0, "reward": -0.15, "reward_std": 0.7819308876991272, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.7819309115409852, "sampling/importance_sampling_ratio/max": 1.3243802666664124, "sampling/importance_sampling_ratio/mean": 0.9995549380779266, "sampling/importance_sampling_ratio/min": 0.7370250523090363, "sampling/sampling_logp_difference/max": 0.33394137024879456, "sampling/sampling_logp_difference/mean": 0.021831047534942628, "step": 10690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.6, "completions/max_terminated_length": 322.6, "completions/mean_length": 221.6375, "completions/mean_terminated_length": 221.6375, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.0358936220407486, "epoch": 0.9837271306426404, "frac_reward_zero_std": 0.4, "grad_norm": 5.600763320922852, "kl": 0.011799824988702311, "learning_rate": 4.016747669542227e-06, "loss": 0.0007, "num_tokens": 46670330.0, "reward": 0.4, "reward_std": 0.5664041936397552, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.5664042294025421, "sampling/importance_sampling_ratio/max": 1.4153640866279602, "sampling/importance_sampling_ratio/mean": 1.0002652049064635, "sampling/importance_sampling_ratio/min": 0.7077944129705429, "sampling/sampling_logp_difference/max": 0.42104795575141907, "sampling/sampling_logp_difference/mean": 0.02147216200828552, "step": 10700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 225.0375, "completions/mean_terminated_length": 225.0375, "completions/min_length": 159.7, "completions/min_terminated_length": 159.7, "entropy": 1.0227445937693118, "epoch": 0.9846465017927737, "frac_reward_zero_std": 0.5, "grad_norm": 5.311712265014648, "kl": 0.011620811466127635, "learning_rate": 4.014732833055284e-06, "loss": 0.0008, "num_tokens": 46714237.0, "reward": -0.275, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.2826321840286254, "sampling/importance_sampling_ratio/mean": 0.9998381018638611, "sampling/importance_sampling_ratio/min": 0.7839523613452911, "sampling/sampling_logp_difference/max": 0.2635746955871582, "sampling/sampling_logp_difference/mean": 0.020489681512117386, "step": 10710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 194.1125, "completions/mean_terminated_length": 194.1125, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 0.994840344786644, "epoch": 0.9855658729429071, "frac_reward_zero_std": 0.6, "grad_norm": 0.1205238327383995, "kl": 0.011544624727685004, "learning_rate": 4.0127164408163e-06, "loss": 0.0005, "num_tokens": 46753758.0, "reward": 0.075, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2942651748657226, "sampling/importance_sampling_ratio/mean": 0.9996994853019714, "sampling/importance_sampling_ratio/min": 0.7938290536403656, "sampling/sampling_logp_difference/max": 0.27872936725616454, "sampling/sampling_logp_difference/mean": 0.020908510871231557, "step": 10720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.7, "completions/max_terminated_length": 323.7, "completions/mean_length": 207.0, "completions/mean_terminated_length": 207.0, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 0.9935784243047238, "epoch": 0.9864852440930404, "frac_reward_zero_std": 0.4, "grad_norm": 0.10101485997438431, "kl": 0.013497065799310804, "learning_rate": 4.010698494896274e-06, "loss": 0.0003, "num_tokens": 46794278.0, "reward": -0.35, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.2817034244537353, "sampling/importance_sampling_ratio/mean": 0.9997171878814697, "sampling/importance_sampling_ratio/min": 0.736820787191391, "sampling/sampling_logp_difference/max": 0.3516658663749695, "sampling/sampling_logp_difference/mean": 0.020309263095259666, "step": 10730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.9, "completions/max_terminated_length": 288.9, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 114.2, "completions/min_terminated_length": 114.2, "entropy": 1.0953383773565293, "epoch": 0.9874046152431737, "frac_reward_zero_std": 0.5, "grad_norm": 0.1209261566400528, "kl": 0.011374162056017666, "learning_rate": 4.008678997367795e-06, "loss": 0.0009, "num_tokens": 46839220.0, "reward": -0.075, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2814908742904663, "sampling/importance_sampling_ratio/mean": 1.0003227829933166, "sampling/importance_sampling_ratio/min": 0.7846705675125122, "sampling/sampling_logp_difference/max": 0.26508822441101076, "sampling/sampling_logp_difference/mean": 0.022103776596486567, "step": 10740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.1, "completions/max_terminated_length": 300.1, "completions/mean_length": 206.3, "completions/mean_terminated_length": 206.3, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.014088123291731, "epoch": 0.988323986393307, "frac_reward_zero_std": 0.6, "grad_norm": 0.15840712189674377, "kl": 0.012429623247589916, "learning_rate": 4.006657950305051e-06, "loss": 0.0005, "num_tokens": 46882604.0, "reward": -0.1, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.28409184217453, "sampling/importance_sampling_ratio/mean": 1.0002512812614441, "sampling/importance_sampling_ratio/min": 0.7490102469921112, "sampling/sampling_logp_difference/max": 0.3203671336174011, "sampling/sampling_logp_difference/mean": 0.02111120764166117, "step": 10750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.1, "completions/max_terminated_length": 302.1, "completions/mean_length": 213.9375, "completions/mean_terminated_length": 213.9375, "completions/min_length": 157.4, "completions/min_terminated_length": 157.4, "entropy": 1.0738131895661354, "epoch": 0.9892433575434403, "frac_reward_zero_std": 0.6, "grad_norm": 5.321116924285889, "kl": 0.012251192107214592, "learning_rate": 4.0046353557838165e-06, "loss": 0.0005, "num_tokens": 46929079.0, "reward": 0.175, "reward_std": 0.40311150550842284, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.40311154127120974, "sampling/importance_sampling_ratio/max": 1.264352059364319, "sampling/importance_sampling_ratio/mean": 1.0004097998142243, "sampling/importance_sampling_ratio/min": 0.7858464777469635, "sampling/sampling_logp_difference/max": 0.2778226375579834, "sampling/sampling_logp_difference/mean": 0.02139456644654274, "step": 10760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 202.675, "completions/mean_terminated_length": 202.675, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 0.992853494733572, "epoch": 0.9901627286935736, "frac_reward_zero_std": 0.4, "grad_norm": 5.612648963928223, "kl": 0.011176531616365537, "learning_rate": 4.0026112158814624e-06, "loss": 0.0003, "num_tokens": 46971101.0, "reward": 0.075, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2673466801643372, "sampling/importance_sampling_ratio/mean": 1.000549328327179, "sampling/importance_sampling_ratio/min": 0.7741914570331574, "sampling/sampling_logp_difference/max": 0.27910593450069426, "sampling/sampling_logp_difference/mean": 0.020497377961874008, "step": 10770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.1, "completions/max_terminated_length": 311.1, "completions/mean_length": 210.4, "completions/mean_terminated_length": 210.4, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.1294088236987592, "epoch": 0.9910820998437069, "frac_reward_zero_std": 0.5, "grad_norm": 6.1998677253723145, "kl": 0.013240093138301745, "learning_rate": 4.00058553267694e-06, "loss": 0.0007, "num_tokens": 47018045.0, "reward": 0.34375, "reward_std": 0.4573550045490265, "rewards/reward_fn/mean": 0.34375, "rewards/reward_fn/std": 0.4573550224304199, "sampling/importance_sampling_ratio/max": 1.2807923197746276, "sampling/importance_sampling_ratio/mean": 0.9998382031917572, "sampling/importance_sampling_ratio/min": 0.7900917112827301, "sampling/sampling_logp_difference/max": 0.2704246461391449, "sampling/sampling_logp_difference/mean": 0.022445071674883365, "step": 10780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.4, "completions/max_terminated_length": 276.4, "completions/mean_length": 211.95, "completions/mean_terminated_length": 211.95, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0664664126932621, "epoch": 0.9920014709938402, "frac_reward_zero_std": 0.7, "grad_norm": 0.08921969681978226, "kl": 0.011625161347910761, "learning_rate": 3.99855830825079e-06, "loss": 0.0006, "num_tokens": 47062329.0, "reward": -0.225, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2825191140174865, "sampling/importance_sampling_ratio/mean": 1.0002468526363373, "sampling/importance_sampling_ratio/min": 0.768056184053421, "sampling/sampling_logp_difference/max": 0.2815258800983429, "sampling/sampling_logp_difference/mean": 0.022006993182003498, "step": 10790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.7, "completions/max_terminated_length": 295.7, "completions/mean_length": 211.825, "completions/mean_terminated_length": 211.825, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 0.9881664909422397, "epoch": 0.9929208421439735, "frac_reward_zero_std": 0.5, "grad_norm": 7.234914779663086, "kl": 0.010937193233985454, "learning_rate": 3.996529544685132e-06, "loss": 0.0003, "num_tokens": 47102659.0, "reward": -0.10625, "reward_std": 0.44591636657714845, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.44591636657714845, "sampling/importance_sampling_ratio/max": 1.3186107397079467, "sampling/importance_sampling_ratio/mean": 0.9997430622577668, "sampling/importance_sampling_ratio/min": 0.7571651995182037, "sampling/sampling_logp_difference/max": 0.32798842191696165, "sampling/sampling_logp_difference/mean": 0.020231287740170956, "step": 10800 }, { "epoch": 0.9929208421439735, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 309.630017452007, "eval_completions/max_terminated_length": 307.44677137870855, "eval_completions/mean_length": 215.84162303664922, "eval_completions/mean_terminated_length": 215.33050800742905, "eval_completions/min_length": 141.7539267015707, "eval_completions/min_terminated_length": 141.7539267015707, "eval_entropy": 1.0980020672551, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.01227544814943097, "eval_loss": 0.0005808474961668253, "eval_num_tokens": 47102659.0, "eval_reward": -0.10613001745200698, "eval_reward_std": 0.4419573767022937, "eval_rewards/reward_fn/mean": -0.10613001745200698, "eval_rewards/reward_fn/std": 0.44195738835276527, "eval_runtime": 1790.9661, "eval_samples_per_second": 0.32, "eval_sampling/importance_sampling_ratio/max": 1.2770196642551122, "eval_sampling/importance_sampling_ratio/mean": 0.9999685656753926, "eval_sampling/importance_sampling_ratio/min": 0.7653508507248411, "eval_sampling/sampling_logp_difference/max": 0.29898290333531496, "eval_sampling/sampling_logp_difference/mean": 0.021588580523589935, "eval_steps_per_second": 0.04, "step": 10800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 207.15, "completions/mean_terminated_length": 207.15, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 0.9771002300083638, "epoch": 0.9938402132941069, "frac_reward_zero_std": 0.5, "grad_norm": 5.445786952972412, "kl": 0.011812932181055658, "learning_rate": 3.9944992440636705e-06, "loss": 0.0006, "num_tokens": 47143959.0, "reward": 0.55, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.55, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.28260977268219, "sampling/importance_sampling_ratio/mean": 0.9999678313732148, "sampling/importance_sampling_ratio/min": 0.7845486521720886, "sampling/sampling_logp_difference/max": 0.2755212545394897, "sampling/sampling_logp_difference/mean": 0.02078477796167135, "step": 10810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.4, "completions/max_terminated_length": 292.4, "completions/mean_length": 207.8, "completions/mean_terminated_length": 207.8, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.0761551901698112, "epoch": 0.9947595844442402, "frac_reward_zero_std": 0.4, "grad_norm": 0.10872579365968704, "kl": 0.012170712515944615, "learning_rate": 3.992467408471686e-06, "loss": 0.0009, "num_tokens": 47183791.0, "reward": -0.175, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2365238666534424, "sampling/importance_sampling_ratio/mean": 0.9995595216751099, "sampling/importance_sampling_ratio/min": 0.7766785204410553, "sampling/sampling_logp_difference/max": 0.2566903352737427, "sampling/sampling_logp_difference/mean": 0.022490829415619373, "step": 10820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.7, "completions/max_terminated_length": 298.7, "completions/mean_length": 219.2125, "completions/mean_terminated_length": 219.2125, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.1114941738545894, "epoch": 0.9956789555943735, "frac_reward_zero_std": 0.7, "grad_norm": 4.6691083908081055, "kl": 0.01139875758672133, "learning_rate": 3.990434039996038e-06, "loss": 0.0007, "num_tokens": 47227400.0, "reward": -0.225, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2393807649612427, "sampling/importance_sampling_ratio/mean": 1.0003805816173554, "sampling/importance_sampling_ratio/min": 0.8003561198711395, "sampling/sampling_logp_difference/max": 0.23186404705047609, "sampling/sampling_logp_difference/mean": 0.021559333987534048, "step": 10830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.6, "completions/max_terminated_length": 292.6, "completions/mean_length": 197.375, "completions/mean_terminated_length": 197.375, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 1.0714954614639283, "epoch": 0.9965983267445068, "frac_reward_zero_std": 0.4, "grad_norm": 0.17516125738620758, "kl": 0.015245552733540535, "learning_rate": 3.988399140725157e-06, "loss": 0.0005, "num_tokens": 47267862.0, "reward": -0.38125, "reward_std": 0.5291592717170716, "rewards/reward_fn/mean": -0.38125, "rewards/reward_fn/std": 0.5291592717170716, "sampling/importance_sampling_ratio/max": 1.2487855792045592, "sampling/importance_sampling_ratio/mean": 1.0004419922828673, "sampling/importance_sampling_ratio/min": 0.7686104059219361, "sampling/sampling_logp_difference/max": 0.2690562307834625, "sampling/sampling_logp_difference/mean": 0.022495317831635477, "step": 10840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.3, "completions/max_terminated_length": 285.3, "completions/mean_length": 207.8, "completions/mean_terminated_length": 207.8, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0692164301872253, "epoch": 0.9975176978946401, "frac_reward_zero_std": 0.4, "grad_norm": 0.11505123227834702, "kl": 0.010665275389328599, "learning_rate": 3.98636271274905e-06, "loss": 0.0004, "num_tokens": 47313782.0, "reward": 0.25, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2651538610458375, "sampling/importance_sampling_ratio/mean": 1.0001372992992401, "sampling/importance_sampling_ratio/min": 0.762544858455658, "sampling/sampling_logp_difference/max": 0.29176448881626127, "sampling/sampling_logp_difference/mean": 0.02196824289858341, "step": 10850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.1, "completions/max_terminated_length": 327.1, "completions/mean_length": 207.575, "completions/mean_terminated_length": 207.575, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.034953060746193, "epoch": 0.9984370690447734, "frac_reward_zero_std": 0.4, "grad_norm": 0.15880456566810608, "kl": 0.012392496946267784, "learning_rate": 3.984324758159289e-06, "loss": 0.0008, "num_tokens": 47356956.0, "reward": -0.25, "reward_std": 0.5698145508766175, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5698145508766175, "sampling/importance_sampling_ratio/max": 1.275181269645691, "sampling/importance_sampling_ratio/mean": 1.000321054458618, "sampling/importance_sampling_ratio/min": 0.728776079416275, "sampling/sampling_logp_difference/max": 0.3459524750709534, "sampling/sampling_logp_difference/mean": 0.02107987031340599, "step": 10860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.2, "completions/max_terminated_length": 339.2, "completions/mean_length": 238.125, "completions/mean_terminated_length": 238.125, "completions/min_length": 157.4, "completions/min_terminated_length": 157.4, "entropy": 1.1065384201705455, "epoch": 0.9993564401949067, "frac_reward_zero_std": 0.5, "grad_norm": 0.9892842769622803, "kl": 0.012559228239115327, "learning_rate": 3.98228527904902e-06, "loss": 0.0005, "num_tokens": 47404998.0, "reward": -0.15, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.3022117853164672, "sampling/importance_sampling_ratio/mean": 0.9999133348464966, "sampling/importance_sampling_ratio/min": 0.7744441866874695, "sampling/sampling_logp_difference/max": 0.28889501094818115, "sampling/sampling_logp_difference/mean": 0.021766994334757327, "step": 10870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 205.05, "completions/mean_terminated_length": 205.05, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 1.1159393712878227, "epoch": 1.00027581134504, "frac_reward_zero_std": 0.6, "grad_norm": 6.0393877029418945, "kl": 0.014147518604295329, "learning_rate": 3.980244277512948e-06, "loss": 0.0005, "num_tokens": 47448442.0, "reward": 0.1, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2651680827140808, "sampling/importance_sampling_ratio/mean": 0.9995355725288391, "sampling/importance_sampling_ratio/min": 0.7660524308681488, "sampling/sampling_logp_difference/max": 0.2927773714065552, "sampling/sampling_logp_difference/mean": 0.02232601810246706, "step": 10880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 206.7125, "completions/mean_terminated_length": 206.7125, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0685516715049743, "epoch": 1.0011951824951733, "frac_reward_zero_std": 0.7, "grad_norm": 0.12168069183826447, "kl": 0.011449149955296889, "learning_rate": 3.978201755647348e-06, "loss": 0.0005, "num_tokens": 47494971.0, "reward": 0.15, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2533765196800233, "sampling/importance_sampling_ratio/mean": 0.999724143743515, "sampling/importance_sampling_ratio/min": 0.7888372838497162, "sampling/sampling_logp_difference/max": 0.24326168298721312, "sampling/sampling_logp_difference/mean": 0.021373784355819225, "step": 10890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.8, "completions/max_terminated_length": 319.8, "completions/mean_length": 224.3875, "completions/mean_terminated_length": 224.3875, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.1475985459983349, "epoch": 1.0021145536453067, "frac_reward_zero_std": 0.5, "grad_norm": 0.101255863904953, "kl": 0.011982133478159084, "learning_rate": 3.9761577155500515e-06, "loss": 0.0004, "num_tokens": 47544490.0, "reward": 0.15, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3005151391029357, "sampling/importance_sampling_ratio/mean": 1.0001923859119415, "sampling/importance_sampling_ratio/min": 0.7497951626777649, "sampling/sampling_logp_difference/max": 0.3161309093236923, "sampling/sampling_logp_difference/mean": 0.02279981318861246, "step": 10900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.9, "completions/max_terminated_length": 289.9, "completions/mean_length": 217.075, "completions/mean_terminated_length": 217.075, "completions/min_length": 147.9, "completions/min_terminated_length": 147.9, "entropy": 1.096438567340374, "epoch": 1.0030339247954398, "frac_reward_zero_std": 0.6, "grad_norm": 6.417723178863525, "kl": 0.0125884247594513, "learning_rate": 3.974112159320453e-06, "loss": 0.0009, "num_tokens": 47587880.0, "reward": -0.2, "reward_std": 0.3552303433418274, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3552303433418274, "sampling/importance_sampling_ratio/max": 1.2976077556610108, "sampling/importance_sampling_ratio/mean": 0.9998224914073944, "sampling/importance_sampling_ratio/min": 0.7289830356836319, "sampling/sampling_logp_difference/max": 0.3590756893157959, "sampling/sampling_logp_difference/mean": 0.022319248132407665, "step": 10910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 311.8, "completions/max_terminated_length": 311.7, "completions/mean_length": 215.2, "completions/mean_terminated_length": 210.74166870117188, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 0.9501383002847433, "epoch": 1.0039532959455733, "frac_reward_zero_std": 0.6, "grad_norm": 5.375459671020508, "kl": 0.011130613228306174, "learning_rate": 3.972065089059503e-06, "loss": 0.0004, "num_tokens": 47631904.0, "reward": -0.1625, "reward_std": 0.36667447090148925, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.3666744947433472, "sampling/importance_sampling_ratio/max": 1.28761146068573, "sampling/importance_sampling_ratio/mean": 1.0002388536930085, "sampling/importance_sampling_ratio/min": 0.7938076853752136, "sampling/sampling_logp_difference/max": 0.2748871684074402, "sampling/sampling_logp_difference/mean": 0.020040078181773426, "step": 10920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 189.8125, "completions/mean_terminated_length": 189.8125, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 1.050632456690073, "epoch": 1.0048726670957064, "frac_reward_zero_std": 0.5, "grad_norm": 4.582894325256348, "kl": 0.012559800368035213, "learning_rate": 3.970016506869706e-06, "loss": 0.0007, "num_tokens": 47673689.0, "reward": -0.48125, "reward_std": 0.47427003979682925, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.4742700457572937, "sampling/importance_sampling_ratio/max": 1.2451741576194764, "sampling/importance_sampling_ratio/mean": 0.9990383327007294, "sampling/importance_sampling_ratio/min": 0.7789610862731934, "sampling/sampling_logp_difference/max": 0.2712019383907318, "sampling/sampling_logp_difference/mean": 0.021190133318305014, "step": 10930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 327.1, "completions/max_terminated_length": 325.4, "completions/mean_length": 223.7, "completions/mean_terminated_length": 217.99583435058594, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.1034546680748463, "epoch": 1.0057920382458398, "frac_reward_zero_std": 0.4, "grad_norm": 0.09546683728694916, "kl": 0.0127999771037139, "learning_rate": 3.967966414855119e-06, "loss": 0.0008, "num_tokens": 47717553.0, "reward": -0.3125, "reward_std": 0.5091521501541137, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.5091521620750428, "sampling/importance_sampling_ratio/max": 1.2261754751205445, "sampling/importance_sampling_ratio/mean": 0.9995520353317261, "sampling/importance_sampling_ratio/min": 0.7514885604381562, "sampling/sampling_logp_difference/max": 0.28927347660064695, "sampling/sampling_logp_difference/mean": 0.021265049278736115, "step": 10940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.1, "completions/max_terminated_length": 313.1, "completions/mean_length": 206.9, "completions/mean_terminated_length": 206.9, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.1140766091644765, "epoch": 1.0067114093959733, "frac_reward_zero_std": 0.4, "grad_norm": 5.316096782684326, "kl": 0.011616931238677353, "learning_rate": 3.965914815121353e-06, "loss": 0.0005, "num_tokens": 47763657.0, "reward": -0.025, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.3249024868011474, "sampling/importance_sampling_ratio/mean": 0.9997348606586456, "sampling/importance_sampling_ratio/min": 0.7714164435863495, "sampling/sampling_logp_difference/max": 0.32184330821037294, "sampling/sampling_logp_difference/mean": 0.022723668813705446, "step": 10950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.0, "completions/max_terminated_length": 260.0, "completions/mean_length": 189.725, "completions/mean_terminated_length": 189.725, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.0804389111697674, "epoch": 1.0076307805461064, "frac_reward_zero_std": 0.7, "grad_norm": 7.653928756713867, "kl": 0.011850291374139487, "learning_rate": 3.963861709775565e-06, "loss": 0.0004, "num_tokens": 47802555.0, "reward": 0.05, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.3273279666900635, "sampling/importance_sampling_ratio/mean": 1.0005610227584838, "sampling/importance_sampling_ratio/min": 0.7550930082798004, "sampling/sampling_logp_difference/max": 0.32707982063293456, "sampling/sampling_logp_difference/mean": 0.02232574038207531, "step": 10960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.1, "completions/max_terminated_length": 320.1, "completions/mean_length": 201.25, "completions/mean_terminated_length": 201.25, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.112407323718071, "epoch": 1.0085501516962398, "frac_reward_zero_std": 0.5, "grad_norm": 6.697634220123291, "kl": 0.01102877560770139, "learning_rate": 3.9618071009264565e-06, "loss": 0.0009, "num_tokens": 47843639.0, "reward": -0.15, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.2591657042503357, "sampling/importance_sampling_ratio/mean": 1.0000011444091796, "sampling/importance_sampling_ratio/min": 0.7677303373813629, "sampling/sampling_logp_difference/max": 0.27403625249862673, "sampling/sampling_logp_difference/mean": 0.022400553710758685, "step": 10970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 203.25, "completions/mean_terminated_length": 203.25, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.054585601016879, "epoch": 1.009469522846373, "frac_reward_zero_std": 0.4, "grad_norm": 6.299182891845703, "kl": 0.01274861343554221, "learning_rate": 3.959750990684278e-06, "loss": 0.001, "num_tokens": 47885539.0, "reward": -0.25, "reward_std": 0.45706321597099303, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.457063227891922, "sampling/importance_sampling_ratio/max": 1.3349952101707458, "sampling/importance_sampling_ratio/mean": 1.00018070936203, "sampling/importance_sampling_ratio/min": 0.763856852054596, "sampling/sampling_logp_difference/max": 0.3131102740764618, "sampling/sampling_logp_difference/mean": 0.02182245533913374, "step": 10980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 337.1, "completions/max_terminated_length": 305.1, "completions/mean_length": 225.1125, "completions/mean_terminated_length": 218.42857513427734, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.1089841097593307, "epoch": 1.0103888939965064, "frac_reward_zero_std": 0.4, "grad_norm": 0.1209523007273674, "kl": 0.014180375105934218, "learning_rate": 3.957693381160819e-06, "loss": 0.001, "num_tokens": 47930236.0, "reward": -0.2375, "reward_std": 0.6159808814525605, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.6159808933734894, "sampling/importance_sampling_ratio/max": 1.2748249769210815, "sampling/importance_sampling_ratio/mean": 0.999982750415802, "sampling/importance_sampling_ratio/min": 0.7579933524131774, "sampling/sampling_logp_difference/max": 0.28384809494018554, "sampling/sampling_logp_difference/mean": 0.02203010767698288, "step": 10990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.1, "completions/max_terminated_length": 270.1, "completions/mean_length": 204.4375, "completions/mean_terminated_length": 204.4375, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.0282232008874417, "epoch": 1.0113082651466396, "frac_reward_zero_std": 0.5, "grad_norm": 0.11833937466144562, "kl": 0.012674096395494416, "learning_rate": 3.95563427446941e-06, "loss": 0.0003, "num_tokens": 47973135.0, "reward": 0.125, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2901448607444763, "sampling/importance_sampling_ratio/mean": 0.9999722182750702, "sampling/importance_sampling_ratio/min": 0.7721134006977082, "sampling/sampling_logp_difference/max": 0.29380475282669066, "sampling/sampling_logp_difference/mean": 0.021725603751838206, "step": 11000 }, { "epoch": 1.0113082651466396, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 289.32111692844677, "eval_completions/max_terminated_length": 287.0331588132635, "eval_completions/mean_length": 202.26767015706807, "eval_completions/mean_terminated_length": 201.7208927281032, "eval_completions/min_length": 131.58813263525306, "eval_completions/min_terminated_length": 131.58813263525306, "eval_entropy": 1.0666385579067583, "eval_frac_reward_zero_std": 0.46596858638743455, "eval_kl": 0.012280205212676504, "eval_loss": 0.0006726750289089978, "eval_num_tokens": 47973135.0, "eval_reward": -0.08867801047120419, "eval_reward_std": 0.46518478530863816, "eval_rewards/reward_fn/mean": -0.08867801047120419, "eval_rewards/reward_fn/std": 0.46518479893552905, "eval_runtime": 1689.3206, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2751796979970749, "eval_sampling/importance_sampling_ratio/mean": 0.9999383104409223, "eval_sampling/importance_sampling_ratio/min": 0.7549623813590662, "eval_sampling/sampling_logp_difference/max": 0.309049535328181, "eval_sampling/sampling_logp_difference/mean": 0.022237387333019762, "eval_steps_per_second": 0.043, "step": 11000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 300.4, "completions/max_terminated_length": 278.6, "completions/mean_length": 207.15, "completions/mean_terminated_length": 203.58928680419922, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0174428172409535, "epoch": 1.012227636296773, "frac_reward_zero_std": 0.4, "grad_norm": 0.14286381006240845, "kl": 0.014011326135369017, "learning_rate": 3.953573672724916e-06, "loss": 0.0007, "num_tokens": 48014179.0, "reward": 0.14375, "reward_std": 0.5569750487804412, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.5569750607013703, "sampling/importance_sampling_ratio/max": 1.2448230624198913, "sampling/importance_sampling_ratio/mean": 0.9995634138584137, "sampling/importance_sampling_ratio/min": 0.730736929178238, "sampling/sampling_logp_difference/max": 0.31840546131134034, "sampling/sampling_logp_difference/mean": 0.021372945979237556, "step": 11010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.9, "completions/max_terminated_length": 267.9, "completions/mean_length": 183.6625, "completions/mean_terminated_length": 183.6625, "completions/min_length": 109.9, "completions/min_terminated_length": 109.9, "entropy": 0.9787335265427828, "epoch": 1.0131470074469062, "frac_reward_zero_std": 0.6, "grad_norm": 0.13669298589229584, "kl": 0.011762327575706878, "learning_rate": 3.951511578043743e-06, "loss": 0.0004, "num_tokens": 48056024.0, "reward": 0.625, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.625, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.238806962966919, "sampling/importance_sampling_ratio/mean": 1.000305312871933, "sampling/importance_sampling_ratio/min": 0.7799228608608246, "sampling/sampling_logp_difference/max": 0.27378504276275634, "sampling/sampling_logp_difference/mean": 0.020147304516285658, "step": 11020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.1, "completions/max_terminated_length": 329.1, "completions/mean_length": 208.2375, "completions/mean_terminated_length": 208.2375, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 0.9933396566659212, "epoch": 1.0140663785970396, "frac_reward_zero_std": 0.4, "grad_norm": 0.13693122565746307, "kl": 0.011405628960346803, "learning_rate": 3.949447992543826e-06, "loss": 0.0, "num_tokens": 48100931.0, "reward": -0.15, "reward_std": 0.5882598340511322, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.588259893655777, "sampling/importance_sampling_ratio/max": 1.243122661113739, "sampling/importance_sampling_ratio/mean": 0.9998728513717652, "sampling/importance_sampling_ratio/min": 0.7810057461261749, "sampling/sampling_logp_difference/max": 0.2501262426376343, "sampling/sampling_logp_difference/mean": 0.02053765272721648, "step": 11030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.5, "completions/max_terminated_length": 274.5, "completions/mean_length": 203.2, "completions/mean_terminated_length": 203.2, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.115533644706011, "epoch": 1.014985749747173, "frac_reward_zero_std": 0.8, "grad_norm": 0.22635255753993988, "kl": 0.011864803597563877, "learning_rate": 3.947382918344633e-06, "loss": 0.0006, "num_tokens": 48142979.0, "reward": -0.4, "reward_std": 0.20701966285705567, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.20701968669891357, "sampling/importance_sampling_ratio/max": 1.2573408246040345, "sampling/importance_sampling_ratio/mean": 1.0005990862846375, "sampling/importance_sampling_ratio/min": 0.774325555562973, "sampling/sampling_logp_difference/max": 0.26276989579200744, "sampling/sampling_logp_difference/mean": 0.021887937001883982, "step": 11040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 206.8125, "completions/mean_terminated_length": 206.8125, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.0743591755628585, "epoch": 1.0159051208973062, "frac_reward_zero_std": 0.3, "grad_norm": 0.12813560664653778, "kl": 0.013007755111902952, "learning_rate": 3.945316357567161e-06, "loss": 0.0004, "num_tokens": 48185628.0, "reward": 0.04375, "reward_std": 0.6626865863800049, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.6626866102218628, "sampling/importance_sampling_ratio/max": 1.2436376810073853, "sampling/importance_sampling_ratio/mean": 1.0002606451511382, "sampling/importance_sampling_ratio/min": 0.7153660625219345, "sampling/sampling_logp_difference/max": 0.36845476627349855, "sampling/sampling_logp_difference/mean": 0.02208632566034794, "step": 11050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.5, "completions/max_terminated_length": 311.5, "completions/mean_length": 217.9125, "completions/mean_terminated_length": 217.9125, "completions/min_length": 153.3, "completions/min_terminated_length": 153.3, "entropy": 1.0853649266064167, "epoch": 1.0168244920474396, "frac_reward_zero_std": 0.5, "grad_norm": 0.11751161515712738, "kl": 0.011899649031693115, "learning_rate": 3.943248312333934e-06, "loss": 0.0005, "num_tokens": 48229541.0, "reward": 0.15, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.3080873489379883, "sampling/importance_sampling_ratio/mean": 0.9994646847248078, "sampling/importance_sampling_ratio/min": 0.7759413123130798, "sampling/sampling_logp_difference/max": 0.2877399086952209, "sampling/sampling_logp_difference/mean": 0.022447820007801055, "step": 11060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.7, "completions/max_terminated_length": 293.7, "completions/mean_length": 198.2625, "completions/mean_terminated_length": 198.2625, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.1556138277053833, "epoch": 1.0177438631975728, "frac_reward_zero_std": 0.6, "grad_norm": 7.024492263793945, "kl": 0.014940342458430678, "learning_rate": 3.941178784768999e-06, "loss": 0.0008, "num_tokens": 48272146.0, "reward": -0.325, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.239829683303833, "sampling/importance_sampling_ratio/mean": 0.9999184906482697, "sampling/importance_sampling_ratio/min": 0.722997534275055, "sampling/sampling_logp_difference/max": 0.33385769426822665, "sampling/sampling_logp_difference/mean": 0.02278587855398655, "step": 11070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 198.1875, "completions/mean_terminated_length": 198.1875, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.0105963110923768, "epoch": 1.0186632343477062, "frac_reward_zero_std": 0.4, "grad_norm": 0.08856610953807831, "kl": 0.013779973331838846, "learning_rate": 3.939107776997928e-06, "loss": 0.0007, "num_tokens": 48315521.0, "reward": -0.175, "reward_std": 0.5882755279541015, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5882755637168884, "sampling/importance_sampling_ratio/max": 1.3486381649971009, "sampling/importance_sampling_ratio/mean": 0.9998264074325561, "sampling/importance_sampling_ratio/min": 0.7284772217273712, "sampling/sampling_logp_difference/max": 0.3707382619380951, "sampling/sampling_logp_difference/mean": 0.02088180314749479, "step": 11080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 200.0625, "completions/mean_terminated_length": 200.0625, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.0081307135522366, "epoch": 1.0195826054978394, "frac_reward_zero_std": 0.6, "grad_norm": 0.11012496054172516, "kl": 0.012927171064075083, "learning_rate": 3.9370352911478115e-06, "loss": 0.0007, "num_tokens": 48354854.0, "reward": -0.2, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.3158208727836609, "sampling/importance_sampling_ratio/mean": 0.9993376433849335, "sampling/importance_sampling_ratio/min": 0.7429900884628295, "sampling/sampling_logp_difference/max": 0.3391778588294983, "sampling/sampling_logp_difference/mean": 0.020964860264211894, "step": 11090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.8, "completions/max_terminated_length": 257.8, "completions/mean_length": 198.0125, "completions/mean_terminated_length": 198.0125, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0605291992425918, "epoch": 1.0205019766479728, "frac_reward_zero_std": 0.7, "grad_norm": 6.169411659240723, "kl": 0.011880262079648674, "learning_rate": 3.9349613293472575e-06, "loss": 0.0011, "num_tokens": 48396015.0, "reward": -0.175, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2994048714637756, "sampling/importance_sampling_ratio/mean": 0.9998404145240783, "sampling/importance_sampling_ratio/min": 0.771422016620636, "sampling/sampling_logp_difference/max": 0.3180035829544067, "sampling/sampling_logp_difference/mean": 0.02197933755815029, "step": 11100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.0, "completions/max_terminated_length": 308.0, "completions/mean_length": 212.6625, "completions/mean_terminated_length": 212.6625, "completions/min_length": 150.6, "completions/min_terminated_length": 150.6, "entropy": 1.0576028101146222, "epoch": 1.021421347798106, "frac_reward_zero_std": 0.4, "grad_norm": 6.384306907653809, "kl": 0.010777623439207674, "learning_rate": 3.932885893726394e-06, "loss": 0.0003, "num_tokens": 48437340.0, "reward": -0.275, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2493065357208253, "sampling/importance_sampling_ratio/mean": 0.9995339274406433, "sampling/importance_sampling_ratio/min": 0.7500420898199082, "sampling/sampling_logp_difference/max": 0.32480884790420533, "sampling/sampling_logp_difference/mean": 0.021993971057236196, "step": 11110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.5, "completions/max_terminated_length": 266.5, "completions/mean_length": 195.0125, "completions/mean_terminated_length": 195.0125, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.033710353821516, "epoch": 1.0223407189482394, "frac_reward_zero_std": 0.6, "grad_norm": 9.816394805908203, "kl": 0.011916697793640196, "learning_rate": 3.930808986416855e-06, "loss": 0.0007, "num_tokens": 48478941.0, "reward": 0.225, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.302098536491394, "sampling/importance_sampling_ratio/mean": 1.0003216803073882, "sampling/importance_sampling_ratio/min": 0.780885761976242, "sampling/sampling_logp_difference/max": 0.28467867374420164, "sampling/sampling_logp_difference/mean": 0.021464051492512227, "step": 11120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.3, "completions/max_terminated_length": 280.3, "completions/mean_length": 193.3375, "completions/mean_terminated_length": 193.3375, "completions/min_length": 110.6, "completions/min_terminated_length": 110.6, "entropy": 1.0197542317211628, "epoch": 1.0232600900983728, "frac_reward_zero_std": 0.5, "grad_norm": 0.14544712007045746, "kl": 0.012955218291608617, "learning_rate": 3.928730609551796e-06, "loss": 0.0008, "num_tokens": 48518080.0, "reward": 0.05, "reward_std": 0.4881446659564972, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.48814470171928404, "sampling/importance_sampling_ratio/max": 1.3189576625823975, "sampling/importance_sampling_ratio/mean": 0.9996566712856293, "sampling/importance_sampling_ratio/min": 0.7634957730770111, "sampling/sampling_logp_difference/max": 0.31993948221206664, "sampling/sampling_logp_difference/mean": 0.021056263893842696, "step": 11130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 351.3, "completions/max_terminated_length": 351.3, "completions/mean_length": 218.375, "completions/mean_terminated_length": 218.375, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0449808582663536, "epoch": 1.024179461248506, "frac_reward_zero_std": 0.4, "grad_norm": 0.13378651440143585, "kl": 0.01264138956030365, "learning_rate": 3.926650765265873e-06, "loss": 0.0004, "num_tokens": 48562630.0, "reward": 0.35, "reward_std": 0.49665169715881347, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.49665169715881347, "sampling/importance_sampling_ratio/max": 1.2779836297035216, "sampling/importance_sampling_ratio/mean": 1.000650370121002, "sampling/importance_sampling_ratio/min": 0.7654518485069275, "sampling/sampling_logp_difference/max": 0.2980603456497192, "sampling/sampling_logp_difference/mean": 0.02133942786604166, "step": 11140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.3, "completions/max_terminated_length": 290.3, "completions/mean_length": 193.625, "completions/mean_terminated_length": 193.625, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0263184577226638, "epoch": 1.0250988323986394, "frac_reward_zero_std": 0.3, "grad_norm": 6.371429920196533, "kl": 0.014237830776255577, "learning_rate": 3.924569455695255e-06, "loss": 0.001, "num_tokens": 48602064.0, "reward": 0.25, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.3158673167228698, "sampling/importance_sampling_ratio/mean": 0.9995928287506104, "sampling/importance_sampling_ratio/min": 0.7822855472564697, "sampling/sampling_logp_difference/max": 0.2803648948669434, "sampling/sampling_logp_difference/mean": 0.021117339469492437, "step": 11150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.8, "completions/max_terminated_length": 250.8, "completions/mean_length": 182.675, "completions/mean_terminated_length": 182.675, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.04907458499074, "epoch": 1.0260182035487726, "frac_reward_zero_std": 0.4, "grad_norm": 0.10365435481071472, "kl": 0.01473314329632558, "learning_rate": 3.922486682977612e-06, "loss": 0.0006, "num_tokens": 48640646.0, "reward": 0.475, "reward_std": 0.5841213405132294, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.5841213643550873, "sampling/importance_sampling_ratio/max": 1.2362766861915588, "sampling/importance_sampling_ratio/mean": 1.0001892626285553, "sampling/importance_sampling_ratio/min": 0.7849908590316772, "sampling/sampling_logp_difference/max": 0.25057594776153563, "sampling/sampling_logp_difference/mean": 0.02182502318173647, "step": 11160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 352.7, "completions/max_terminated_length": 334.0, "completions/mean_length": 231.2375, "completions/mean_terminated_length": 227.07321472167968, "completions/min_length": 153.7, "completions/min_terminated_length": 153.7, "entropy": 1.069216436892748, "epoch": 1.026937574698906, "frac_reward_zero_std": 0.4, "grad_norm": 0.10825134068727493, "kl": 0.011672251025447621, "learning_rate": 3.920402449252119e-06, "loss": 0.0009, "num_tokens": 48690921.0, "reward": -0.2125, "reward_std": 0.5608916282653809, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.5608916282653809, "sampling/importance_sampling_ratio/max": 1.3098754048347474, "sampling/importance_sampling_ratio/mean": 1.0003329455852508, "sampling/importance_sampling_ratio/min": 0.7751417279243469, "sampling/sampling_logp_difference/max": 0.29227988719940184, "sampling/sampling_logp_difference/mean": 0.021208170987665653, "step": 11170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.8, "completions/max_terminated_length": 275.8, "completions/mean_length": 194.0375, "completions/mean_terminated_length": 194.0375, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 0.9919597506523132, "epoch": 1.0278569458490392, "frac_reward_zero_std": 0.6, "grad_norm": 5.5540642738342285, "kl": 0.013016691018128768, "learning_rate": 3.918316756659451e-06, "loss": 0.0007, "num_tokens": 48732724.0, "reward": 0.175, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2567264676094054, "sampling/importance_sampling_ratio/mean": 0.9996828496456146, "sampling/importance_sampling_ratio/min": 0.7537489831447601, "sampling/sampling_logp_difference/max": 0.29254813194274903, "sampling/sampling_logp_difference/mean": 0.020818907395005228, "step": 11180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.3, "completions/max_terminated_length": 286.3, "completions/mean_length": 198.925, "completions/mean_terminated_length": 198.925, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.0753859736025333, "epoch": 1.0287763169991726, "frac_reward_zero_std": 0.5, "grad_norm": 5.485382556915283, "kl": 0.013433269882807509, "learning_rate": 3.916229607341784e-06, "loss": 0.001, "num_tokens": 48771886.0, "reward": -0.15, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.3036324024200439, "sampling/importance_sampling_ratio/mean": 1.000176101922989, "sampling/importance_sampling_ratio/min": 0.7517321467399597, "sampling/sampling_logp_difference/max": 0.32063148319721224, "sampling/sampling_logp_difference/mean": 0.022442197240889073, "step": 11190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 345.8, "completions/max_terminated_length": 345.8, "completions/mean_length": 231.05, "completions/mean_terminated_length": 231.05, "completions/min_length": 151.3, "completions/min_terminated_length": 151.3, "entropy": 0.9836396843194961, "epoch": 1.0296956881493058, "frac_reward_zero_std": 0.3, "grad_norm": 0.09697076678276062, "kl": 0.010530374979134649, "learning_rate": 3.914141003442784e-06, "loss": 0.0005, "num_tokens": 48824610.0, "reward": -0.025, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.2354024052619934, "sampling/importance_sampling_ratio/mean": 1.0004201114177704, "sampling/importance_sampling_ratio/min": 0.7520128905773162, "sampling/sampling_logp_difference/max": 0.2878832697868347, "sampling/sampling_logp_difference/mean": 0.019991323910653593, "step": 11200 }, { "epoch": 1.0296956881493058, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 308.97731239092496, "eval_completions/max_terminated_length": 307.4869109947644, "eval_completions/mean_length": 217.20724258289704, "eval_completions/mean_terminated_length": 216.57992404584903, "eval_completions/min_length": 142.12739965095986, "eval_completions/min_terminated_length": 142.12739965095986, "eval_entropy": 1.0946668304281917, "eval_frac_reward_zero_std": 0.4781849912739965, "eval_kl": 0.012034103269469364, "eval_loss": 0.0005955103551968932, "eval_num_tokens": 48824610.0, "eval_reward": -0.10962041884816753, "eval_reward_std": 0.46446717505783724, "eval_rewards/reward_fn/mean": -0.10962041884816753, "eval_rewards/reward_fn/std": 0.4644671909732136, "eval_runtime": 1791.1517, "eval_samples_per_second": 0.32, "eval_sampling/importance_sampling_ratio/max": 1.274242328931732, "eval_sampling/importance_sampling_ratio/mean": 0.9999610781669617, "eval_sampling/importance_sampling_ratio/min": 0.7669180130771317, "eval_sampling/sampling_logp_difference/max": 0.29457904962761033, "eval_sampling/sampling_logp_difference/mean": 0.021457105575951815, "eval_steps_per_second": 0.04, "step": 11200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.1, "completions/max_terminated_length": 284.1, "completions/mean_length": 194.25, "completions/mean_terminated_length": 194.25, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0008852936327457, "epoch": 1.0306150592994392, "frac_reward_zero_std": 0.5, "grad_norm": 0.28401684761047363, "kl": 0.013380560564110055, "learning_rate": 3.912050947107617e-06, "loss": 0.0008, "num_tokens": 48864486.0, "reward": 0.15, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.34915691614151, "sampling/importance_sampling_ratio/mean": 1.0004407167434692, "sampling/importance_sampling_ratio/min": 0.7642388641834259, "sampling/sampling_logp_difference/max": 0.35341944694519045, "sampling/sampling_logp_difference/mean": 0.02090334687381983, "step": 11210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/max_terminated_length": 275.3, "completions/mean_length": 207.8875, "completions/mean_terminated_length": 207.8875, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.0732942007482051, "epoch": 1.0315344304495726, "frac_reward_zero_std": 0.5, "grad_norm": 5.498587131500244, "kl": 0.012092841247795149, "learning_rate": 3.909959440482937e-06, "loss": 0.0002, "num_tokens": 48907085.0, "reward": -0.13125, "reward_std": 0.4522722601890564, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.45227227210998533, "sampling/importance_sampling_ratio/max": 1.3099765181541443, "sampling/importance_sampling_ratio/mean": 0.9995568096637726, "sampling/importance_sampling_ratio/min": 0.7292412042617797, "sampling/sampling_logp_difference/max": 0.35706021785736086, "sampling/sampling_logp_difference/mean": 0.02182839121669531, "step": 11220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.1, "completions/max_terminated_length": 269.1, "completions/mean_length": 201.4125, "completions/mean_terminated_length": 201.4125, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 0.9671141259372235, "epoch": 1.0324538015997058, "frac_reward_zero_std": 0.6, "grad_norm": 0.08664987981319427, "kl": 0.013095744169550016, "learning_rate": 3.9078664857168895e-06, "loss": 0.0003, "num_tokens": 48946454.0, "reward": -0.125, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.270704483985901, "sampling/importance_sampling_ratio/mean": 0.9999219417572022, "sampling/importance_sampling_ratio/min": 0.7709027111530304, "sampling/sampling_logp_difference/max": 0.29471745491027834, "sampling/sampling_logp_difference/mean": 0.02034684382379055, "step": 11230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.8, "completions/max_terminated_length": 260.8, "completions/mean_length": 186.4125, "completions/mean_terminated_length": 186.4125, "completions/min_length": 121.6, "completions/min_terminated_length": 121.6, "entropy": 1.0488259248435496, "epoch": 1.0333731727498392, "frac_reward_zero_std": 0.5, "grad_norm": 0.14891086518764496, "kl": 0.011846903822151944, "learning_rate": 3.905772084959107e-06, "loss": 0.0001, "num_tokens": 48985191.0, "reward": 0.05, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.3083297848701476, "sampling/importance_sampling_ratio/mean": 1.0008127033710479, "sampling/importance_sampling_ratio/min": 0.7388583451509476, "sampling/sampling_logp_difference/max": 0.3286851167678833, "sampling/sampling_logp_difference/mean": 0.021554893255233763, "step": 11240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 295.1, "completions/max_terminated_length": 292.6, "completions/mean_length": 213.6375, "completions/mean_terminated_length": 211.40357360839843, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.0274410344660283, "epoch": 1.0342925438999724, "frac_reward_zero_std": 0.4, "grad_norm": 0.10038329660892487, "kl": 0.012952277477597817, "learning_rate": 3.903676240360706e-06, "loss": 0.0009, "num_tokens": 49029258.0, "reward": -0.3125, "reward_std": 0.48943452835083007, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.48943454027175903, "sampling/importance_sampling_ratio/max": 1.2465028285980224, "sampling/importance_sampling_ratio/mean": 0.9996135234832764, "sampling/importance_sampling_ratio/min": 0.7583682000637054, "sampling/sampling_logp_difference/max": 0.2927185595035553, "sampling/sampling_logp_difference/mean": 0.021291779540479183, "step": 11250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 337.9, "completions/max_terminated_length": 329.3, "completions/mean_length": 240.3125, "completions/mean_terminated_length": 229.5125, "completions/min_length": 155.0, "completions/min_terminated_length": 155.0, "entropy": 1.1211106978356837, "epoch": 1.0352119150501058, "frac_reward_zero_std": 0.3, "grad_norm": 6.096879005432129, "kl": 0.012946603738237172, "learning_rate": 3.90157895407429e-06, "loss": 0.0003, "num_tokens": 49078187.0, "reward": -0.475, "reward_std": 0.7000348150730134, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.7000348508358002, "sampling/importance_sampling_ratio/max": 1.2378435015678406, "sampling/importance_sampling_ratio/mean": 0.9996953666210174, "sampling/importance_sampling_ratio/min": 0.7493651509284973, "sampling/sampling_logp_difference/max": 0.2922859191894531, "sampling/sampling_logp_difference/mean": 0.022397983446717264, "step": 11260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.0, "completions/max_terminated_length": 327.0, "completions/mean_length": 229.6375, "completions/mean_terminated_length": 229.6375, "completions/min_length": 154.1, "completions/min_terminated_length": 154.1, "entropy": 1.0686769105494023, "epoch": 1.036131286200239, "frac_reward_zero_std": 0.6, "grad_norm": 0.12725220620632172, "kl": 0.012996875424869358, "learning_rate": 3.899480228253936e-06, "loss": 0.0007, "num_tokens": 49123782.0, "reward": -0.375, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2331422805786132, "sampling/importance_sampling_ratio/mean": 0.9999563694000244, "sampling/importance_sampling_ratio/min": 0.7896353900432587, "sampling/sampling_logp_difference/max": 0.2410505533218384, "sampling/sampling_logp_difference/mean": 0.021381815522909166, "step": 11270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 203.0375, "completions/mean_terminated_length": 203.0375, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.1391637817025184, "epoch": 1.0370506573503724, "frac_reward_zero_std": 0.4, "grad_norm": 6.0452070236206055, "kl": 0.01340807974920608, "learning_rate": 3.8973800650552055e-06, "loss": 0.0009, "num_tokens": 49166769.0, "reward": 0.1, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.2621471762657166, "sampling/importance_sampling_ratio/mean": 0.9994838178157807, "sampling/importance_sampling_ratio/min": 0.7519998490810395, "sampling/sampling_logp_difference/max": 0.3061943888664246, "sampling/sampling_logp_difference/mean": 0.02224174216389656, "step": 11280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 342.5, "completions/max_terminated_length": 342.5, "completions/mean_length": 225.7, "completions/mean_terminated_length": 225.7, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.090907410532236, "epoch": 1.0379700285005056, "frac_reward_zero_std": 0.5, "grad_norm": 5.039519309997559, "kl": 0.012661583547014743, "learning_rate": 3.895278466635135e-06, "loss": 0.0008, "num_tokens": 49215369.0, "reward": -0.125, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3125270485877991, "sampling/importance_sampling_ratio/mean": 1.0000868141651154, "sampling/importance_sampling_ratio/min": 0.7229714393615723, "sampling/sampling_logp_difference/max": 0.3608816385269165, "sampling/sampling_logp_difference/mean": 0.02120861131697893, "step": 11290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.7, "completions/max_terminated_length": 314.9, "completions/mean_length": 218.9875, "completions/mean_terminated_length": 215.63392944335936, "completions/min_length": 151.9, "completions/min_terminated_length": 151.9, "entropy": 1.1030934996902944, "epoch": 1.038889399650639, "frac_reward_zero_std": 0.5, "grad_norm": 0.10633941739797592, "kl": 0.011882779491133987, "learning_rate": 3.893175435152234e-06, "loss": 0.0005, "num_tokens": 49258544.0, "reward": 0.16875, "reward_std": 0.4162048876285553, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.4162048935890198, "sampling/importance_sampling_ratio/max": 1.2653239846229554, "sampling/importance_sampling_ratio/mean": 0.9999420642852783, "sampling/importance_sampling_ratio/min": 0.754098790884018, "sampling/sampling_logp_difference/max": 0.31006699800491333, "sampling/sampling_logp_difference/mean": 0.021820681728422643, "step": 11300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.6, "completions/max_terminated_length": 314.6, "completions/mean_length": 224.475, "completions/mean_terminated_length": 224.475, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.020575575158, "epoch": 1.0398087708007724, "frac_reward_zero_std": 0.4, "grad_norm": 7.12863826751709, "kl": 0.012081520585343241, "learning_rate": 3.891070972766484e-06, "loss": 0.0005, "num_tokens": 49303702.0, "reward": -0.05, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.300493347644806, "sampling/importance_sampling_ratio/mean": 1.0001903414726256, "sampling/importance_sampling_ratio/min": 0.7643527030944824, "sampling/sampling_logp_difference/max": 0.28899269104003905, "sampling/sampling_logp_difference/mean": 0.021093147434294225, "step": 11310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.0, "completions/max_terminated_length": 296.0, "completions/mean_length": 209.9875, "completions/mean_terminated_length": 209.9875, "completions/min_length": 120.2, "completions/min_terminated_length": 120.2, "entropy": 1.0661780476570129, "epoch": 1.0407281419509056, "frac_reward_zero_std": 0.6, "grad_norm": 6.358207702636719, "kl": 0.011870672303484753, "learning_rate": 3.888965081639337e-06, "loss": 0.0004, "num_tokens": 49345221.0, "reward": -0.225, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2703086614608765, "sampling/importance_sampling_ratio/mean": 1.0002701103687286, "sampling/importance_sampling_ratio/min": 0.7701701581478119, "sampling/sampling_logp_difference/max": 0.26885907649993895, "sampling/sampling_logp_difference/mean": 0.021037255600094795, "step": 11320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.8, "completions/max_terminated_length": 297.8, "completions/mean_length": 204.7375, "completions/mean_terminated_length": 204.7375, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.0639249227941037, "epoch": 1.041647513101039, "frac_reward_zero_std": 0.4, "grad_norm": 0.09935665875673294, "kl": 0.012328722339589149, "learning_rate": 3.8868577639337134e-06, "loss": 0.0012, "num_tokens": 49389944.0, "reward": -0.2, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.2692451000213623, "sampling/importance_sampling_ratio/mean": 1.0003483593463898, "sampling/importance_sampling_ratio/min": 0.7709422647953034, "sampling/sampling_logp_difference/max": 0.27484177350997924, "sampling/sampling_logp_difference/mean": 0.021281786635518073, "step": 11330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 301.7, "completions/max_terminated_length": 298.0, "completions/mean_length": 210.1, "completions/mean_terminated_length": 207.90178833007812, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.043719507753849, "epoch": 1.0425668842511722, "frac_reward_zero_std": 0.5, "grad_norm": 0.12085122615098953, "kl": 0.011677204092848115, "learning_rate": 3.8847490218139945e-06, "loss": 0.0004, "num_tokens": 49429736.0, "reward": -0.08125, "reward_std": 0.40972254872322084, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.40972254872322084, "sampling/importance_sampling_ratio/max": 1.255012047290802, "sampling/importance_sampling_ratio/mean": 1.0005573451519012, "sampling/importance_sampling_ratio/min": 0.7757728278636933, "sampling/sampling_logp_difference/max": 0.26816582679748535, "sampling/sampling_logp_difference/mean": 0.02137120384722948, "step": 11340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 196.5125, "completions/mean_terminated_length": 196.5125, "completions/min_length": 135.5, "completions/min_terminated_length": 135.5, "entropy": 1.1152990221977235, "epoch": 1.0434862554013056, "frac_reward_zero_std": 0.2, "grad_norm": 6.749570846557617, "kl": 0.01267337606404908, "learning_rate": 3.88263885744603e-06, "loss": 0.0003, "num_tokens": 49469329.0, "reward": -0.075, "reward_std": 0.791141003370285, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.7911410510540009, "sampling/importance_sampling_ratio/max": 1.242386531829834, "sampling/importance_sampling_ratio/mean": 1.0003835022449494, "sampling/importance_sampling_ratio/min": 0.7647222936153412, "sampling/sampling_logp_difference/max": 0.278611421585083, "sampling/sampling_logp_difference/mean": 0.02194594480097294, "step": 11350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.2, "completions/max_terminated_length": 321.2, "completions/mean_length": 203.8125, "completions/mean_terminated_length": 203.8125, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.102077254652977, "epoch": 1.0444056265514388, "frac_reward_zero_std": 0.2, "grad_norm": 5.517813682556152, "kl": 0.013295578479301184, "learning_rate": 3.880527272997126e-06, "loss": 0.0009, "num_tokens": 49510882.0, "reward": -0.125, "reward_std": 0.6203559041023254, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.6203559160232544, "sampling/importance_sampling_ratio/max": 1.3139956593513489, "sampling/importance_sampling_ratio/mean": 0.9999396800994873, "sampling/importance_sampling_ratio/min": 0.7504272818565368, "sampling/sampling_logp_difference/max": 0.32207608222961426, "sampling/sampling_logp_difference/mean": 0.02201604600995779, "step": 11360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.1, "completions/max_terminated_length": 306.1, "completions/mean_length": 210.05, "completions/mean_terminated_length": 210.05, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.1235182344913484, "epoch": 1.0453249977015722, "frac_reward_zero_std": 0.3, "grad_norm": 6.009998798370361, "kl": 0.014630350231891498, "learning_rate": 3.87841427063605e-06, "loss": 0.0005, "num_tokens": 49553766.0, "reward": -0.325, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.281016218662262, "sampling/importance_sampling_ratio/mean": 1.0007501780986785, "sampling/importance_sampling_ratio/min": 0.7947853147983551, "sampling/sampling_logp_difference/max": 0.25943883657455447, "sampling/sampling_logp_difference/mean": 0.02174790482968092, "step": 11370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.2, "completions/max_terminated_length": 294.2, "completions/mean_length": 204.775, "completions/mean_terminated_length": 204.775, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 0.9753870207816362, "epoch": 1.0462443688517054, "frac_reward_zero_std": 0.8, "grad_norm": 0.09832281619310379, "kl": 0.013801208772929386, "learning_rate": 3.876299852533025e-06, "loss": 0.0008, "num_tokens": 49598196.0, "reward": -0.65, "reward_std": 0.19948650598526002, "rewards/reward_fn/mean": -0.65, "rewards/reward_fn/std": 0.19948650598526002, "sampling/importance_sampling_ratio/max": 1.2612277746200562, "sampling/importance_sampling_ratio/mean": 0.9999829411506653, "sampling/importance_sampling_ratio/min": 0.7636411964893342, "sampling/sampling_logp_difference/max": 0.2846742868423462, "sampling/sampling_logp_difference/mean": 0.020243800431489944, "step": 11380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.8, "completions/max_terminated_length": 268.8, "completions/mean_length": 203.825, "completions/mean_terminated_length": 203.825, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.032214344292879, "epoch": 1.0471637400018388, "frac_reward_zero_std": 0.6, "grad_norm": 0.09909027814865112, "kl": 0.011282480630325154, "learning_rate": 3.874184020859727e-06, "loss": 0.0007, "num_tokens": 49641046.0, "reward": 0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2680658221244812, "sampling/importance_sampling_ratio/mean": 1.0000929296016694, "sampling/importance_sampling_ratio/min": 0.7891350269317627, "sampling/sampling_logp_difference/max": 0.2567887306213379, "sampling/sampling_logp_difference/mean": 0.02140982113778591, "step": 11390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 202.275, "completions/mean_terminated_length": 202.275, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0132545970380307, "epoch": 1.048083111151972, "frac_reward_zero_std": 0.7, "grad_norm": 0.13034018874168396, "kl": 0.011459749704226852, "learning_rate": 3.8720667777892854e-06, "loss": 0.0005, "num_tokens": 49681980.0, "reward": 0.15, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2578633427619934, "sampling/importance_sampling_ratio/mean": 0.9998104691505432, "sampling/importance_sampling_ratio/min": 0.7825340807437897, "sampling/sampling_logp_difference/max": 0.2563755869865417, "sampling/sampling_logp_difference/mean": 0.020131714921444655, "step": 11400 }, { "epoch": 1.048083111151972, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 299.19895287958116, "eval_completions/max_terminated_length": 297.4240837696335, "eval_completions/mean_length": 207.80977312390925, "eval_completions/mean_terminated_length": 207.37291212897443, "eval_completions/min_length": 135.20942408376965, "eval_completions/min_terminated_length": 135.20942408376965, "eval_entropy": 1.082604638777062, "eval_frac_reward_zero_std": 0.4677137870855148, "eval_kl": 0.012091753788188386, "eval_loss": 0.0005968280602246523, "eval_num_tokens": 49681980.0, "eval_reward": -0.10383944153577661, "eval_reward_std": 0.46132939921311683, "eval_rewards/reward_fn/mean": -0.10383944153577661, "eval_rewards/reward_fn/std": 0.4613294112796767, "eval_runtime": 1739.7769, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.2811716497464538, "eval_sampling/importance_sampling_ratio/mean": 1.0000206533734919, "eval_sampling/importance_sampling_ratio/min": 0.772622075068389, "eval_sampling/sampling_logp_difference/max": 0.2929233185699889, "eval_sampling/sampling_logp_difference/mean": 0.02121710469279874, "eval_steps_per_second": 0.041, "step": 11400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.0, "completions/max_terminated_length": 277.0, "completions/mean_length": 199.65, "completions/mean_terminated_length": 199.65, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 0.972197837382555, "epoch": 1.0490024823021054, "frac_reward_zero_std": 0.3, "grad_norm": 0.14206066727638245, "kl": 0.012821767243440262, "learning_rate": 3.869948125496278e-06, "loss": 0.0009, "num_tokens": 49724832.0, "reward": -0.075, "reward_std": 0.6001615285873413, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6001615405082703, "sampling/importance_sampling_ratio/max": 1.3095091104507446, "sampling/importance_sampling_ratio/mean": 0.9998895049095153, "sampling/importance_sampling_ratio/min": 0.7485587179660798, "sampling/sampling_logp_difference/max": 0.33096650838851926, "sampling/sampling_logp_difference/mean": 0.02062437925487757, "step": 11410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 195.675, "completions/mean_terminated_length": 195.675, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 1.0133689280599356, "epoch": 1.0499218534522388, "frac_reward_zero_std": 0.5, "grad_norm": 0.13699328899383545, "kl": 0.011245220198179594, "learning_rate": 3.867828066156731e-06, "loss": 0.0005, "num_tokens": 49765430.0, "reward": 0.025, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3226598143577575, "sampling/importance_sampling_ratio/mean": 1.000219887495041, "sampling/importance_sampling_ratio/min": 0.7736425638198853, "sampling/sampling_logp_difference/max": 0.30491560101509096, "sampling/sampling_logp_difference/mean": 0.02047500591725111, "step": 11420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.7, "completions/max_terminated_length": 267.7, "completions/mean_length": 201.55, "completions/mean_terminated_length": 201.55, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.0684121113270522, "epoch": 1.050841224602372, "frac_reward_zero_std": 0.8, "grad_norm": 0.10336429625749588, "kl": 0.011629712567082607, "learning_rate": 3.865706601948116e-06, "loss": 0.0007, "num_tokens": 49812114.0, "reward": 0.125, "reward_std": 0.1960918426513672, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.19609185457229614, "sampling/importance_sampling_ratio/max": 1.2246903777122498, "sampling/importance_sampling_ratio/mean": 1.0003299355506896, "sampling/importance_sampling_ratio/min": 0.7723728358745575, "sampling/sampling_logp_difference/max": 0.26472503542900083, "sampling/sampling_logp_difference/mean": 0.02169965784996748, "step": 11430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.7, "completions/max_terminated_length": 294.7, "completions/mean_length": 222.0625, "completions/mean_terminated_length": 222.0625, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0378835648298264, "epoch": 1.0517605957525054, "frac_reward_zero_std": 0.4, "grad_norm": 0.13455000519752502, "kl": 0.010222897568019107, "learning_rate": 3.863583735049345e-06, "loss": 0.0003, "num_tokens": 49856343.0, "reward": 0.0, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.2769035816192627, "sampling/importance_sampling_ratio/mean": 1.0000732481479644, "sampling/importance_sampling_ratio/min": 0.7892293453216552, "sampling/sampling_logp_difference/max": 0.26728981733322144, "sampling/sampling_logp_difference/mean": 0.021069535799324513, "step": 11440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.7, "completions/max_terminated_length": 320.7, "completions/mean_length": 224.15, "completions/mean_terminated_length": 224.15, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0578704535961152, "epoch": 1.0526799669026385, "frac_reward_zero_std": 0.6, "grad_norm": 5.754577159881592, "kl": 0.012268659466644748, "learning_rate": 3.861459467640774e-06, "loss": 0.0007, "num_tokens": 49901499.0, "reward": 0.375, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2603435754776, "sampling/importance_sampling_ratio/mean": 1.0001131653785706, "sampling/importance_sampling_ratio/min": 0.7432458698749542, "sampling/sampling_logp_difference/max": 0.30799046754837034, "sampling/sampling_logp_difference/mean": 0.02147901300340891, "step": 11450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.2, "completions/max_terminated_length": 268.2, "completions/mean_length": 193.675, "completions/mean_terminated_length": 193.675, "completions/min_length": 121.6, "completions/min_terminated_length": 121.6, "entropy": 1.0891003482043744, "epoch": 1.053599338052772, "frac_reward_zero_std": 0.4, "grad_norm": 6.526791572570801, "kl": 0.011839455674635246, "learning_rate": 3.859333801904196e-06, "loss": 0.0002, "num_tokens": 49944681.0, "reward": -0.025, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2860971093177795, "sampling/importance_sampling_ratio/mean": 0.9999497592449188, "sampling/importance_sampling_ratio/min": 0.7381620287895203, "sampling/sampling_logp_difference/max": 0.3301543056964874, "sampling/sampling_logp_difference/mean": 0.02192082554101944, "step": 11460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 211.15, "completions/mean_terminated_length": 211.15, "completions/min_length": 121.8, "completions/min_terminated_length": 121.8, "entropy": 1.1125635713338853, "epoch": 1.0545187092029051, "frac_reward_zero_std": 0.4, "grad_norm": 6.988927364349365, "kl": 0.012130877753952518, "learning_rate": 3.85720674002284e-06, "loss": 0.0005, "num_tokens": 49988893.0, "reward": -0.35625, "reward_std": 0.5205168306827546, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.5205168485641479, "sampling/importance_sampling_ratio/max": 1.2894035458564759, "sampling/importance_sampling_ratio/mean": 0.999910193681717, "sampling/importance_sampling_ratio/min": 0.7130011074244976, "sampling/sampling_logp_difference/max": 0.4759462594985962, "sampling/sampling_logp_difference/mean": 0.022314657084643842, "step": 11470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.9, "completions/max_terminated_length": 288.9, "completions/mean_length": 194.8875, "completions/mean_terminated_length": 194.8875, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 0.9913123443722724, "epoch": 1.0554380803530385, "frac_reward_zero_std": 0.4, "grad_norm": 5.194202423095703, "kl": 0.011956768698291853, "learning_rate": 3.855078284181367e-06, "loss": 0.0006, "num_tokens": 50031796.0, "reward": -0.25, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.2860854625701905, "sampling/importance_sampling_ratio/mean": 1.0003305912017821, "sampling/importance_sampling_ratio/min": 0.7794673442840576, "sampling/sampling_logp_difference/max": 0.2854765057563782, "sampling/sampling_logp_difference/mean": 0.02077314741909504, "step": 11480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 217.15, "completions/mean_terminated_length": 217.15, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0929586686193944, "epoch": 1.0563574515031717, "frac_reward_zero_std": 0.6, "grad_norm": 5.850671291351318, "kl": 0.011769636342069133, "learning_rate": 3.852948436565874e-06, "loss": 0.0008, "num_tokens": 50076840.0, "reward": -0.25, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.244045090675354, "sampling/importance_sampling_ratio/mean": 0.9998554527759552, "sampling/importance_sampling_ratio/min": 0.7906237065792083, "sampling/sampling_logp_difference/max": 0.24729437828063966, "sampling/sampling_logp_difference/mean": 0.021712209284305572, "step": 11490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 191.4125, "completions/mean_terminated_length": 191.4125, "completions/min_length": 112.6, "completions/min_terminated_length": 112.6, "entropy": 0.9589162796735764, "epoch": 1.0572768226533051, "frac_reward_zero_std": 0.6, "grad_norm": 0.08879353106021881, "kl": 0.00993330758647062, "learning_rate": 3.850817199363884e-06, "loss": 0.0004, "num_tokens": 50117489.0, "reward": 0.1, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.3541927695274354, "sampling/importance_sampling_ratio/mean": 0.9996457397937775, "sampling/importance_sampling_ratio/min": 0.7764240026473999, "sampling/sampling_logp_difference/max": 0.3285588800907135, "sampling/sampling_logp_difference/mean": 0.02091619335114956, "step": 11500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 202.45, "completions/mean_terminated_length": 202.45, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.0595204167068004, "epoch": 1.0581961938034385, "frac_reward_zero_std": 0.5, "grad_norm": 7.759405612945557, "kl": 0.012182599381776526, "learning_rate": 3.84868457476435e-06, "loss": 0.0004, "num_tokens": 50161397.0, "reward": 0.125, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2466429471969604, "sampling/importance_sampling_ratio/mean": 0.9997367560863495, "sampling/importance_sampling_ratio/min": 0.7725393831729889, "sampling/sampling_logp_difference/max": 0.2712979793548584, "sampling/sampling_logp_difference/mean": 0.021095259115099907, "step": 11510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 386.7, "completions/max_terminated_length": 369.6, "completions/mean_length": 252.9875, "completions/mean_terminated_length": 249.44464416503905, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.10062515437603, "epoch": 1.0591155649535717, "frac_reward_zero_std": 0.4, "grad_norm": 5.315279960632324, "kl": 0.010802366049028933, "learning_rate": 3.8465505649576465e-06, "loss": 0.0005, "num_tokens": 50209892.0, "reward": -0.2875, "reward_std": 0.5684404790401458, "rewards/reward_fn/mean": -0.2875, "rewards/reward_fn/std": 0.5684404790401458, "sampling/importance_sampling_ratio/max": 1.2595566153526305, "sampling/importance_sampling_ratio/mean": 1.0001823008060455, "sampling/importance_sampling_ratio/min": 0.7868696570396423, "sampling/sampling_logp_difference/max": 0.24926526546478273, "sampling/sampling_logp_difference/mean": 0.021618236042559146, "step": 11520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 343.4, "completions/max_terminated_length": 327.4, "completions/mean_length": 225.75, "completions/mean_terminated_length": 222.78571472167968, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.0759357497096063, "epoch": 1.0600349361037051, "frac_reward_zero_std": 0.3, "grad_norm": 5.1660356521606445, "kl": 0.01147253904491663, "learning_rate": 3.844415172135574e-06, "loss": 0.0005, "num_tokens": 50254992.0, "reward": -0.28125, "reward_std": 0.623224550485611, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.6232245802879334, "sampling/importance_sampling_ratio/max": 1.2331055283546448, "sampling/importance_sampling_ratio/mean": 0.9995919644832612, "sampling/importance_sampling_ratio/min": 0.7166139394044876, "sampling/sampling_logp_difference/max": 0.3435733914375305, "sampling/sampling_logp_difference/mean": 0.020914221182465553, "step": 11530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.4, "completions/max_terminated_length": 269.4, "completions/mean_length": 208.8375, "completions/mean_terminated_length": 208.8375, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.1498083770275116, "epoch": 1.0609543072538383, "frac_reward_zero_std": 0.6, "grad_norm": 6.036616325378418, "kl": 0.01218783960212022, "learning_rate": 3.842278398491351e-06, "loss": 0.0006, "num_tokens": 50300179.0, "reward": -0.1, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.270482850074768, "sampling/importance_sampling_ratio/mean": 0.9996578216552734, "sampling/importance_sampling_ratio/min": 0.7579574704170227, "sampling/sampling_logp_difference/max": 0.30354454517364504, "sampling/sampling_logp_difference/mean": 0.02241687700152397, "step": 11540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.0, "completions/max_terminated_length": 320.0, "completions/mean_length": 221.6875, "completions/mean_terminated_length": 221.6875, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.1066163629293442, "epoch": 1.0618736784039717, "frac_reward_zero_std": 0.3, "grad_norm": 0.1074402779340744, "kl": 0.011923844873672351, "learning_rate": 3.840140246219617e-06, "loss": 0.0004, "num_tokens": 50343058.0, "reward": -0.125, "reward_std": 0.5967825591564179, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5967825591564179, "sampling/importance_sampling_ratio/max": 1.269965696334839, "sampling/importance_sampling_ratio/mean": 0.9996135354042053, "sampling/importance_sampling_ratio/min": 0.7645990490913391, "sampling/sampling_logp_difference/max": 0.2851519286632538, "sampling/sampling_logp_difference/mean": 0.021835525520145893, "step": 11550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.1, "completions/max_terminated_length": 303.1, "completions/mean_length": 222.7375, "completions/mean_terminated_length": 222.7375, "completions/min_length": 153.1, "completions/min_terminated_length": 153.1, "entropy": 1.0481165856122971, "epoch": 1.062793049554105, "frac_reward_zero_std": 0.2, "grad_norm": 4.854297637939453, "kl": 0.012744397018104791, "learning_rate": 3.8380007175164244e-06, "loss": 0.0007, "num_tokens": 50384541.0, "reward": -0.4, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.2910713076591491, "sampling/importance_sampling_ratio/mean": 1.0003466963768006, "sampling/importance_sampling_ratio/min": 0.798139613866806, "sampling/sampling_logp_difference/max": 0.2659681081771851, "sampling/sampling_logp_difference/mean": 0.021402235701680185, "step": 11560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 190.3, "completions/mean_terminated_length": 190.3, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.047072496265173, "epoch": 1.0637124207042383, "frac_reward_zero_std": 0.6, "grad_norm": 0.11767557263374329, "kl": 0.012312555027892812, "learning_rate": 3.835859814579241e-06, "loss": 0.0006, "num_tokens": 50428333.0, "reward": 0.0, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2481474161148072, "sampling/importance_sampling_ratio/mean": 0.9998003304004669, "sampling/importance_sampling_ratio/min": 0.7799118399620056, "sampling/sampling_logp_difference/max": 0.2563986420631409, "sampling/sampling_logp_difference/mean": 0.021669193170964718, "step": 11570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/max_terminated_length": 275.3, "completions/mean_length": 197.1625, "completions/mean_terminated_length": 197.1625, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0684965535998345, "epoch": 1.0646317918543715, "frac_reward_zero_std": 0.3, "grad_norm": 0.17978397011756897, "kl": 0.011721038905670866, "learning_rate": 3.833717539606946e-06, "loss": 0.0006, "num_tokens": 50471402.0, "reward": 0.11875, "reward_std": 0.6287827134132385, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.628782719373703, "sampling/importance_sampling_ratio/max": 1.297558641433716, "sampling/importance_sampling_ratio/mean": 1.000475388765335, "sampling/importance_sampling_ratio/min": 0.7705258071422577, "sampling/sampling_logp_difference/max": 0.3086832284927368, "sampling/sampling_logp_difference/mean": 0.022593646310269833, "step": 11580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 192.3375, "completions/mean_terminated_length": 192.3375, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.1307223491370677, "epoch": 1.065551163004505, "frac_reward_zero_std": 0.5, "grad_norm": 6.1593098640441895, "kl": 0.015281668049283326, "learning_rate": 3.831573894799826e-06, "loss": 0.0009, "num_tokens": 50515869.0, "reward": -0.175, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.3376430153846741, "sampling/importance_sampling_ratio/mean": 1.0001913487911225, "sampling/importance_sampling_ratio/min": 0.7857745826244354, "sampling/sampling_logp_difference/max": 0.30853155851364134, "sampling/sampling_logp_difference/mean": 0.02265640292316675, "step": 11590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.7, "completions/max_terminated_length": 326.7, "completions/mean_length": 222.6875, "completions/mean_terminated_length": 222.6875, "completions/min_length": 149.5, "completions/min_terminated_length": 149.5, "entropy": 1.0194007858633996, "epoch": 1.0664705341546383, "frac_reward_zero_std": 0.5, "grad_norm": 0.12159879505634308, "kl": 0.012336983531713485, "learning_rate": 3.829428882359579e-06, "loss": 0.0002, "num_tokens": 50558756.0, "reward": -0.075, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.3114680647850037, "sampling/importance_sampling_ratio/mean": 1.0006254076957704, "sampling/importance_sampling_ratio/min": 0.7624415516853332, "sampling/sampling_logp_difference/max": 0.308012866973877, "sampling/sampling_logp_difference/mean": 0.020576577819883823, "step": 11600 }, { "epoch": 1.0664705341546383, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 298.6369982547993, "eval_completions/max_terminated_length": 295.21465968586386, "eval_completions/mean_length": 208.9618237347295, "eval_completions/mean_terminated_length": 208.22416503267138, "eval_completions/min_length": 133.8656195462478, "eval_completions/min_terminated_length": 133.8656195462478, "eval_entropy": 1.0841063062349956, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.012346180599592431, "eval_loss": 0.000606193789280951, "eval_num_tokens": 50558756.0, "eval_reward": -0.11278359511343805, "eval_reward_std": 0.4555218500876718, "eval_rewards/reward_fn/mean": -0.11278359511343805, "eval_rewards/reward_fn/std": 0.4555218639226068, "eval_runtime": 1735.3878, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.276397603552587, "eval_sampling/importance_sampling_ratio/mean": 0.9999570253631831, "eval_sampling/importance_sampling_ratio/min": 0.7668701161160727, "eval_sampling/sampling_logp_difference/max": 0.2950984150379324, "eval_sampling/sampling_logp_difference/mean": 0.02146574270214242, "eval_steps_per_second": 0.041, "step": 11600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 292.3, "completions/max_terminated_length": 275.8, "completions/mean_length": 208.675, "completions/mean_terminated_length": 205.27857208251953, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.076709371805191, "epoch": 1.0673899053047715, "frac_reward_zero_std": 0.5, "grad_norm": 0.145280122756958, "kl": 0.011121398233808578, "learning_rate": 3.827282504489303e-06, "loss": 0.0004, "num_tokens": 50601746.0, "reward": -0.05625, "reward_std": 0.33331953138113024, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.33331954330205915, "sampling/importance_sampling_ratio/max": 1.24095675945282, "sampling/importance_sampling_ratio/mean": 0.9996973097324371, "sampling/importance_sampling_ratio/min": 0.7847473442554473, "sampling/sampling_logp_difference/max": 0.2507594883441925, "sampling/sampling_logp_difference/mean": 0.021719700656831264, "step": 11610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.4, "completions/max_terminated_length": 291.4, "completions/mean_length": 205.75, "completions/mean_terminated_length": 205.75, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.0744511514902115, "epoch": 1.068309276454905, "frac_reward_zero_std": 0.8, "grad_norm": 0.16982753574848175, "kl": 0.011246615316485986, "learning_rate": 3.8251347633934985e-06, "loss": 0.0005, "num_tokens": 50644366.0, "reward": 0.05, "reward_std": 0.1414213538169861, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.1414213538169861, "sampling/importance_sampling_ratio/max": 1.3032821059226989, "sampling/importance_sampling_ratio/mean": 1.0005024909973144, "sampling/importance_sampling_ratio/min": 0.7718646109104157, "sampling/sampling_logp_difference/max": 0.28256821632385254, "sampling/sampling_logp_difference/mean": 0.021300866827368738, "step": 11620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 215.9, "completions/mean_terminated_length": 215.9, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.0264776892960072, "epoch": 1.069228647605038, "frac_reward_zero_std": 0.4, "grad_norm": 6.870695114135742, "kl": 0.012008451425936073, "learning_rate": 3.82298566127807e-06, "loss": 0.0003, "num_tokens": 50690278.0, "reward": 0.2, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.276243543624878, "sampling/importance_sampling_ratio/mean": 1.0000853359699249, "sampling/importance_sampling_ratio/min": 0.7775788605213165, "sampling/sampling_logp_difference/max": 0.27798149585723875, "sampling/sampling_logp_difference/mean": 0.020903737097978593, "step": 11630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.4, "completions/max_terminated_length": 321.4, "completions/mean_length": 223.0, "completions/mean_terminated_length": 223.0, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.1053647346794606, "epoch": 1.0701480187551715, "frac_reward_zero_std": 0.4, "grad_norm": 0.13875523209571838, "kl": 0.020103890367317946, "learning_rate": 3.820835200350318e-06, "loss": 0.0009, "num_tokens": 50736742.0, "reward": -0.275, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2931953191757202, "sampling/importance_sampling_ratio/mean": 1.000016552209854, "sampling/importance_sampling_ratio/min": 0.7708314299583435, "sampling/sampling_logp_difference/max": 0.2929481565952301, "sampling/sampling_logp_difference/mean": 0.021595017239451408, "step": 11640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.9, "completions/max_terminated_length": 286.9, "completions/mean_length": 200.075, "completions/mean_terminated_length": 200.075, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.1392993167042733, "epoch": 1.0710673899053047, "frac_reward_zero_std": 0.4, "grad_norm": 0.2483462691307068, "kl": 0.012240531184943393, "learning_rate": 3.8186833828189365e-06, "loss": 0.0006, "num_tokens": 50778684.0, "reward": 0.0, "reward_std": 0.5554606795310975, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5554607272148132, "sampling/importance_sampling_ratio/max": 1.2480322480201722, "sampling/importance_sampling_ratio/mean": 1.000108355283737, "sampling/importance_sampling_ratio/min": 0.7689512848854065, "sampling/sampling_logp_difference/max": 0.2899738371372223, "sampling/sampling_logp_difference/mean": 0.023080662451684476, "step": 11650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 197.7125, "completions/mean_terminated_length": 197.7125, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.036741978302598, "epoch": 1.071986761055438, "frac_reward_zero_std": 0.3, "grad_norm": 4.739720344543457, "kl": 0.013061958586331458, "learning_rate": 3.816530210894016e-06, "loss": 0.0009, "num_tokens": 50819677.0, "reward": -0.025, "reward_std": 0.6480583846569061, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6480584084987641, "sampling/importance_sampling_ratio/max": 1.3182566285133361, "sampling/importance_sampling_ratio/mean": 1.0004715502262116, "sampling/importance_sampling_ratio/min": 0.7846891164779664, "sampling/sampling_logp_difference/max": 0.29631446599960326, "sampling/sampling_logp_difference/mean": 0.020509306993335485, "step": 11660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 199.7125, "completions/mean_terminated_length": 199.7125, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.0478048533201219, "epoch": 1.0729061322055713, "frac_reward_zero_std": 0.4, "grad_norm": 5.1947760581970215, "kl": 0.011694100528256968, "learning_rate": 3.8143756867870363e-06, "loss": 0.0006, "num_tokens": 50864502.0, "reward": -0.125, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.3269116282463074, "sampling/importance_sampling_ratio/mean": 1.0000395774841309, "sampling/importance_sampling_ratio/min": 0.7712133288383484, "sampling/sampling_logp_difference/max": 0.3029501438140869, "sampling/sampling_logp_difference/mean": 0.021460026130080224, "step": 11670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.6, "completions/max_terminated_length": 295.6, "completions/mean_length": 219.3875, "completions/mean_terminated_length": 219.3875, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.1542843587696552, "epoch": 1.0738255033557047, "frac_reward_zero_std": 0.4, "grad_norm": 5.206074237823486, "kl": 0.013399100332753733, "learning_rate": 3.8122198127108656e-06, "loss": 0.0009, "num_tokens": 50911589.0, "reward": 0.325, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.273347759246826, "sampling/importance_sampling_ratio/mean": 0.9999823868274689, "sampling/importance_sampling_ratio/min": 0.7585310697555542, "sampling/sampling_logp_difference/max": 0.29173781871795657, "sampling/sampling_logp_difference/mean": 0.022161070443689822, "step": 11680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 336.6, "completions/max_terminated_length": 322.8, "completions/mean_length": 233.775, "completions/mean_terminated_length": 231.05178833007812, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 1.1012139365077018, "epoch": 1.074744874505838, "frac_reward_zero_std": 0.2, "grad_norm": 5.178716659545898, "kl": 0.012718527368269861, "learning_rate": 3.810062590879761e-06, "loss": 0.0008, "num_tokens": 50957675.0, "reward": -0.18125, "reward_std": 0.701484078168869, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.7014841079711914, "sampling/importance_sampling_ratio/max": 1.244872784614563, "sampling/importance_sampling_ratio/mean": 0.9992318332195282, "sampling/importance_sampling_ratio/min": 0.7512075960636139, "sampling/sampling_logp_difference/max": 0.291546157002449, "sampling/sampling_logp_difference/mean": 0.021836154349148273, "step": 11690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 200.3375, "completions/mean_terminated_length": 200.3375, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.1313504185527563, "epoch": 1.0756642456559713, "frac_reward_zero_std": 0.4, "grad_norm": 6.047445774078369, "kl": 0.013459657458588481, "learning_rate": 3.807904023509359e-06, "loss": 0.0007, "num_tokens": 51001150.0, "reward": 0.25, "reward_std": 0.5882598340511322, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.588259893655777, "sampling/importance_sampling_ratio/max": 1.3522792220115663, "sampling/importance_sampling_ratio/mean": 1.0000236690044404, "sampling/importance_sampling_ratio/min": 0.7709965288639069, "sampling/sampling_logp_difference/max": 0.3304100275039673, "sampling/sampling_logp_difference/mean": 0.021902582515031098, "step": 11700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.1759981699287891, "epoch": 1.0765836168061047, "frac_reward_zero_std": 0.3, "grad_norm": 5.245700836181641, "kl": 0.013428171491250395, "learning_rate": 3.8057441128166848e-06, "loss": 0.0009, "num_tokens": 51047436.0, "reward": -0.325, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.2607858896255493, "sampling/importance_sampling_ratio/mean": 0.9999971270561219, "sampling/importance_sampling_ratio/min": 0.7798064649105072, "sampling/sampling_logp_difference/max": 0.27202500104904176, "sampling/sampling_logp_difference/mean": 0.022269155643880368, "step": 11710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.1, "completions/max_terminated_length": 326.1, "completions/mean_length": 232.05, "completions/mean_terminated_length": 232.05, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.09086379930377, "epoch": 1.077502987956238, "frac_reward_zero_std": 0.4, "grad_norm": 0.14156757295131683, "kl": 0.011756887816591188, "learning_rate": 3.8035828610201393e-06, "loss": 0.0008, "num_tokens": 51093432.0, "reward": 0.325, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.310868787765503, "sampling/importance_sampling_ratio/mean": 0.9998906195163727, "sampling/importance_sampling_ratio/min": 0.7850990355014801, "sampling/sampling_logp_difference/max": 0.28288434743881224, "sampling/sampling_logp_difference/mean": 0.020670896954834463, "step": 11720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.2, "completions/max_terminated_length": 272.2, "completions/mean_length": 203.375, "completions/mean_terminated_length": 203.375, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.121577075496316, "epoch": 1.0784223591063713, "frac_reward_zero_std": 0.1, "grad_norm": 5.78856897354126, "kl": 0.01363185719237663, "learning_rate": 3.8014202703395e-06, "loss": 0.0007, "num_tokens": 51137270.0, "reward": -0.425, "reward_std": 0.8038022220134735, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.8038022458553314, "sampling/importance_sampling_ratio/max": 1.2887015700340272, "sampling/importance_sampling_ratio/mean": 1.00061896443367, "sampling/importance_sampling_ratio/min": 0.7697213113307952, "sampling/sampling_logp_difference/max": 0.2922583520412445, "sampling/sampling_logp_difference/mean": 0.022272107750177385, "step": 11730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 217.425, "completions/mean_terminated_length": 217.425, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 1.0894629172980785, "epoch": 1.0793417302565045, "frac_reward_zero_std": 0.7, "grad_norm": 0.09397537261247635, "kl": 0.013225799071369693, "learning_rate": 3.7992563429959206e-06, "loss": 0.0008, "num_tokens": 51180712.0, "reward": -0.375, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.355242085456848, "sampling/importance_sampling_ratio/mean": 1.000502210855484, "sampling/importance_sampling_ratio/min": 0.7851688921451568, "sampling/sampling_logp_difference/max": 0.34725008010864256, "sampling/sampling_logp_difference/mean": 0.022087206691503526, "step": 11740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.4, "completions/max_terminated_length": 303.4, "completions/mean_length": 218.875, "completions/mean_terminated_length": 218.875, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 1.0197887785732747, "epoch": 1.080261101406638, "frac_reward_zero_std": 0.5, "grad_norm": 5.177273750305176, "kl": 0.011912795819807798, "learning_rate": 3.79709108121193e-06, "loss": 0.0006, "num_tokens": 51225246.0, "reward": -0.025, "reward_std": 0.5100160002708435, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5100160360336303, "sampling/importance_sampling_ratio/max": 1.2412066817283631, "sampling/importance_sampling_ratio/mean": 0.9999794483184814, "sampling/importance_sampling_ratio/min": 0.7671703815460205, "sampling/sampling_logp_difference/max": 0.27690961956977844, "sampling/sampling_logp_difference/mean": 0.02124053854495287, "step": 11750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 210.2125, "completions/mean_terminated_length": 210.2125, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0312680225819348, "epoch": 1.081180472556771, "frac_reward_zero_std": 0.5, "grad_norm": 6.101497650146484, "kl": 0.013216507848119364, "learning_rate": 3.794924487211423e-06, "loss": 0.001, "num_tokens": 51267511.0, "reward": -0.3, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.3101846575737, "sampling/importance_sampling_ratio/mean": 0.9996418416500091, "sampling/importance_sampling_ratio/min": 0.7581876695156098, "sampling/sampling_logp_difference/max": 0.3070618391036987, "sampling/sampling_logp_difference/mean": 0.021854897029697895, "step": 11760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 208.7, "completions/mean_terminated_length": 208.7, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.074728637933731, "epoch": 1.0820998437069045, "frac_reward_zero_std": 0.5, "grad_norm": 5.745357036590576, "kl": 0.012779380229767412, "learning_rate": 3.7927565632196683e-06, "loss": 0.0007, "num_tokens": 51308495.0, "reward": -0.075, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.2562051296234131, "sampling/importance_sampling_ratio/mean": 0.9999316513538361, "sampling/importance_sampling_ratio/min": 0.7638644456863404, "sampling/sampling_logp_difference/max": 0.2928103804588318, "sampling/sampling_logp_difference/mean": 0.021982464008033276, "step": 11770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 313.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 208.575, "completions/mean_terminated_length": 205.6089294433594, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.0631131019443274, "epoch": 1.0830192148570377, "frac_reward_zero_std": 0.4, "grad_norm": 0.19634611904621124, "kl": 0.013309776119422167, "learning_rate": 3.7905873114632953e-06, "loss": 0.0005, "num_tokens": 51350781.0, "reward": -0.23125, "reward_std": 0.5619584262371063, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5619584381580353, "sampling/importance_sampling_ratio/max": 1.3079081177711487, "sampling/importance_sampling_ratio/mean": 0.9997788667678833, "sampling/importance_sampling_ratio/min": 0.7598217844963073, "sampling/sampling_logp_difference/max": 0.3128166079521179, "sampling/sampling_logp_difference/mean": 0.021717039868235588, "step": 11780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 206.2875, "completions/mean_terminated_length": 206.2875, "completions/min_length": 134.1, "completions/min_terminated_length": 134.1, "entropy": 0.9411828134208917, "epoch": 1.083938586007171, "frac_reward_zero_std": 0.3, "grad_norm": 3.81012225151062, "kl": 0.010456260776845739, "learning_rate": 3.7884167341703006e-06, "loss": 0.0005, "num_tokens": 51392548.0, "reward": 0.05, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.2617175459861756, "sampling/importance_sampling_ratio/mean": 0.9997236609458924, "sampling/importance_sampling_ratio/min": 0.775988495349884, "sampling/sampling_logp_difference/max": 0.26111459732055664, "sampling/sampling_logp_difference/mean": 0.019882215652614832, "step": 11790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 202.0875, "completions/mean_terminated_length": 202.0875, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.0978728011250496, "epoch": 1.0848579571573045, "frac_reward_zero_std": 0.5, "grad_norm": 5.666393756866455, "kl": 0.011912991703138687, "learning_rate": 3.7862448335700406e-06, "loss": 0.0008, "num_tokens": 51435787.0, "reward": 0.25, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.2756593585014344, "sampling/importance_sampling_ratio/mean": 1.000272160768509, "sampling/importance_sampling_ratio/min": 0.7346574187278747, "sampling/sampling_logp_difference/max": 0.31960815787315366, "sampling/sampling_logp_difference/mean": 0.02272746227681637, "step": 11800 }, { "epoch": 1.0848579571573045, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 302.30366492146595, "eval_completions/max_terminated_length": 299.7137870855148, "eval_completions/mean_length": 207.5185427574171, "eval_completions/mean_terminated_length": 206.84734702651204, "eval_completions/min_length": 132.71204188481676, "eval_completions/min_terminated_length": 132.71204188481676, "eval_entropy": 1.0914559840219806, "eval_frac_reward_zero_std": 0.4694589877835951, "eval_kl": 0.012720382769948057, "eval_loss": 0.0006381540442816913, "eval_num_tokens": 51435787.0, "eval_reward": -0.10711169284467714, "eval_reward_std": 0.4567602691330003, "eval_rewards/reward_fn/mean": -0.10711169284467714, "eval_rewards/reward_fn/std": 0.45676028119956014, "eval_runtime": 1756.1756, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.2734522746703596, "eval_sampling/importance_sampling_ratio/mean": 0.9999942057627866, "eval_sampling/importance_sampling_ratio/min": 0.7648420043640736, "eval_sampling/sampling_logp_difference/max": 0.2987393721235984, "eval_sampling/sampling_logp_difference/mean": 0.021653764617887778, "eval_steps_per_second": 0.041, "step": 11800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 191.55, "completions/mean_terminated_length": 191.55, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0997645482420921, "epoch": 1.0857773283074377, "frac_reward_zero_std": 0.2, "grad_norm": 0.08522821217775345, "kl": 0.013874411938013509, "learning_rate": 3.784071611893231e-06, "loss": 0.0003, "num_tokens": 51477295.0, "reward": 0.275, "reward_std": 0.7296968817710876, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.7296969175338746, "sampling/importance_sampling_ratio/max": 1.326323103904724, "sampling/importance_sampling_ratio/mean": 1.000279438495636, "sampling/importance_sampling_ratio/min": 0.7854861736297607, "sampling/sampling_logp_difference/max": 0.29127162098884585, "sampling/sampling_logp_difference/mean": 0.022202770784497262, "step": 11810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.6, "completions/max_terminated_length": 321.6, "completions/mean_length": 216.2125, "completions/mean_terminated_length": 216.2125, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.0776939615607262, "epoch": 1.086696699457571, "frac_reward_zero_std": 0.2, "grad_norm": 6.125853061676025, "kl": 0.01164720052620396, "learning_rate": 3.781897071371945e-06, "loss": 0.0003, "num_tokens": 51523976.0, "reward": -0.175, "reward_std": 0.7078412413597107, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.7078412532806396, "sampling/importance_sampling_ratio/max": 1.2621911764144897, "sampling/importance_sampling_ratio/mean": 0.9994366645812989, "sampling/importance_sampling_ratio/min": 0.7469263970851898, "sampling/sampling_logp_difference/max": 0.29345571994781494, "sampling/sampling_logp_difference/mean": 0.021161204576492308, "step": 11820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.1, "completions/max_terminated_length": 329.1, "completions/mean_length": 235.7, "completions/mean_terminated_length": 235.7, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0861261457204818, "epoch": 1.0876160706077043, "frac_reward_zero_std": 0.6, "grad_norm": 0.13349515199661255, "kl": 0.014273710531415417, "learning_rate": 3.77972121423961e-06, "loss": 0.0007, "num_tokens": 51571064.0, "reward": -0.225, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.2626847624778748, "sampling/importance_sampling_ratio/mean": 0.9995636641979218, "sampling/importance_sampling_ratio/min": 0.7385914534330368, "sampling/sampling_logp_difference/max": 0.34769346117973327, "sampling/sampling_logp_difference/mean": 0.02118592578917742, "step": 11830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.8, "completions/max_terminated_length": 268.8, "completions/mean_length": 187.0875, "completions/mean_terminated_length": 187.0875, "completions/min_length": 118.7, "completions/min_terminated_length": 118.7, "entropy": 1.0517103064805269, "epoch": 1.0885354417578377, "frac_reward_zero_std": 0.8, "grad_norm": 0.1297086775302887, "kl": 0.012536121053562966, "learning_rate": 3.777544042731006e-06, "loss": 0.0006, "num_tokens": 51610471.0, "reward": -0.175, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.3384899020195007, "sampling/importance_sampling_ratio/mean": 1.0003536939620972, "sampling/importance_sampling_ratio/min": 0.7738674700260162, "sampling/sampling_logp_difference/max": 0.32102856040000916, "sampling/sampling_logp_difference/mean": 0.021225943882018327, "step": 11840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 196.675, "completions/mean_terminated_length": 196.675, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.0676908604800701, "epoch": 1.0894548129079709, "frac_reward_zero_std": 0.8, "grad_norm": 0.09373633563518524, "kl": 0.01255338720511645, "learning_rate": 3.7753655590822626e-06, "loss": 0.0005, "num_tokens": 51653045.0, "reward": -0.125, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.3082706809043885, "sampling/importance_sampling_ratio/mean": 1.000382936000824, "sampling/importance_sampling_ratio/min": 0.8047965109348297, "sampling/sampling_logp_difference/max": 0.2742955684661865, "sampling/sampling_logp_difference/mean": 0.021045747213065624, "step": 11850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.6, "completions/max_terminated_length": 279.6, "completions/mean_length": 196.6625, "completions/mean_terminated_length": 196.6625, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.103640115261078, "epoch": 1.0903741840581043, "frac_reward_zero_std": 0.6, "grad_norm": 0.10349728912115097, "kl": 0.012792379490565508, "learning_rate": 3.773185765530857e-06, "loss": 0.0004, "num_tokens": 51698138.0, "reward": -0.15, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.293302834033966, "sampling/importance_sampling_ratio/mean": 0.9999572455883026, "sampling/importance_sampling_ratio/min": 0.7724321901798248, "sampling/sampling_logp_difference/max": 0.28988142013549806, "sampling/sampling_logp_difference/mean": 0.022127051278948782, "step": 11860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 214.55, "completions/mean_terminated_length": 214.55, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0762911647558213, "epoch": 1.0912935552082375, "frac_reward_zero_std": 0.4, "grad_norm": 0.09592319279909134, "kl": 0.012576487334445118, "learning_rate": 3.7710046643156116e-06, "loss": 0.0005, "num_tokens": 51743558.0, "reward": 0.0, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.3961949467658996, "sampling/importance_sampling_ratio/mean": 0.9999123871326446, "sampling/importance_sampling_ratio/min": 0.7492159187793732, "sampling/sampling_logp_difference/max": 0.44339885115623473, "sampling/sampling_logp_difference/mean": 0.0216865299269557, "step": 11870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.7, "completions/max_terminated_length": 319.7, "completions/mean_length": 220.725, "completions/mean_terminated_length": 220.725, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0884543120861054, "epoch": 1.0922129263583709, "frac_reward_zero_std": 0.4, "grad_norm": 4.983007431030273, "kl": 0.013891686918213964, "learning_rate": 3.768822257676693e-06, "loss": 0.0009, "num_tokens": 51785256.0, "reward": -0.075, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.3134590148925782, "sampling/importance_sampling_ratio/mean": 1.0000295698642732, "sampling/importance_sampling_ratio/min": 0.7941475987434388, "sampling/sampling_logp_difference/max": 0.289564847946167, "sampling/sampling_logp_difference/mean": 0.02144571449607611, "step": 11880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.1, "completions/max_terminated_length": 313.1, "completions/mean_length": 223.45, "completions/mean_terminated_length": 223.45, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0320164576172828, "epoch": 1.0931322975085043, "frac_reward_zero_std": 0.4, "grad_norm": 0.1221429780125618, "kl": 0.012643771158764139, "learning_rate": 3.766638547855607e-06, "loss": 0.0005, "num_tokens": 51829612.0, "reward": -0.075, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.3030574440956115, "sampling/importance_sampling_ratio/mean": 0.9999997973442077, "sampling/importance_sampling_ratio/min": 0.7684139907360077, "sampling/sampling_logp_difference/max": 0.30124289989471437, "sampling/sampling_logp_difference/mean": 0.020610379055142403, "step": 11890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 226.0, "completions/mean_terminated_length": 226.0, "completions/min_length": 151.8, "completions/min_terminated_length": 151.8, "entropy": 1.1088496267795562, "epoch": 1.0940516686586375, "frac_reward_zero_std": 0.3, "grad_norm": 6.1014790534973145, "kl": 0.012183726805960759, "learning_rate": 3.764453537095198e-06, "loss": 0.0001, "num_tokens": 51876604.0, "reward": -0.075, "reward_std": 0.6514373540878295, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6514373898506165, "sampling/importance_sampling_ratio/max": 1.2925419688224793, "sampling/importance_sampling_ratio/mean": 1.0004818618297577, "sampling/importance_sampling_ratio/min": 0.7804545819759369, "sampling/sampling_logp_difference/max": 0.2832693815231323, "sampling/sampling_logp_difference/mean": 0.021248507499694824, "step": 11900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.0, "completions/max_terminated_length": 322.0, "completions/mean_length": 207.7, "completions/mean_terminated_length": 207.7, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0591843016445637, "epoch": 1.0949710398087709, "frac_reward_zero_std": 0.6, "grad_norm": 0.12105593085289001, "kl": 0.013825469592120498, "learning_rate": 3.762267227639649e-06, "loss": 0.0004, "num_tokens": 51918324.0, "reward": -0.35625, "reward_std": 0.32022821307182314, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.32022823095321656, "sampling/importance_sampling_ratio/max": 1.31835697889328, "sampling/importance_sampling_ratio/mean": 1.0000235199928285, "sampling/importance_sampling_ratio/min": 0.7712680339813233, "sampling/sampling_logp_difference/max": 0.3048670530319214, "sampling/sampling_logp_difference/mean": 0.021994415484368802, "step": 11910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 213.175, "completions/mean_terminated_length": 213.175, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.098762384057045, "epoch": 1.095890410958904, "frac_reward_zero_std": 0.5, "grad_norm": 0.1259351223707199, "kl": 0.01290148679399863, "learning_rate": 3.760079621734473e-06, "loss": 0.0004, "num_tokens": 51963778.0, "reward": -0.15, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.3907026410102845, "sampling/importance_sampling_ratio/mean": 0.9996917366981506, "sampling/importance_sampling_ratio/min": 0.7762544989585877, "sampling/sampling_logp_difference/max": 0.35651962757110595, "sampling/sampling_logp_difference/mean": 0.0217586949467659, "step": 11920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 364.1, "completions/max_terminated_length": 364.1, "completions/mean_length": 220.475, "completions/mean_terminated_length": 220.475, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.0525891669094563, "epoch": 1.0968097821090375, "frac_reward_zero_std": 0.3, "grad_norm": 6.4990057945251465, "kl": 0.011686168843880296, "learning_rate": 3.757890721626519e-06, "loss": 0.0005, "num_tokens": 52009040.0, "reward": 0.05, "reward_std": 0.6220328629016876, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6220328748226166, "sampling/importance_sampling_ratio/max": 1.2597884774208068, "sampling/importance_sampling_ratio/mean": 0.9997317373752594, "sampling/importance_sampling_ratio/min": 0.7856112658977509, "sampling/sampling_logp_difference/max": 0.2554481506347656, "sampling/sampling_logp_difference/mean": 0.020678455010056496, "step": 11930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.9, "completions/max_terminated_length": 277.9, "completions/mean_length": 204.225, "completions/mean_terminated_length": 204.225, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0660004422068596, "epoch": 1.0977291532591706, "frac_reward_zero_std": 0.6, "grad_norm": 6.509206295013428, "kl": 0.011528434557840228, "learning_rate": 3.7557005295639613e-06, "loss": 0.0002, "num_tokens": 52050226.0, "reward": -0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.3193410992622376, "sampling/importance_sampling_ratio/mean": 1.0000329673290254, "sampling/importance_sampling_ratio/min": 0.7878987848758697, "sampling/sampling_logp_difference/max": 0.29163293838500975, "sampling/sampling_logp_difference/mean": 0.021086104027926923, "step": 11940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 306.4, "completions/max_terminated_length": 290.9, "completions/mean_length": 222.5875, "completions/mean_terminated_length": 219.45714416503907, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.1115539219230413, "epoch": 1.098648524409304, "frac_reward_zero_std": 0.4, "grad_norm": 5.809169769287109, "kl": 0.01234441773267463, "learning_rate": 3.7535090477963025e-06, "loss": 0.0009, "num_tokens": 52094681.0, "reward": -0.08125, "reward_std": 0.5437338590621948, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5437338888645172, "sampling/importance_sampling_ratio/max": 1.2410333514213563, "sampling/importance_sampling_ratio/mean": 0.9997781634330749, "sampling/importance_sampling_ratio/min": 0.7733692049980163, "sampling/sampling_logp_difference/max": 0.25982133150100706, "sampling/sampling_logp_difference/mean": 0.021699629724025726, "step": 11950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.4, "completions/max_terminated_length": 316.4, "completions/mean_length": 211.9, "completions/mean_terminated_length": 211.9, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.066618439555168, "epoch": 1.0995678955594372, "frac_reward_zero_std": 0.6, "grad_norm": 0.11276761442422867, "kl": 0.011120591461076402, "learning_rate": 3.7513162785743713e-06, "loss": 0.0005, "num_tokens": 52136345.0, "reward": -0.05, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.308535599708557, "sampling/importance_sampling_ratio/mean": 0.9998794436454773, "sampling/importance_sampling_ratio/min": 0.7581147193908692, "sampling/sampling_logp_difference/max": 0.33313372135162356, "sampling/sampling_logp_difference/mean": 0.020861812494695188, "step": 11960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.6, "completions/max_terminated_length": 310.6, "completions/mean_length": 222.6875, "completions/mean_terminated_length": 222.6875, "completions/min_length": 153.2, "completions/min_terminated_length": 153.2, "entropy": 1.141600826382637, "epoch": 1.1004872667095706, "frac_reward_zero_std": 0.2, "grad_norm": 5.386699199676514, "kl": 0.012541209283517674, "learning_rate": 3.749122224150316e-06, "loss": 0.0008, "num_tokens": 52183064.0, "reward": 0.075, "reward_std": 0.6968977272510528, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.6968977510929107, "sampling/importance_sampling_ratio/max": 1.318000364303589, "sampling/importance_sampling_ratio/mean": 0.9998856484889984, "sampling/importance_sampling_ratio/min": 0.7892673730850219, "sampling/sampling_logp_difference/max": 0.2974181294441223, "sampling/sampling_logp_difference/mean": 0.02225194051861763, "step": 11970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 209.8, "completions/mean_terminated_length": 209.8, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0078404664993286, "epoch": 1.101406637859704, "frac_reward_zero_std": 0.5, "grad_norm": 0.08608261495828629, "kl": 0.012700253544608131, "learning_rate": 3.7469268867776076e-06, "loss": 0.0006, "num_tokens": 52227976.0, "reward": 0.1, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2155197262763977, "sampling/importance_sampling_ratio/mean": 0.9998282194137573, "sampling/importance_sampling_ratio/min": 0.7712497651576996, "sampling/sampling_logp_difference/max": 0.2652600646018982, "sampling/sampling_logp_difference/mean": 0.021637541800737382, "step": 11980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.6, "completions/max_terminated_length": 295.6, "completions/mean_length": 208.525, "completions/mean_terminated_length": 208.525, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.0428888522088529, "epoch": 1.1023260090098372, "frac_reward_zero_std": 0.3, "grad_norm": 5.340268611907959, "kl": 0.012862533424049616, "learning_rate": 3.7447302687110334e-06, "loss": 0.0007, "num_tokens": 52270082.0, "reward": -0.05, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.251796507835388, "sampling/importance_sampling_ratio/mean": 0.9998016595840454, "sampling/importance_sampling_ratio/min": 0.756479012966156, "sampling/sampling_logp_difference/max": 0.29776694178581237, "sampling/sampling_logp_difference/mean": 0.021404031850397586, "step": 11990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.3, "completions/max_terminated_length": 256.3, "completions/mean_length": 190.1625, "completions/mean_terminated_length": 190.1625, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0424237847328186, "epoch": 1.1032453801599706, "frac_reward_zero_std": 0.7, "grad_norm": 0.10349927097558975, "kl": 0.010948076145723462, "learning_rate": 3.7425323722066945e-06, "loss": 0.0009, "num_tokens": 52311975.0, "reward": 0.525, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.525, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.220604705810547, "sampling/importance_sampling_ratio/mean": 0.9997669517993927, "sampling/importance_sampling_ratio/min": 0.7676752269268036, "sampling/sampling_logp_difference/max": 0.26500836610794065, "sampling/sampling_logp_difference/mean": 0.021062465757131575, "step": 12000 }, { "epoch": 1.1032453801599706, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 296.60907504363, "eval_completions/max_terminated_length": 295.4589877835951, "eval_completions/mean_length": 208.29864746945898, "eval_completions/mean_terminated_length": 207.93061801983632, "eval_completions/min_length": 135.8324607329843, "eval_completions/min_terminated_length": 135.8324607329843, "eval_entropy": 1.066349299114204, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.01208790734306264, "eval_loss": 0.0005837922799400985, "eval_num_tokens": 52311975.0, "eval_reward": -0.09511343804537521, "eval_reward_std": 0.4529418665753193, "eval_rewards/reward_fn/mean": -0.09511343804537521, "eval_rewards/reward_fn/std": 0.4529418810343867, "eval_runtime": 1727.3633, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.2700483809382919, "eval_sampling/importance_sampling_ratio/mean": 1.0000248741729096, "eval_sampling/importance_sampling_ratio/min": 0.7638621101152626, "eval_sampling/sampling_logp_difference/max": 0.2959837594165436, "eval_sampling/sampling_logp_difference/mean": 0.02132432518712201, "eval_steps_per_second": 0.042, "step": 12000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.1, "completions/max_terminated_length": 312.1, "completions/mean_length": 220.775, "completions/mean_terminated_length": 220.775, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.0697701558470727, "epoch": 1.1041647513101038, "frac_reward_zero_std": 0.3, "grad_norm": 4.874321460723877, "kl": 0.011053756420733408, "learning_rate": 3.7403331995220093e-06, "loss": -0.0, "num_tokens": 52356717.0, "reward": -0.375, "reward_std": 0.6329606831073761, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.632960706949234, "sampling/importance_sampling_ratio/max": 1.2624102711677552, "sampling/importance_sampling_ratio/mean": 1.00037522315979, "sampling/importance_sampling_ratio/min": 0.7806853294372559, "sampling/sampling_logp_difference/max": 0.25961199402809143, "sampling/sampling_logp_difference/mean": 0.02145510446280241, "step": 12010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 200.375, "completions/mean_terminated_length": 200.375, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.0320680551230907, "epoch": 1.1050841224602372, "frac_reward_zero_std": 0.5, "grad_norm": 5.575322151184082, "kl": 0.010187164606759325, "learning_rate": 3.7381327529157037e-06, "loss": 0.0006, "num_tokens": 52398099.0, "reward": -0.125, "reward_std": 0.4587401747703552, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4587401866912842, "sampling/importance_sampling_ratio/max": 1.2542080521583556, "sampling/importance_sampling_ratio/mean": 1.0000251531600952, "sampling/importance_sampling_ratio/min": 0.7259479820728302, "sampling/sampling_logp_difference/max": 0.34059094190597533, "sampling/sampling_logp_difference/mean": 0.021747568808495998, "step": 12020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 214.8, "completions/mean_terminated_length": 214.8, "completions/min_length": 152.7, "completions/min_terminated_length": 152.7, "entropy": 1.10516489520669, "epoch": 1.1060034936103704, "frac_reward_zero_std": 0.3, "grad_norm": 0.10179571062326431, "kl": 0.014242794597521424, "learning_rate": 3.7359310346478106e-06, "loss": 0.0004, "num_tokens": 52442299.0, "reward": 0.3, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2940588116645813, "sampling/importance_sampling_ratio/mean": 1.0003439366817475, "sampling/importance_sampling_ratio/min": 0.7361156910657882, "sampling/sampling_logp_difference/max": 0.3492574214935303, "sampling/sampling_logp_difference/mean": 0.02198427841067314, "step": 12030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.7, "completions/max_terminated_length": 308.7, "completions/mean_length": 209.25, "completions/mean_terminated_length": 209.25, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0409267634153365, "epoch": 1.1069228647605038, "frac_reward_zero_std": 0.6, "grad_norm": 5.386318206787109, "kl": 0.010697001498192549, "learning_rate": 3.733728046979675e-06, "loss": 0.0002, "num_tokens": 52483799.0, "reward": 0.25, "reward_std": 0.3846505284309387, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3846505284309387, "sampling/importance_sampling_ratio/max": 1.2607258200645446, "sampling/importance_sampling_ratio/mean": 1.0003026962280273, "sampling/importance_sampling_ratio/min": 0.8057772696018219, "sampling/sampling_logp_difference/max": 0.2526862144470215, "sampling/sampling_logp_difference/mean": 0.020737999863922597, "step": 12040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 208.925, "completions/mean_terminated_length": 208.925, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 0.9548827763646841, "epoch": 1.107842235910637, "frac_reward_zero_std": 0.3, "grad_norm": 4.624998092651367, "kl": 0.011330754656228237, "learning_rate": 3.7315237921739387e-06, "loss": 0.0006, "num_tokens": 52527473.0, "reward": 0.05, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2620401859283448, "sampling/importance_sampling_ratio/mean": 1.0005826830863953, "sampling/importance_sampling_ratio/min": 0.7723708152770996, "sampling/sampling_logp_difference/max": 0.2792785048484802, "sampling/sampling_logp_difference/mean": 0.020407964289188386, "step": 12050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 198.65, "completions/mean_terminated_length": 198.65, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0900205194950103, "epoch": 1.1087616070607704, "frac_reward_zero_std": 0.4, "grad_norm": 7.098525524139404, "kl": 0.012966111651621759, "learning_rate": 3.7293182724945493e-06, "loss": 0.0004, "num_tokens": 52571357.0, "reward": 0.025, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2399284720420838, "sampling/importance_sampling_ratio/mean": 1.0004975616931915, "sampling/importance_sampling_ratio/min": 0.7842021107673645, "sampling/sampling_logp_difference/max": 0.24746726751327514, "sampling/sampling_logp_difference/mean": 0.02272295542061329, "step": 12060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.9, "completions/max_terminated_length": 307.9, "completions/mean_length": 215.2125, "completions/mean_terminated_length": 215.2125, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0299555510282516, "epoch": 1.1096809782109038, "frac_reward_zero_std": 0.6, "grad_norm": 6.873339653015137, "kl": 0.011521656514378264, "learning_rate": 3.7271114902067517e-06, "loss": 0.0005, "num_tokens": 52616686.0, "reward": 0.1, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2477415800094604, "sampling/importance_sampling_ratio/mean": 1.0002832174301148, "sampling/importance_sampling_ratio/min": 0.7666098237037658, "sampling/sampling_logp_difference/max": 0.28679571151733396, "sampling/sampling_logp_difference/mean": 0.021515472047030925, "step": 12070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 217.425, "completions/mean_terminated_length": 217.425, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.1186378851532937, "epoch": 1.110600349361037, "frac_reward_zero_std": 0.5, "grad_norm": 4.687838077545166, "kl": 0.011616805969970301, "learning_rate": 3.724903447577089e-06, "loss": 0.0007, "num_tokens": 52664256.0, "reward": 0.025, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2769362211227417, "sampling/importance_sampling_ratio/mean": 1.0002563893795013, "sampling/importance_sampling_ratio/min": 0.7698371946811676, "sampling/sampling_logp_difference/max": 0.2867067813873291, "sampling/sampling_logp_difference/mean": 0.022433517314493657, "step": 12080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.2, "completions/max_terminated_length": 335.2, "completions/mean_length": 226.1375, "completions/mean_terminated_length": 226.1375, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1086138941347599, "epoch": 1.1115197205111704, "frac_reward_zero_std": 0.7, "grad_norm": 9.21669864654541, "kl": 0.01113628870807588, "learning_rate": 3.722694146873398e-06, "loss": 0.0008, "num_tokens": 52714363.0, "reward": -0.075, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.3319299697875977, "sampling/importance_sampling_ratio/mean": 1.0004727244377136, "sampling/importance_sampling_ratio/min": 0.7382274061441422, "sampling/sampling_logp_difference/max": 0.4186274170875549, "sampling/sampling_logp_difference/mean": 0.022668248787522317, "step": 12090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 218.4, "completions/mean_terminated_length": 218.4, "completions/min_length": 150.4, "completions/min_terminated_length": 150.4, "entropy": 1.0622360736131669, "epoch": 1.1124390916613036, "frac_reward_zero_std": 0.3, "grad_norm": 6.323678970336914, "kl": 0.012343563977628947, "learning_rate": 3.7204835903648067e-06, "loss": 0.0007, "num_tokens": 52761555.0, "reward": 0.175, "reward_std": 0.643904197216034, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6439042091369629, "sampling/importance_sampling_ratio/max": 1.3600587606430055, "sampling/importance_sampling_ratio/mean": 0.9998730182647705, "sampling/importance_sampling_ratio/min": 0.7725848019123077, "sampling/sampling_logp_difference/max": 0.3303189516067505, "sampling/sampling_logp_difference/mean": 0.021846701204776765, "step": 12100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 311.0, "completions/max_terminated_length": 294.5, "completions/mean_length": 226.025, "completions/mean_terminated_length": 222.74642944335938, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.1164533965289594, "epoch": 1.113358462811437, "frac_reward_zero_std": 0.2, "grad_norm": 0.10772456228733063, "kl": 0.011555361264618114, "learning_rate": 3.7182717803217346e-06, "loss": 0.0006, "num_tokens": 52809045.0, "reward": -0.43125, "reward_std": 0.733397263288498, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.7333972871303558, "sampling/importance_sampling_ratio/max": 1.2802724719047547, "sampling/importance_sampling_ratio/mean": 1.0004316926002503, "sampling/importance_sampling_ratio/min": 0.7816884815692902, "sampling/sampling_logp_difference/max": 0.27067041397094727, "sampling/sampling_logp_difference/mean": 0.022474785149097443, "step": 12110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 224.5125, "completions/mean_terminated_length": 224.5125, "completions/min_length": 153.3, "completions/min_terminated_length": 153.3, "entropy": 1.010707065463066, "epoch": 1.1142778339615702, "frac_reward_zero_std": 0.5, "grad_norm": 5.694643020629883, "kl": 0.012935551023110748, "learning_rate": 3.716058719015887e-06, "loss": 0.0005, "num_tokens": 52857350.0, "reward": 0.16875, "reward_std": 0.3879900202155113, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.38799004405736925, "sampling/importance_sampling_ratio/max": 1.2752809524536133, "sampling/importance_sampling_ratio/mean": 0.9999164521694184, "sampling/importance_sampling_ratio/min": 0.7900763511657715, "sampling/sampling_logp_difference/max": 0.2598080515861511, "sampling/sampling_logp_difference/mean": 0.020537781342864037, "step": 12120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 217.4625, "completions/mean_terminated_length": 217.4625, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.0676752768456936, "epoch": 1.1151972051117036, "frac_reward_zero_std": 0.4, "grad_norm": 0.09744422882795334, "kl": 0.012172073108376934, "learning_rate": 3.7138444087202576e-06, "loss": 0.0004, "num_tokens": 52900963.0, "reward": 0.225, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2574677348136902, "sampling/importance_sampling_ratio/mean": 0.9998047351837158, "sampling/importance_sampling_ratio/min": 0.7550015270709991, "sampling/sampling_logp_difference/max": 0.2907137632369995, "sampling/sampling_logp_difference/mean": 0.02196618840098381, "step": 12130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.3, "completions/max_terminated_length": 307.3, "completions/mean_length": 212.0375, "completions/mean_terminated_length": 212.0375, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0313968159258367, "epoch": 1.1161165762618368, "frac_reward_zero_std": 0.5, "grad_norm": 0.2258492261171341, "kl": 0.011169203574536369, "learning_rate": 3.711628851709117e-06, "loss": 0.0007, "num_tokens": 52942358.0, "reward": 0.175, "reward_std": 0.48475000262260437, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4847500503063202, "sampling/importance_sampling_ratio/max": 1.2501624584198, "sampling/importance_sampling_ratio/mean": 1.0001565635204315, "sampling/importance_sampling_ratio/min": 0.7726091623306275, "sampling/sampling_logp_difference/max": 0.28137069940567017, "sampling/sampling_logp_difference/mean": 0.020938819646835326, "step": 12140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.9, "completions/max_terminated_length": 314.9, "completions/mean_length": 236.775, "completions/mean_terminated_length": 236.775, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 1.1054537616670133, "epoch": 1.1170359474119702, "frac_reward_zero_std": 0.3, "grad_norm": 5.911098957061768, "kl": 0.011043734388658777, "learning_rate": 3.7094120502580226e-06, "loss": 0.0005, "num_tokens": 52990020.0, "reward": 0.1, "reward_std": 0.6589862048625946, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6589862406253815, "sampling/importance_sampling_ratio/max": 1.2665410399436952, "sampling/importance_sampling_ratio/mean": 1.0005334317684174, "sampling/importance_sampling_ratio/min": 0.7548340320587158, "sampling/sampling_logp_difference/max": 0.2993365049362183, "sampling/sampling_logp_difference/mean": 0.022264179214835166, "step": 12150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.1, "completions/max_terminated_length": 277.1, "completions/mean_length": 203.225, "completions/mean_terminated_length": 203.225, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.0178836390376091, "epoch": 1.1179553185621036, "frac_reward_zero_std": 0.2, "grad_norm": 4.887229919433594, "kl": 0.012441344145918265, "learning_rate": 3.7071940066438068e-06, "loss": 0.0012, "num_tokens": 53032374.0, "reward": 0.025, "reward_std": 0.6346783876419068, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6346783995628357, "sampling/importance_sampling_ratio/max": 1.3561378717422485, "sampling/importance_sampling_ratio/mean": 0.9996085047721863, "sampling/importance_sampling_ratio/min": 0.8018588960170746, "sampling/sampling_logp_difference/max": 0.31487315595149995, "sampling/sampling_logp_difference/mean": 0.02117849076166749, "step": 12160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 209.4375, "completions/mean_terminated_length": 209.4375, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.0983270026743412, "epoch": 1.1188746897122368, "frac_reward_zero_std": 0.4, "grad_norm": 0.09900511056184769, "kl": 0.012520093680359423, "learning_rate": 3.704974723144577e-06, "loss": 0.0008, "num_tokens": 53075841.0, "reward": 0.225, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.291766595840454, "sampling/importance_sampling_ratio/mean": 1.000008475780487, "sampling/importance_sampling_ratio/min": 0.7781517386436463, "sampling/sampling_logp_difference/max": 0.28410058915615083, "sampling/sampling_logp_difference/mean": 0.02223222330212593, "step": 12170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 197.375, "completions/mean_terminated_length": 197.375, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.014752572029829, "epoch": 1.1197940608623702, "frac_reward_zero_std": 0.2, "grad_norm": 6.474295139312744, "kl": 0.012678937206510454, "learning_rate": 3.702754202039716e-06, "loss": 0.0005, "num_tokens": 53117519.0, "reward": 0.05, "reward_std": 0.6312837243080139, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6312837481498719, "sampling/importance_sampling_ratio/max": 1.2308696269989015, "sampling/importance_sampling_ratio/mean": 1.00050208568573, "sampling/importance_sampling_ratio/min": 0.7363070279359818, "sampling/sampling_logp_difference/max": 0.3217036485671997, "sampling/sampling_logp_difference/mean": 0.021117912605404854, "step": 12180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.3, "completions/max_terminated_length": 287.3, "completions/mean_length": 203.5, "completions/mean_terminated_length": 203.5, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.0140774600207805, "epoch": 1.1207134320125034, "frac_reward_zero_std": 0.4, "grad_norm": 5.435579299926758, "kl": 0.014961501909419894, "learning_rate": 3.7005324456098775e-06, "loss": 0.0006, "num_tokens": 53159423.0, "reward": -0.0625, "reward_std": 0.552970016002655, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.552970016002655, "sampling/importance_sampling_ratio/max": 1.28999183177948, "sampling/importance_sampling_ratio/mean": 0.9998874962329865, "sampling/importance_sampling_ratio/min": 0.7625764787197113, "sampling/sampling_logp_difference/max": 0.29125497341156004, "sampling/sampling_logp_difference/mean": 0.021246341057121754, "step": 12190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.3, "completions/max_terminated_length": 301.3, "completions/mean_length": 208.0125, "completions/mean_terminated_length": 208.0125, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.055083828419447, "epoch": 1.1216328031626368, "frac_reward_zero_std": 0.9, "grad_norm": 0.10548204183578491, "kl": 0.012017998000374064, "learning_rate": 3.6983094561369824e-06, "loss": 0.0008, "num_tokens": 53205400.0, "reward": 0.55, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": 0.55, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.3170812845230102, "sampling/importance_sampling_ratio/mean": 0.9998515963554382, "sampling/importance_sampling_ratio/min": 0.7803776383399963, "sampling/sampling_logp_difference/max": 0.2890640914440155, "sampling/sampling_logp_difference/mean": 0.02129198405891657, "step": 12200 }, { "epoch": 1.1216328031626368, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 289.46771378708553, "eval_completions/max_terminated_length": 288.47469458987786, "eval_completions/mean_length": 201.8776178010471, "eval_completions/mean_terminated_length": 201.59090637952661, "eval_completions/min_length": 130.76265270506107, "eval_completions/min_terminated_length": 130.76265270506107, "eval_entropy": 1.0561792978441529, "eval_frac_reward_zero_std": 0.46596858638743455, "eval_kl": 0.012627460533041136, "eval_loss": 0.0006139742908999324, "eval_num_tokens": 53205400.0, "eval_reward": -0.09216841186736474, "eval_reward_std": 0.46341219427385877, "eval_rewards/reward_fn/mean": -0.09216841186736474, "eval_rewards/reward_fn/std": 0.4634122061323745, "eval_runtime": 1690.2351, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2783575143281494, "eval_sampling/importance_sampling_ratio/mean": 0.9999982508689321, "eval_sampling/importance_sampling_ratio/min": 0.774383738610965, "eval_sampling/sampling_logp_difference/max": 0.2861264813528843, "eval_sampling/sampling_logp_difference/mean": 0.021093751232336432, "eval_steps_per_second": 0.043, "step": 12200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.6, "completions/max_terminated_length": 319.6, "completions/mean_length": 217.15, "completions/mean_terminated_length": 217.15, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.031428410857916, "epoch": 1.12255217431277, "frac_reward_zero_std": 0.8, "grad_norm": 0.10296295583248138, "kl": 0.011911520262947306, "learning_rate": 3.69608523590422e-06, "loss": 0.0006, "num_tokens": 53250468.0, "reward": -0.275, "reward_std": 0.1960918426513672, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.19609185457229614, "sampling/importance_sampling_ratio/max": 1.2859318852424622, "sampling/importance_sampling_ratio/mean": 1.000338476896286, "sampling/importance_sampling_ratio/min": 0.7407127380371094, "sampling/sampling_logp_difference/max": 0.3302626371383667, "sampling/sampling_logp_difference/mean": 0.020454227365553378, "step": 12210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 286.1, "completions/max_terminated_length": 273.7, "completions/mean_length": 181.4625, "completions/mean_terminated_length": 177.89285888671876, "completions/min_length": 104.3, "completions/min_terminated_length": 104.3, "entropy": 1.0507142577320336, "epoch": 1.1234715454629034, "frac_reward_zero_std": 0.5, "grad_norm": 0.1630687415599823, "kl": 0.011167748243315146, "learning_rate": 3.693859787196042e-06, "loss": 0.0002, "num_tokens": 53288753.0, "reward": -0.25625, "reward_std": 0.4859999716281891, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.4859999895095825, "sampling/importance_sampling_ratio/max": 1.2490258693695069, "sampling/importance_sampling_ratio/mean": 0.9999625861644745, "sampling/importance_sampling_ratio/min": 0.7808464825153351, "sampling/sampling_logp_difference/max": 0.2540972352027893, "sampling/sampling_logp_difference/mean": 0.022200950235128403, "step": 12220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 294.3, "completions/max_terminated_length": 270.5, "completions/mean_length": 198.475, "completions/mean_terminated_length": 194.6714294433594, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.0345068112015725, "epoch": 1.1243909166130366, "frac_reward_zero_std": 0.4, "grad_norm": 5.59700345993042, "kl": 0.012248511024517938, "learning_rate": 3.691633112298163e-06, "loss": 0.0003, "num_tokens": 53329191.0, "reward": -0.60625, "reward_std": 0.4368450567126274, "rewards/reward_fn/mean": -0.60625, "rewards/reward_fn/std": 0.4368450567126274, "sampling/importance_sampling_ratio/max": 1.2517698287963868, "sampling/importance_sampling_ratio/mean": 0.9999489605426788, "sampling/importance_sampling_ratio/min": 0.7544875562191009, "sampling/sampling_logp_difference/max": 0.28643131256103516, "sampling/sampling_logp_difference/mean": 0.021634946577250957, "step": 12230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.6, "completions/max_terminated_length": 304.6, "completions/mean_length": 207.2875, "completions/mean_terminated_length": 207.2875, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.097717361152172, "epoch": 1.12531028776317, "frac_reward_zero_std": 0.6, "grad_norm": 0.10334161669015884, "kl": 0.012060048483544961, "learning_rate": 3.689405213497557e-06, "loss": 0.0007, "num_tokens": 53370390.0, "reward": 0.45, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2643091201782226, "sampling/importance_sampling_ratio/mean": 0.9998683333396912, "sampling/importance_sampling_ratio/min": 0.783367919921875, "sampling/sampling_logp_difference/max": 0.25273096561431885, "sampling/sampling_logp_difference/mean": 0.022522908076643942, "step": 12240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 220.4875, "completions/mean_terminated_length": 220.4875, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.0166353404521942, "epoch": 1.1262296589133034, "frac_reward_zero_std": 0.5, "grad_norm": 4.756493091583252, "kl": 0.01124943799804896, "learning_rate": 3.6871760930824536e-06, "loss": 0.0005, "num_tokens": 53417645.0, "reward": 0.325, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2322880029678345, "sampling/importance_sampling_ratio/mean": 0.9998500883579254, "sampling/importance_sampling_ratio/min": 0.7762690603733062, "sampling/sampling_logp_difference/max": 0.2614967584609985, "sampling/sampling_logp_difference/mean": 0.020522818714380265, "step": 12250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.1, "completions/max_terminated_length": 258.1, "completions/mean_length": 194.9875, "completions/mean_terminated_length": 194.9875, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 1.0728314191102981, "epoch": 1.1271490300634366, "frac_reward_zero_std": 0.3, "grad_norm": 6.189232349395752, "kl": 0.012317131971940398, "learning_rate": 3.6849457533423393e-06, "loss": 0.0006, "num_tokens": 53459004.0, "reward": 0.075, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.3343515753746034, "sampling/importance_sampling_ratio/mean": 0.9999606072902679, "sampling/importance_sampling_ratio/min": 0.7747191667556763, "sampling/sampling_logp_difference/max": 0.305237752199173, "sampling/sampling_logp_difference/mean": 0.022095377184450628, "step": 12260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.9, "completions/max_terminated_length": 289.9, "completions/mean_length": 206.7625, "completions/mean_terminated_length": 206.7625, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.028277723491192, "epoch": 1.12806840121357, "frac_reward_zero_std": 0.7, "grad_norm": 0.12364787608385086, "kl": 0.01302525169448927, "learning_rate": 3.6827141965679514e-06, "loss": 0.0002, "num_tokens": 53502737.0, "reward": 0.125, "reward_std": 0.30299633741378784, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3029963493347168, "sampling/importance_sampling_ratio/max": 1.3401803374290466, "sampling/importance_sampling_ratio/mean": 1.0001133918762206, "sampling/importance_sampling_ratio/min": 0.7420195698738098, "sampling/sampling_logp_difference/max": 0.35154828429222107, "sampling/sampling_logp_difference/mean": 0.021223937533795832, "step": 12270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 216.7875, "completions/mean_terminated_length": 216.7875, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 1.0130896851420403, "epoch": 1.1289877723637032, "frac_reward_zero_std": 0.3, "grad_norm": 6.130321979522705, "kl": 0.012559604799025691, "learning_rate": 3.6804814250512773e-06, "loss": 0.0006, "num_tokens": 53548808.0, "reward": 0.175, "reward_std": 0.7019693493843079, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.7019693613052368, "sampling/importance_sampling_ratio/max": 1.2700276851654053, "sampling/importance_sampling_ratio/mean": 1.0001122593879699, "sampling/importance_sampling_ratio/min": 0.7117498219013214, "sampling/sampling_logp_difference/max": 0.34780893921852113, "sampling/sampling_logp_difference/mean": 0.021048117242753506, "step": 12280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 200.075, "completions/mean_terminated_length": 200.075, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.078808134049177, "epoch": 1.1299071435138366, "frac_reward_zero_std": 0.4, "grad_norm": 7.019436836242676, "kl": 0.01656638670247048, "learning_rate": 3.678247441085552e-06, "loss": 0.0006, "num_tokens": 53591942.0, "reward": -0.275, "reward_std": 0.5185230314731598, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5185230314731598, "sampling/importance_sampling_ratio/max": 1.3220712184906005, "sampling/importance_sampling_ratio/mean": 0.9999464333057404, "sampling/importance_sampling_ratio/min": 0.7833312153816223, "sampling/sampling_logp_difference/max": 0.2960884690284729, "sampling/sampling_logp_difference/mean": 0.022220443189144134, "step": 12290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.8, "completions/max_terminated_length": 272.8, "completions/mean_length": 198.3125, "completions/mean_terminated_length": 198.3125, "completions/min_length": 119.9, "completions/min_terminated_length": 119.9, "entropy": 0.9818216241896153, "epoch": 1.1308265146639698, "frac_reward_zero_std": 0.4, "grad_norm": 5.682991027832031, "kl": 0.01657593290437944, "learning_rate": 3.676012246965257e-06, "loss": 0.0009, "num_tokens": 53633519.0, "reward": -0.275, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.275973868370056, "sampling/importance_sampling_ratio/mean": 0.9999193668365478, "sampling/importance_sampling_ratio/min": 0.710552242398262, "sampling/sampling_logp_difference/max": 0.37530871033668517, "sampling/sampling_logp_difference/mean": 0.02069101985543966, "step": 12300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 352.5, "completions/max_terminated_length": 329.9, "completions/mean_length": 224.725, "completions/mean_terminated_length": 220.9857147216797, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.9959320686757565, "epoch": 1.1317458858141032, "frac_reward_zero_std": 0.3, "grad_norm": 5.514424800872803, "kl": 0.013531449815491214, "learning_rate": 3.6737758449861157e-06, "loss": 0.0006, "num_tokens": 53677969.0, "reward": 0.29375, "reward_std": 0.5184835538268089, "rewards/reward_fn/mean": 0.29375, "rewards/reward_fn/std": 0.5184835657477379, "sampling/importance_sampling_ratio/max": 1.2968328833580016, "sampling/importance_sampling_ratio/mean": 0.9998658776283265, "sampling/importance_sampling_ratio/min": 0.7535026609897614, "sampling/sampling_logp_difference/max": 0.30188372135162356, "sampling/sampling_logp_difference/mean": 0.020398037508130074, "step": 12310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 347.3, "completions/max_terminated_length": 325.0, "completions/mean_length": 232.225, "completions/mean_terminated_length": 228.44821472167968, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 1.025636949390173, "epoch": 1.1326652569642364, "frac_reward_zero_std": 0.6, "grad_norm": 5.157952785491943, "kl": 0.011742288072127849, "learning_rate": 3.6715382374450913e-06, "loss": 0.0003, "num_tokens": 53722875.0, "reward": -0.25625, "reward_std": 0.3925050973892212, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.3925050973892212, "sampling/importance_sampling_ratio/max": 1.3037067770957946, "sampling/importance_sampling_ratio/mean": 0.9997952461242676, "sampling/importance_sampling_ratio/min": 0.728518596291542, "sampling/sampling_logp_difference/max": 0.37299697399139403, "sampling/sampling_logp_difference/mean": 0.020400658436119556, "step": 12320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 323.9, "completions/max_terminated_length": 320.1, "completions/mean_length": 223.6625, "completions/mean_terminated_length": 218.4200012207031, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.0313350148499012, "epoch": 1.1335846281143698, "frac_reward_zero_std": 0.3, "grad_norm": 6.633230686187744, "kl": 0.01263767980854027, "learning_rate": 3.669299426640387e-06, "loss": 0.0008, "num_tokens": 53767088.0, "reward": 0.05625, "reward_std": 0.6436951994895935, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.6436952114105224, "sampling/importance_sampling_ratio/max": 1.3144375324249267, "sampling/importance_sampling_ratio/mean": 1.0002842605113984, "sampling/importance_sampling_ratio/min": 0.7571153312921524, "sampling/sampling_logp_difference/max": 0.32949955463409425, "sampling/sampling_logp_difference/mean": 0.020890774298459293, "step": 12330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 201.2875, "completions/mean_terminated_length": 201.2875, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.0728293262422084, "epoch": 1.1345039992645032, "frac_reward_zero_std": 0.5, "grad_norm": 0.1090034544467926, "kl": 0.015453766449354589, "learning_rate": 3.667059414871441e-06, "loss": 0.0009, "num_tokens": 53809823.0, "reward": -0.125, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2584784030914307, "sampling/importance_sampling_ratio/mean": 1.0003907978534698, "sampling/importance_sampling_ratio/min": 0.7751726329326629, "sampling/sampling_logp_difference/max": 0.2661453664302826, "sampling/sampling_logp_difference/mean": 0.021406669169664383, "step": 12340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 199.675, "completions/mean_terminated_length": 199.675, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0186984270811081, "epoch": 1.1354233704146364, "frac_reward_zero_std": 0.8, "grad_norm": 6.500293254852295, "kl": 0.011501302220858633, "learning_rate": 3.6648182044389253e-06, "loss": 0.0006, "num_tokens": 53853461.0, "reward": -0.225, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2849084496498109, "sampling/importance_sampling_ratio/mean": 0.9993336796760559, "sampling/importance_sampling_ratio/min": 0.7571995735168457, "sampling/sampling_logp_difference/max": 0.3020566523075104, "sampling/sampling_logp_difference/mean": 0.02102095019072294, "step": 12350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 224.1375, "completions/mean_terminated_length": 224.1375, "completions/min_length": 150.9, "completions/min_terminated_length": 150.9, "entropy": 1.0887417621910571, "epoch": 1.1363427415647698, "frac_reward_zero_std": 0.4, "grad_norm": 4.169373989105225, "kl": 0.011916199151892215, "learning_rate": 3.662575797644741e-06, "loss": 0.0011, "num_tokens": 53899440.0, "reward": -0.03125, "reward_std": 0.5776534259319306, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5776534497737884, "sampling/importance_sampling_ratio/max": 1.2663657426834107, "sampling/importance_sampling_ratio/mean": 1.0003524541854858, "sampling/importance_sampling_ratio/min": 0.7796760857105255, "sampling/sampling_logp_difference/max": 0.28089700937271117, "sampling/sampling_logp_difference/mean": 0.02172746378928423, "step": 12360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.9, "completions/max_terminated_length": 325.9, "completions/mean_length": 207.8375, "completions/mean_terminated_length": 207.8375, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.087399535626173, "epoch": 1.137262112714903, "frac_reward_zero_std": 0.3, "grad_norm": 6.147736549377441, "kl": 0.011948530719382688, "learning_rate": 3.6603321967920214e-06, "loss": 0.0009, "num_tokens": 53946027.0, "reward": 0.25, "reward_std": 0.5496452271938324, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.5496452391147614, "sampling/importance_sampling_ratio/max": 1.3125287532806396, "sampling/importance_sampling_ratio/mean": 1.0001753389835357, "sampling/importance_sampling_ratio/min": 0.7263323843479157, "sampling/sampling_logp_difference/max": 0.37815306186676023, "sampling/sampling_logp_difference/mean": 0.02185120861977339, "step": 12370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 213.5, "completions/mean_terminated_length": 213.5, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.0151874765753746, "epoch": 1.1381814838650364, "frac_reward_zero_std": 0.5, "grad_norm": 0.09354651719331741, "kl": 0.010899989551398903, "learning_rate": 3.6580874041851227e-06, "loss": 0.0004, "num_tokens": 53987771.0, "reward": 0.2, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.2574939727783203, "sampling/importance_sampling_ratio/mean": 1.0001673698425293, "sampling/importance_sampling_ratio/min": 0.7818777740001679, "sampling/sampling_logp_difference/max": 0.2598349392414093, "sampling/sampling_logp_difference/mean": 0.020879349298775197, "step": 12380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.3, "completions/max_terminated_length": 330.3, "completions/mean_length": 232.425, "completions/mean_terminated_length": 232.425, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.0512599933892488, "epoch": 1.1391008550151696, "frac_reward_zero_std": 0.4, "grad_norm": 4.839200496673584, "kl": 0.012590197002282366, "learning_rate": 3.6558414221296286e-06, "loss": 0.0009, "num_tokens": 54032205.0, "reward": -0.25, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.264869487285614, "sampling/importance_sampling_ratio/mean": 1.0003723621368408, "sampling/importance_sampling_ratio/min": 0.7837377965450287, "sampling/sampling_logp_difference/max": 0.26692872643470766, "sampling/sampling_logp_difference/mean": 0.020969119668006898, "step": 12390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.3, "completions/max_terminated_length": 327.3, "completions/mean_length": 225.3375, "completions/mean_terminated_length": 225.3375, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.1029418855905533, "epoch": 1.140020226165303, "frac_reward_zero_std": 0.2, "grad_norm": 5.164229393005371, "kl": 0.012617702328134328, "learning_rate": 3.6535942529323414e-06, "loss": 0.0001, "num_tokens": 54077736.0, "reward": 0.1, "reward_std": 0.6708722054958344, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6708722174167633, "sampling/importance_sampling_ratio/max": 1.253482210636139, "sampling/importance_sampling_ratio/mean": 0.9998850703239441, "sampling/importance_sampling_ratio/min": 0.7733970522880554, "sampling/sampling_logp_difference/max": 0.29305444955825805, "sampling/sampling_logp_difference/mean": 0.021436109393835067, "step": 12400 }, { "epoch": 1.140020226165303, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 308.9214659685864, "eval_completions/max_terminated_length": 308.043630017452, "eval_completions/mean_length": 216.02574171029667, "eval_completions/mean_terminated_length": 215.54278878682987, "eval_completions/min_length": 140.63874345549738, "eval_completions/min_terminated_length": 140.63874345549738, "eval_entropy": 1.083174330557828, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.014242578396999503, "eval_loss": 0.0006650012801401317, "eval_num_tokens": 54077736.0, "eval_reward": -0.10471204188481675, "eval_reward_std": 0.45165308930278864, "eval_rewards/reward_fn/mean": -0.10471204188481675, "eval_rewards/reward_fn/std": 0.4516531017854368, "eval_runtime": 1789.4179, "eval_samples_per_second": 0.32, "eval_sampling/importance_sampling_ratio/max": 1.266556337033683, "eval_sampling/importance_sampling_ratio/mean": 1.0000241674469819, "eval_sampling/importance_sampling_ratio/min": 0.7708774748897054, "eval_sampling/sampling_logp_difference/max": 0.28883706301086653, "eval_sampling/sampling_logp_difference/mean": 0.020960821040351768, "eval_steps_per_second": 0.04, "step": 12400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.1, "completions/max_terminated_length": 303.1, "completions/mean_length": 212.05, "completions/mean_terminated_length": 212.05, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0402690783143043, "epoch": 1.1409395973154361, "frac_reward_zero_std": 0.4, "grad_norm": 5.434593677520752, "kl": 0.010846780260908417, "learning_rate": 3.651345898901285e-06, "loss": 0.0008, "num_tokens": 54118916.0, "reward": 0.025, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.262868916988373, "sampling/importance_sampling_ratio/mean": 0.9996683776378632, "sampling/importance_sampling_ratio/min": 0.7702636182308197, "sampling/sampling_logp_difference/max": 0.2819303870201111, "sampling/sampling_logp_difference/mean": 0.02094754595309496, "step": 12410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.4, "completions/max_terminated_length": 275.4, "completions/mean_length": 192.275, "completions/mean_terminated_length": 192.275, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0168660305440427, "epoch": 1.1418589684655696, "frac_reward_zero_std": 0.5, "grad_norm": 0.13410750031471252, "kl": 0.012202571152010932, "learning_rate": 3.6490963623456988e-06, "loss": 0.0006, "num_tokens": 54162362.0, "reward": 0.25, "reward_std": 0.5066213369369507, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.5066213846206665, "sampling/importance_sampling_ratio/max": 1.2801340341567993, "sampling/importance_sampling_ratio/mean": 1.0000076234340667, "sampling/importance_sampling_ratio/min": 0.7842028498649597, "sampling/sampling_logp_difference/max": 0.26668068766593933, "sampling/sampling_logp_difference/mean": 0.021344155818223954, "step": 12420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.1, "completions/max_terminated_length": 270.1, "completions/mean_length": 191.5375, "completions/mean_terminated_length": 191.5375, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 0.9129383124411106, "epoch": 1.142778339615703, "frac_reward_zero_std": 0.5, "grad_norm": 0.2817608416080475, "kl": 0.011682229972211643, "learning_rate": 3.646845645576037e-06, "loss": 0.0006, "num_tokens": 54200869.0, "reward": -0.2, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2705991983413696, "sampling/importance_sampling_ratio/mean": 1.0003530502319335, "sampling/importance_sampling_ratio/min": 0.7745735108852386, "sampling/sampling_logp_difference/max": 0.278059983253479, "sampling/sampling_logp_difference/mean": 0.019903586059808732, "step": 12430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.8, "completions/max_terminated_length": 298.8, "completions/mean_length": 198.5, "completions/mean_terminated_length": 198.5, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.11761676967144, "epoch": 1.1436977107658362, "frac_reward_zero_std": 0.8, "grad_norm": 0.1399431824684143, "kl": 0.01135429558926262, "learning_rate": 3.6445937509039656e-06, "loss": 0.0005, "num_tokens": 54247245.0, "reward": 0.35, "reward_std": 0.19948650598526002, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.19948650598526002, "sampling/importance_sampling_ratio/max": 1.256856656074524, "sampling/importance_sampling_ratio/mean": 0.9998637437820435, "sampling/importance_sampling_ratio/min": 0.7571885824203491, "sampling/sampling_logp_difference/max": 0.29250948429107665, "sampling/sampling_logp_difference/mean": 0.022143567726016045, "step": 12440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.4, "completions/max_terminated_length": 327.4, "completions/mean_length": 217.2625, "completions/mean_terminated_length": 217.2625, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.0745746470987796, "epoch": 1.1446170819159696, "frac_reward_zero_std": 0.6, "grad_norm": 5.988842010498047, "kl": 0.014810545672662555, "learning_rate": 3.6423406806423627e-06, "loss": 0.0006, "num_tokens": 54291058.0, "reward": -0.4, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2883201479911803, "sampling/importance_sampling_ratio/mean": 1.0000767648220061, "sampling/importance_sampling_ratio/min": 0.7697130084037781, "sampling/sampling_logp_difference/max": 0.2913903474807739, "sampling/sampling_logp_difference/mean": 0.021456178463995456, "step": 12450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 202.6125, "completions/mean_terminated_length": 202.6125, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.0493624791502953, "epoch": 1.1455364530661027, "frac_reward_zero_std": 0.3, "grad_norm": 0.10072760283946991, "kl": 0.011389013135340065, "learning_rate": 3.6400864371053107e-06, "loss": 0.0007, "num_tokens": 54333075.0, "reward": 0.0, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.2821411728858947, "sampling/importance_sampling_ratio/mean": 0.99944948554039, "sampling/importance_sampling_ratio/min": 0.7661689281463623, "sampling/sampling_logp_difference/max": 0.2930430233478546, "sampling/sampling_logp_difference/mean": 0.02165175657719374, "step": 12460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.8, "completions/max_terminated_length": 269.8, "completions/mean_length": 187.75, "completions/mean_terminated_length": 187.75, "completions/min_length": 126.3, "completions/min_terminated_length": 126.3, "entropy": 1.0084045365452767, "epoch": 1.1464558242162362, "frac_reward_zero_std": 0.6, "grad_norm": 0.12635846436023712, "kl": 0.014879862201632931, "learning_rate": 3.6378310226080983e-06, "loss": 0.0009, "num_tokens": 54371015.0, "reward": -0.25, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2602856516838075, "sampling/importance_sampling_ratio/mean": 1.0002153038978576, "sampling/importance_sampling_ratio/min": 0.8034046053886413, "sampling/sampling_logp_difference/max": 0.250240683555603, "sampling/sampling_logp_difference/mean": 0.02079767417162657, "step": 12470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 332.9, "completions/max_terminated_length": 325.4, "completions/mean_length": 228.7, "completions/mean_terminated_length": 225.81785888671874, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0148583192378282, "epoch": 1.1473751953663693, "frac_reward_zero_std": 0.6, "grad_norm": 0.09223124384880066, "kl": 0.011133544918266125, "learning_rate": 3.6355744394672175e-06, "loss": 0.0006, "num_tokens": 54418143.0, "reward": -0.68125, "reward_std": 0.2878748521208763, "rewards/reward_fn/mean": -0.68125, "rewards/reward_fn/std": 0.2878748521208763, "sampling/importance_sampling_ratio/max": 1.2456761240959167, "sampling/importance_sampling_ratio/mean": 0.9998093545436859, "sampling/importance_sampling_ratio/min": 0.7932051181793213, "sampling/sampling_logp_difference/max": 0.24386110305786132, "sampling/sampling_logp_difference/mean": 0.02001105807721615, "step": 12480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.0, "completions/max_terminated_length": 301.0, "completions/mean_length": 210.1125, "completions/mean_terminated_length": 210.1125, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.151090920716524, "epoch": 1.1482945665165027, "frac_reward_zero_std": 0.5, "grad_norm": 0.13338078558444977, "kl": 0.012656405195593834, "learning_rate": 3.63331669000036e-06, "loss": 0.0005, "num_tokens": 54458944.0, "reward": -0.075, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.29823397397995, "sampling/importance_sampling_ratio/mean": 1.000055456161499, "sampling/importance_sampling_ratio/min": 0.7115261822938919, "sampling/sampling_logp_difference/max": 0.37314416766166686, "sampling/sampling_logp_difference/mean": 0.022733479738235474, "step": 12490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.9, "completions/max_terminated_length": 307.9, "completions/mean_length": 225.9875, "completions/mean_terminated_length": 225.9875, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 1.021910773590207, "epoch": 1.149213937666636, "frac_reward_zero_std": 0.5, "grad_norm": 0.10317280888557434, "kl": 0.012988445401424542, "learning_rate": 3.6310577765264153e-06, "loss": 0.0005, "num_tokens": 54504511.0, "reward": -0.43125, "reward_std": 0.3879900202155113, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.38799004405736925, "sampling/importance_sampling_ratio/max": 1.336271619796753, "sampling/importance_sampling_ratio/mean": 1.0004246234893799, "sampling/importance_sampling_ratio/min": 0.7740168511867523, "sampling/sampling_logp_difference/max": 0.32091776728630067, "sampling/sampling_logp_difference/mean": 0.020962953194975854, "step": 12500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.5, "completions/max_terminated_length": 336.5, "completions/mean_length": 223.4875, "completions/mean_terminated_length": 223.4875, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.1105285182595253, "epoch": 1.1501333088167693, "frac_reward_zero_std": 0.6, "grad_norm": 0.17854128777980804, "kl": 0.011106331908376888, "learning_rate": 3.6287977013654692e-06, "loss": 0.0008, "num_tokens": 54547478.0, "reward": -0.35, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2459112882614136, "sampling/importance_sampling_ratio/mean": 1.0002007722854613, "sampling/importance_sampling_ratio/min": 0.7771866858005524, "sampling/sampling_logp_difference/max": 0.2644673466682434, "sampling/sampling_logp_difference/mean": 0.022044141218066214, "step": 12510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.0, "completions/max_terminated_length": 309.2, "completions/mean_length": 228.075, "completions/mean_terminated_length": 224.9857147216797, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0584735155105591, "epoch": 1.1510526799669027, "frac_reward_zero_std": 0.4, "grad_norm": 0.10628970712423325, "kl": 0.011398080026265234, "learning_rate": 3.6265364668387992e-06, "loss": 0.0004, "num_tokens": 54590764.0, "reward": -0.1875, "reward_std": 0.4900029540061951, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.49000295996665955, "sampling/importance_sampling_ratio/max": 1.2882118940353393, "sampling/importance_sampling_ratio/mean": 0.9998377203941345, "sampling/importance_sampling_ratio/min": 0.7752443373203277, "sampling/sampling_logp_difference/max": 0.27067553997039795, "sampling/sampling_logp_difference/mean": 0.021263398230075836, "step": 12520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.4, "completions/max_terminated_length": 305.4, "completions/mean_length": 206.7875, "completions/mean_terminated_length": 206.7875, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "entropy": 1.073888222873211, "epoch": 1.151972051117036, "frac_reward_zero_std": 0.2, "grad_norm": 11.248796463012695, "kl": 0.016087236523162574, "learning_rate": 3.6242740752688744e-06, "loss": 0.0007, "num_tokens": 54631131.0, "reward": 0.325, "reward_std": 0.6456219017505646, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.6456219017505646, "sampling/importance_sampling_ratio/max": 1.3054659843444825, "sampling/importance_sampling_ratio/mean": 1.0005244255065917, "sampling/importance_sampling_ratio/min": 0.7720902621746063, "sampling/sampling_logp_difference/max": 0.296278840303421, "sampling/sampling_logp_difference/mean": 0.022569910250604154, "step": 12530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.9, "completions/max_terminated_length": 262.9, "completions/mean_length": 199.95, "completions/mean_terminated_length": 199.95, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 0.9824341624975205, "epoch": 1.1528914222671693, "frac_reward_zero_std": 0.3, "grad_norm": 5.499989986419678, "kl": 0.01499407779192552, "learning_rate": 3.6220105289793533e-06, "loss": 0.0009, "num_tokens": 54669607.0, "reward": -0.05, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.3007944703102112, "sampling/importance_sampling_ratio/mean": 1.0000426352024079, "sampling/importance_sampling_ratio/min": 0.7494945049285888, "sampling/sampling_logp_difference/max": 0.31029879450798037, "sampling/sampling_logp_difference/mean": 0.02065894789993763, "step": 12540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.2, "completions/max_terminated_length": 337.2, "completions/mean_length": 215.3625, "completions/mean_terminated_length": 215.3625, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.124809304624796, "epoch": 1.1538107934173025, "frac_reward_zero_std": 0.3, "grad_norm": 5.543949127197266, "kl": 0.014431826418149284, "learning_rate": 3.619745830295078e-06, "loss": 0.0005, "num_tokens": 54713308.0, "reward": 0.225, "reward_std": 0.6514373540878295, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.6514373898506165, "sampling/importance_sampling_ratio/max": 1.3219092965126038, "sampling/importance_sampling_ratio/mean": 0.9995937347412109, "sampling/importance_sampling_ratio/min": 0.7265441209077835, "sampling/sampling_logp_difference/max": 0.3458713352680206, "sampling/sampling_logp_difference/mean": 0.02273276373744011, "step": 12550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.4, "completions/max_terminated_length": 301.1, "completions/mean_length": 208.2, "completions/mean_terminated_length": 204.18214416503906, "completions/min_length": 120.3, "completions/min_terminated_length": 120.3, "entropy": 1.046748274564743, "epoch": 1.154730164567436, "frac_reward_zero_std": 0.3, "grad_norm": 6.796682357788086, "kl": 0.013036166183883324, "learning_rate": 3.6174799815420776e-06, "loss": 0.0006, "num_tokens": 54754956.0, "reward": -0.48125, "reward_std": 0.5731540426611901, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.5731540665030479, "sampling/importance_sampling_ratio/max": 1.297311246395111, "sampling/importance_sampling_ratio/mean": 1.0001798033714295, "sampling/importance_sampling_ratio/min": 0.7623909890651703, "sampling/sampling_logp_difference/max": 0.3018287181854248, "sampling/sampling_logp_difference/mean": 0.0218272702768445, "step": 12560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 174.0, "completions/mean_terminated_length": 174.0, "completions/min_length": 107.9, "completions/min_terminated_length": 107.9, "entropy": 1.0492434844374656, "epoch": 1.1556495357175691, "frac_reward_zero_std": 0.7, "grad_norm": 0.09331375360488892, "kl": 0.012966365844476968, "learning_rate": 3.6152129850475593e-06, "loss": 0.0006, "num_tokens": 54793540.0, "reward": 0.1, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.269545328617096, "sampling/importance_sampling_ratio/mean": 1.0008475720882415, "sampling/importance_sampling_ratio/min": 0.7867902874946594, "sampling/sampling_logp_difference/max": 0.27173700332641604, "sampling/sampling_logp_difference/mean": 0.022270133160054682, "step": 12570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 318.1, "completions/max_terminated_length": 302.1, "completions/mean_length": 202.975, "completions/mean_terminated_length": 199.61607208251954, "completions/min_length": 124.7, "completions/min_terminated_length": 124.7, "entropy": 1.0635680936276912, "epoch": 1.1565689068677025, "frac_reward_zero_std": 0.6, "grad_norm": 5.3730058670043945, "kl": 0.011928155890200288, "learning_rate": 3.612944843139908e-06, "loss": 0.0008, "num_tokens": 54835490.0, "reward": 0.14375, "reward_std": 0.3647729933261871, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.36477301120758054, "sampling/importance_sampling_ratio/max": 1.3304497361183167, "sampling/importance_sampling_ratio/mean": 1.000501710176468, "sampling/importance_sampling_ratio/min": 0.7894047737121582, "sampling/sampling_logp_difference/max": 0.29821774959564207, "sampling/sampling_logp_difference/mean": 0.02112394478172064, "step": 12580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 205.025, "completions/mean_terminated_length": 205.025, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0804559476673603, "epoch": 1.1574882780178357, "frac_reward_zero_std": 0.2, "grad_norm": 7.238084316253662, "kl": 0.013854928861837834, "learning_rate": 3.610675558148692e-06, "loss": 0.001, "num_tokens": 54884892.0, "reward": -0.60625, "reward_std": 0.7048787415027619, "rewards/reward_fn/mean": -0.60625, "rewards/reward_fn/std": 0.7048787593841552, "sampling/importance_sampling_ratio/max": 1.2882227897644043, "sampling/importance_sampling_ratio/mean": 0.9995758056640625, "sampling/importance_sampling_ratio/min": 0.7788994252681732, "sampling/sampling_logp_difference/max": 0.2810662865638733, "sampling/sampling_logp_difference/mean": 0.02160454485565424, "step": 12590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.3, "completions/max_terminated_length": 257.3, "completions/mean_length": 184.6, "completions/mean_terminated_length": 184.6, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0229171082377433, "epoch": 1.1584076491679691, "frac_reward_zero_std": 0.7, "grad_norm": 9.212437629699707, "kl": 0.012134133110521361, "learning_rate": 3.608405132404644e-06, "loss": 0.0006, "num_tokens": 54923980.0, "reward": 0.2, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.3118669390678406, "sampling/importance_sampling_ratio/mean": 1.0005741178989411, "sampling/importance_sampling_ratio/min": 0.7755885124206543, "sampling/sampling_logp_difference/max": 0.31504634618759153, "sampling/sampling_logp_difference/mean": 0.021250465884804726, "step": 12600 }, { "epoch": 1.1584076491679691, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 306.7294938917976, "eval_completions/max_terminated_length": 304.18150087260034, "eval_completions/mean_length": 210.50305410122164, "eval_completions/mean_terminated_length": 209.76226439151463, "eval_completions/min_length": 134.65968586387436, "eval_completions/min_terminated_length": 134.65968586387436, "eval_entropy": 1.1013592953353235, "eval_frac_reward_zero_std": 0.48342059336823734, "eval_kl": 0.012229366240174514, "eval_loss": 0.0006057066493667662, "eval_num_tokens": 54923980.0, "eval_reward": -0.09489528795811518, "eval_reward_std": 0.456165896842409, "eval_rewards/reward_fn/mean": -0.09489528795811518, "eval_rewards/reward_fn/std": 0.45616590911701294, "eval_runtime": 1778.4196, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.2856926576837404, "eval_sampling/importance_sampling_ratio/mean": 1.0000591595343478, "eval_sampling/importance_sampling_ratio/min": 0.7761192623701395, "eval_sampling/sampling_logp_difference/max": 0.2921760023472405, "eval_sampling/sampling_logp_difference/mean": 0.021350799633895436, "eval_steps_per_second": 0.04, "step": 12600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.3, "completions/max_terminated_length": 305.3, "completions/mean_length": 206.0375, "completions/mean_terminated_length": 206.0375, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.121750009804964, "epoch": 1.1593270203181025, "frac_reward_zero_std": 0.6, "grad_norm": 6.742509365081787, "kl": 0.012727925222134218, "learning_rate": 3.606133568239677e-06, "loss": 0.0003, "num_tokens": 54970287.0, "reward": 0.05, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.357997715473175, "sampling/importance_sampling_ratio/mean": 1.000779354572296, "sampling/importance_sampling_ratio/min": 0.748317939043045, "sampling/sampling_logp_difference/max": 0.43596471548080445, "sampling/sampling_logp_difference/mean": 0.022110162302851677, "step": 12610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 350.0, "completions/max_terminated_length": 350.0, "completions/mean_length": 229.275, "completions/mean_terminated_length": 229.275, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.0642025560140609, "epoch": 1.1602463914682357, "frac_reward_zero_std": 0.4, "grad_norm": 0.20541036128997803, "kl": 0.012259281723527238, "learning_rate": 3.603860867986867e-06, "loss": 0.0007, "num_tokens": 55013389.0, "reward": -0.6, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.2909961581230163, "sampling/importance_sampling_ratio/mean": 1.0001421093940734, "sampling/importance_sampling_ratio/min": 0.7888669669628143, "sampling/sampling_logp_difference/max": 0.2695030927658081, "sampling/sampling_logp_difference/mean": 0.021201279200613498, "step": 12620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.5, "completions/max_terminated_length": 301.5, "completions/mean_length": 206.625, "completions/mean_terminated_length": 206.625, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0769014090299607, "epoch": 1.1611657626183691, "frac_reward_zero_std": 0.3, "grad_norm": 5.442151069641113, "kl": 0.010956723539857193, "learning_rate": 3.6015870339804604e-06, "loss": 0.0004, "num_tokens": 55058887.0, "reward": 0.05, "reward_std": 0.651453047990799, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.6514530599117279, "sampling/importance_sampling_ratio/max": 1.2624728202819824, "sampling/importance_sampling_ratio/mean": 0.999915212392807, "sampling/importance_sampling_ratio/min": 0.7636335611343383, "sampling/sampling_logp_difference/max": 0.29263397455215456, "sampling/sampling_logp_difference/mean": 0.020924648642539977, "step": 12630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.7, "completions/max_terminated_length": 317.7, "completions/mean_length": 211.75, "completions/mean_terminated_length": 211.75, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.0522572085261346, "epoch": 1.1620851337685023, "frac_reward_zero_std": 0.3, "grad_norm": 5.526965141296387, "kl": 0.01228720544022508, "learning_rate": 3.5993120685558676e-06, "loss": 0.0004, "num_tokens": 55102611.0, "reward": 0.0, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.254207146167755, "sampling/importance_sampling_ratio/mean": 1.0001215279102325, "sampling/importance_sampling_ratio/min": 0.7757926166057587, "sampling/sampling_logp_difference/max": 0.2641479015350342, "sampling/sampling_logp_difference/mean": 0.021587990410625934, "step": 12640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.5, "completions/max_terminated_length": 322.5, "completions/mean_length": 214.8375, "completions/mean_terminated_length": 214.8375, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.032759815454483, "epoch": 1.1630045049186357, "frac_reward_zero_std": 0.8, "grad_norm": 5.007786750793457, "kl": 0.010986890603089706, "learning_rate": 3.5970359740496593e-06, "loss": 0.0006, "num_tokens": 55143990.0, "reward": -0.175, "reward_std": 0.2104143261909485, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.21041433811187743, "sampling/importance_sampling_ratio/max": 1.2535508871078491, "sampling/importance_sampling_ratio/mean": 1.0002652645111083, "sampling/importance_sampling_ratio/min": 0.7327168226242066, "sampling/sampling_logp_difference/max": 0.33545646667480467, "sampling/sampling_logp_difference/mean": 0.02101404294371605, "step": 12650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 190.7125, "completions/mean_terminated_length": 190.7125, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 0.9963625859469175, "epoch": 1.163923876068769, "frac_reward_zero_std": 0.3, "grad_norm": 7.835896968841553, "kl": 0.01208912908914499, "learning_rate": 3.5947587527995675e-06, "loss": 0.0009, "num_tokens": 55185927.0, "reward": 0.25, "reward_std": 0.6657755315303803, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.6657755434513092, "sampling/importance_sampling_ratio/max": 1.2728695392608642, "sampling/importance_sampling_ratio/mean": 0.9997296690940857, "sampling/importance_sampling_ratio/min": 0.7763293325901032, "sampling/sampling_logp_difference/max": 0.292890465259552, "sampling/sampling_logp_difference/mean": 0.020640317630022765, "step": 12660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.9, "completions/max_terminated_length": 297.3, "completions/mean_length": 211.725, "completions/mean_terminated_length": 207.89107208251954, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.0390664875507354, "epoch": 1.1648432472189023, "frac_reward_zero_std": 0.4, "grad_norm": 5.17177152633667, "kl": 0.011938359192572533, "learning_rate": 3.5924804071444806e-06, "loss": 0.0006, "num_tokens": 55229993.0, "reward": -0.28125, "reward_std": 0.5729019403457641, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.5729019403457641, "sampling/importance_sampling_ratio/max": 1.2566454410552979, "sampling/importance_sampling_ratio/mean": 1.0000206232070923, "sampling/importance_sampling_ratio/min": 0.7610966980457305, "sampling/sampling_logp_difference/max": 0.29410581588745116, "sampling/sampling_logp_difference/mean": 0.021340812370181084, "step": 12670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 327.1, "completions/max_terminated_length": 307.5, "completions/mean_length": 219.2125, "completions/mean_terminated_length": 212.74166717529297, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 1.0432783015072347, "epoch": 1.1657626183690355, "frac_reward_zero_std": 0.6, "grad_norm": 4.566485404968262, "kl": 0.01149973240098916, "learning_rate": 3.5902009394244436e-06, "loss": 0.0005, "num_tokens": 55274858.0, "reward": -0.5375, "reward_std": 0.32614184021949766, "rewards/reward_fn/mean": -0.5375, "rewards/reward_fn/std": 0.3261418521404266, "sampling/importance_sampling_ratio/max": 1.259331750869751, "sampling/importance_sampling_ratio/mean": 1.0002465486526488, "sampling/importance_sampling_ratio/min": 0.7440794467926025, "sampling/sampling_logp_difference/max": 0.30621405243873595, "sampling/sampling_logp_difference/mean": 0.021598411537706853, "step": 12680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.5, "completions/max_terminated_length": 326.5, "completions/mean_length": 236.3375, "completions/mean_terminated_length": 236.3375, "completions/min_length": 157.3, "completions/min_terminated_length": 157.3, "entropy": 1.0132663749158382, "epoch": 1.166681989519169, "frac_reward_zero_std": 0.2, "grad_norm": 0.10051736235618591, "kl": 0.011953853390878066, "learning_rate": 3.58792035198065e-06, "loss": 0.0002, "num_tokens": 55319245.0, "reward": 0.4, "reward_std": 0.7658906996250152, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.7658907353878022, "sampling/importance_sampling_ratio/max": 1.345170831680298, "sampling/importance_sampling_ratio/mean": 1.0000717043876648, "sampling/importance_sampling_ratio/min": 0.7481326639652253, "sampling/sampling_logp_difference/max": 0.33303438425064086, "sampling/sampling_logp_difference/mean": 0.02075956594198942, "step": 12690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 209.475, "completions/mean_terminated_length": 209.475, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0350405376404523, "epoch": 1.1676013606693023, "frac_reward_zero_std": 0.6, "grad_norm": 0.10715000331401825, "kl": 0.010773051006253809, "learning_rate": 3.5856386471554485e-06, "loss": 0.0008, "num_tokens": 55361043.0, "reward": -0.1, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2382286667823792, "sampling/importance_sampling_ratio/mean": 0.9998479604721069, "sampling/importance_sampling_ratio/min": 0.7848574995994568, "sampling/sampling_logp_difference/max": 0.24637718200683595, "sampling/sampling_logp_difference/mean": 0.020432186499238014, "step": 12700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 212.0625, "completions/mean_terminated_length": 212.0625, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.056779956817627, "epoch": 1.1685207318194355, "frac_reward_zero_std": 0.4, "grad_norm": 4.928626537322998, "kl": 0.012540060345781968, "learning_rate": 3.583355827292331e-06, "loss": 0.0005, "num_tokens": 55403912.0, "reward": 0.06875, "reward_std": 0.559176754951477, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.559176766872406, "sampling/importance_sampling_ratio/max": 1.2459088444709778, "sampling/importance_sampling_ratio/mean": 1.000076562166214, "sampling/importance_sampling_ratio/min": 0.773871660232544, "sampling/sampling_logp_difference/max": 0.2714258313179016, "sampling/sampling_logp_difference/mean": 0.021591346152126788, "step": 12710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 362.4, "completions/max_terminated_length": 359.7, "completions/mean_length": 246.6625, "completions/mean_terminated_length": 243.87857360839843, "completions/min_length": 162.3, "completions/min_terminated_length": 162.3, "entropy": 1.1107692122459412, "epoch": 1.169440102969569, "frac_reward_zero_std": 0.3, "grad_norm": 5.177890777587891, "kl": 0.012221076112473384, "learning_rate": 3.581071894735938e-06, "loss": 0.0009, "num_tokens": 55454069.0, "reward": -0.03125, "reward_std": 0.5563950777053833, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5563950836658478, "sampling/importance_sampling_ratio/max": 1.2985886931419373, "sampling/importance_sampling_ratio/mean": 0.9998702824115753, "sampling/importance_sampling_ratio/min": 0.7680070042610169, "sampling/sampling_logp_difference/max": 0.30726170539855957, "sampling/sampling_logp_difference/mean": 0.021647788025438784, "step": 12720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 211.2375, "completions/mean_terminated_length": 211.2375, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.0863526985049248, "epoch": 1.170359474119702, "frac_reward_zero_std": 0.3, "grad_norm": 6.739457607269287, "kl": 0.011581424210453406, "learning_rate": 3.578786851832051e-06, "loss": 0.0005, "num_tokens": 55498024.0, "reward": -0.275, "reward_std": 0.6582266807556152, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6582266926765442, "sampling/importance_sampling_ratio/max": 1.2751100063323975, "sampling/importance_sampling_ratio/mean": 0.999888825416565, "sampling/importance_sampling_ratio/min": 0.7787648916244507, "sampling/sampling_logp_difference/max": 0.27470033168792723, "sampling/sampling_logp_difference/mean": 0.02203670032322407, "step": 12730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 214.375, "completions/mean_terminated_length": 214.375, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.058341247588396, "epoch": 1.1712788452698355, "frac_reward_zero_std": 0.5, "grad_norm": 0.10166177153587341, "kl": 0.012251917930552737, "learning_rate": 3.57650070092759e-06, "loss": 0.0006, "num_tokens": 55542630.0, "reward": -0.125, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.2957230925559997, "sampling/importance_sampling_ratio/mean": 1.000385767221451, "sampling/importance_sampling_ratio/min": 0.7624628782272339, "sampling/sampling_logp_difference/max": 0.29712232351303103, "sampling/sampling_logp_difference/mean": 0.02128039225935936, "step": 12740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 199.5625, "completions/mean_terminated_length": 199.5625, "completions/min_length": 121.9, "completions/min_terminated_length": 121.9, "entropy": 1.0205810960382222, "epoch": 1.1721982164199687, "frac_reward_zero_std": 0.7, "grad_norm": 0.08669167011976242, "kl": 0.012410928506869822, "learning_rate": 3.574213444370619e-06, "loss": 0.0007, "num_tokens": 55588411.0, "reward": 0.725, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.725, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2518082141876221, "sampling/importance_sampling_ratio/mean": 1.000440388917923, "sampling/importance_sampling_ratio/min": 0.7561547636985779, "sampling/sampling_logp_difference/max": 0.2884357452392578, "sampling/sampling_logp_difference/mean": 0.02118891980499029, "step": 12750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.4, "completions/max_terminated_length": 284.4, "completions/mean_length": 204.3875, "completions/mean_terminated_length": 204.3875, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.0744456112384797, "epoch": 1.173117587570102, "frac_reward_zero_std": 0.7, "grad_norm": 0.1949063390493393, "kl": 0.012988309538923205, "learning_rate": 3.571925084510331e-06, "loss": 0.0006, "num_tokens": 55635394.0, "reward": 0.075, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2975205779075623, "sampling/importance_sampling_ratio/mean": 0.9999358117580414, "sampling/importance_sampling_ratio/min": 0.786727124452591, "sampling/sampling_logp_difference/max": 0.2910907506942749, "sampling/sampling_logp_difference/mean": 0.021737824007868767, "step": 12760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.0, "completions/max_terminated_length": 322.5, "completions/mean_length": 224.6875, "completions/mean_terminated_length": 220.90892944335937, "completions/min_length": 150.9, "completions/min_terminated_length": 150.9, "entropy": 1.0663698039948941, "epoch": 1.1740369587202353, "frac_reward_zero_std": 0.3, "grad_norm": 4.10484504699707, "kl": 0.012995854404289275, "learning_rate": 3.569635623697056e-06, "loss": 0.0005, "num_tokens": 55680593.0, "reward": -0.23125, "reward_std": 0.6759043514728547, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.675904369354248, "sampling/importance_sampling_ratio/max": 1.3322051286697387, "sampling/importance_sampling_ratio/mean": 1.0005125999450684, "sampling/importance_sampling_ratio/min": 0.7877055585384369, "sampling/sampling_logp_difference/max": 0.2962402582168579, "sampling/sampling_logp_difference/mean": 0.0216470830142498, "step": 12770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.7, "completions/max_terminated_length": 286.7, "completions/mean_length": 201.15, "completions/mean_terminated_length": 201.15, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0107021421194076, "epoch": 1.1749563298703687, "frac_reward_zero_std": 0.7, "grad_norm": 5.9101243019104, "kl": 0.010888635157607495, "learning_rate": 3.567345064282253e-06, "loss": 0.0006, "num_tokens": 55723317.0, "reward": -0.275, "reward_std": 0.30299633741378784, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.3029963493347168, "sampling/importance_sampling_ratio/max": 1.413498294353485, "sampling/importance_sampling_ratio/mean": 1.000284332036972, "sampling/importance_sampling_ratio/min": 0.7827315866947174, "sampling/sampling_logp_difference/max": 0.34492877721786497, "sampling/sampling_logp_difference/mean": 0.02071852460503578, "step": 12780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 212.6125, "completions/mean_terminated_length": 212.6125, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.1118604011833668, "epoch": 1.175875701020502, "frac_reward_zero_std": 0.4, "grad_norm": 4.9229888916015625, "kl": 0.012349726917454973, "learning_rate": 3.565053408618512e-06, "loss": 0.0007, "num_tokens": 55770886.0, "reward": -0.125, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2967051267623901, "sampling/importance_sampling_ratio/mean": 0.9996502220630645, "sampling/importance_sampling_ratio/min": 0.7893350720405579, "sampling/sampling_logp_difference/max": 0.2794538974761963, "sampling/sampling_logp_difference/mean": 0.021814958564937113, "step": 12790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 208.9875, "completions/mean_terminated_length": 208.9875, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.0697237335145473, "epoch": 1.1767950721706353, "frac_reward_zero_std": 0.4, "grad_norm": 0.6504345536231995, "kl": 0.014298660718486645, "learning_rate": 3.562760659059545e-06, "loss": 0.001, "num_tokens": 55813589.0, "reward": 0.225, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2672592401504517, "sampling/importance_sampling_ratio/mean": 1.0000210225582122, "sampling/importance_sampling_ratio/min": 0.775453382730484, "sampling/sampling_logp_difference/max": 0.2745931625366211, "sampling/sampling_logp_difference/mean": 0.02135284524410963, "step": 12800 }, { "epoch": 1.1767950721706353, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 298.1849912739965, "eval_completions/max_terminated_length": 296.2687609075044, "eval_completions/mean_length": 207.20244328097732, "eval_completions/mean_terminated_length": 206.6458075558328, "eval_completions/min_length": 134.108202443281, "eval_completions/min_terminated_length": 134.108202443281, "eval_entropy": 1.0864979725129524, "eval_frac_reward_zero_std": 0.49040139616055844, "eval_kl": 0.011991814104758892, "eval_loss": 0.0006465326296165586, "eval_num_tokens": 55813589.0, "eval_reward": -0.10929319371727748, "eval_reward_std": 0.4459930578503517, "eval_rewards/reward_fn/mean": -0.10929319371727748, "eval_rewards/reward_fn/std": 0.44599307064506605, "eval_runtime": 1735.3667, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2807395782770288, "eval_sampling/importance_sampling_ratio/mean": 0.9999793161480423, "eval_sampling/importance_sampling_ratio/min": 0.7749498236865897, "eval_sampling/sampling_logp_difference/max": 0.29157572168122203, "eval_sampling/sampling_logp_difference/mean": 0.02129393328878424, "eval_steps_per_second": 0.041, "step": 12800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 211.8375, "completions/mean_terminated_length": 211.8375, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.1028744913637638, "epoch": 1.1777144433207687, "frac_reward_zero_std": 0.8, "grad_norm": 0.08775947988033295, "kl": 0.011128666019067169, "learning_rate": 3.560466817960191e-06, "loss": 0.0007, "num_tokens": 55856536.0, "reward": -0.125, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.3702365636825562, "sampling/importance_sampling_ratio/mean": 1.0002619087696076, "sampling/importance_sampling_ratio/min": 0.7567494451999665, "sampling/sampling_logp_difference/max": 0.3640476822853088, "sampling/sampling_logp_difference/mean": 0.02163134515285492, "step": 12810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 208.7625, "completions/mean_terminated_length": 208.7625, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.0933176532387734, "epoch": 1.1786338144709019, "frac_reward_zero_std": 0.4, "grad_norm": 6.988748550415039, "kl": 0.012145064072683454, "learning_rate": 3.5581718876764078e-06, "loss": 0.0006, "num_tokens": 55903981.0, "reward": -0.3, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.3265919804573059, "sampling/importance_sampling_ratio/mean": 1.0005363643169403, "sampling/importance_sampling_ratio/min": 0.8034933507442474, "sampling/sampling_logp_difference/max": 0.2880922079086304, "sampling/sampling_logp_difference/mean": 0.02170318327844143, "step": 12820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.0, "completions/max_terminated_length": 300.0, "completions/mean_length": 204.7125, "completions/mean_terminated_length": 204.7125, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.0945002034306526, "epoch": 1.1795531856210353, "frac_reward_zero_std": 0.4, "grad_norm": 6.154784679412842, "kl": 0.014952203584834933, "learning_rate": 3.5558758705652734e-06, "loss": 0.001, "num_tokens": 55949710.0, "reward": -0.025, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.291234052181244, "sampling/importance_sampling_ratio/mean": 0.9996741116046906, "sampling/importance_sampling_ratio/min": 0.7636328399181366, "sampling/sampling_logp_difference/max": 0.29918264150619506, "sampling/sampling_logp_difference/mean": 0.022126870043575764, "step": 12830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.1, "completions/max_terminated_length": 294.1, "completions/mean_length": 208.3375, "completions/mean_terminated_length": 208.3375, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.103837039321661, "epoch": 1.1804725567711685, "frac_reward_zero_std": 0.5, "grad_norm": 5.344085693359375, "kl": 0.012098693894222379, "learning_rate": 3.553578768984981e-06, "loss": 0.0004, "num_tokens": 55993041.0, "reward": -0.35, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.358351182937622, "sampling/importance_sampling_ratio/mean": 0.9996715426445008, "sampling/importance_sampling_ratio/min": 0.7520042598247528, "sampling/sampling_logp_difference/max": 0.3414980202913284, "sampling/sampling_logp_difference/mean": 0.022183686122298242, "step": 12840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.4, "completions/max_terminated_length": 290.4, "completions/mean_length": 211.5, "completions/mean_terminated_length": 211.5, "completions/min_length": 153.2, "completions/min_terminated_length": 153.2, "entropy": 1.2031229145824909, "epoch": 1.1813919279213019, "frac_reward_zero_std": 0.5, "grad_norm": 4.662481307983398, "kl": 0.011975906405132264, "learning_rate": 3.5512805852948373e-06, "loss": 0.0008, "num_tokens": 56037665.0, "reward": -0.25, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2286482572555542, "sampling/importance_sampling_ratio/mean": 0.999864786863327, "sampling/importance_sampling_ratio/min": 0.7971971094608307, "sampling/sampling_logp_difference/max": 0.2382061719894409, "sampling/sampling_logp_difference/mean": 0.022612899728119373, "step": 12850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.4, "completions/max_terminated_length": 289.4, "completions/mean_length": 197.25, "completions/mean_terminated_length": 197.25, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.04543234705925, "epoch": 1.182311299071435, "frac_reward_zero_std": 0.5, "grad_norm": 5.8094329833984375, "kl": 0.013301546478760428, "learning_rate": 3.5489813218552637e-06, "loss": 0.001, "num_tokens": 56080013.0, "reward": 0.14375, "reward_std": 0.43807622194290163, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.4380762279033661, "sampling/importance_sampling_ratio/max": 1.2647297024726867, "sampling/importance_sampling_ratio/mean": 1.000219476222992, "sampling/importance_sampling_ratio/min": 0.6951045632362366, "sampling/sampling_logp_difference/max": 0.389369535446167, "sampling/sampling_logp_difference/mean": 0.021411809697747232, "step": 12860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 211.025, "completions/mean_terminated_length": 211.025, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0509857039898634, "epoch": 1.1832306702215685, "frac_reward_zero_std": 0.2, "grad_norm": 5.337514400482178, "kl": 0.014142689021537081, "learning_rate": 3.5466809810277854e-06, "loss": 0.0007, "num_tokens": 56124527.0, "reward": -0.075, "reward_std": 0.6531550586223602, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6531550824642182, "sampling/importance_sampling_ratio/max": 1.2734569430351257, "sampling/importance_sampling_ratio/mean": 0.9995465576648712, "sampling/importance_sampling_ratio/min": 0.7653161704540252, "sampling/sampling_logp_difference/max": 0.29038324654102327, "sampling/sampling_logp_difference/mean": 0.021617262810468673, "step": 12870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 197.8375, "completions/mean_terminated_length": 197.8375, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.1190309591591359, "epoch": 1.1841500413717019, "frac_reward_zero_std": 0.3, "grad_norm": 4.930581569671631, "kl": 0.011664486338850111, "learning_rate": 3.544379565175038e-06, "loss": 0.0005, "num_tokens": 56166354.0, "reward": -0.075, "reward_std": 0.6699140250682831, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6699140727519989, "sampling/importance_sampling_ratio/max": 1.2508705615997315, "sampling/importance_sampling_ratio/mean": 1.0000218272209167, "sampling/importance_sampling_ratio/min": 0.7803862452507019, "sampling/sampling_logp_difference/max": 0.26537758111953735, "sampling/sampling_logp_difference/mean": 0.02254420518875122, "step": 12880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 201.325, "completions/mean_terminated_length": 201.325, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.1275114081799984, "epoch": 1.185069412521835, "frac_reward_zero_std": 0.7, "grad_norm": 7.318114280700684, "kl": 0.013375258375890552, "learning_rate": 3.54207707666076e-06, "loss": 0.0009, "num_tokens": 56207772.0, "reward": -0.125, "reward_std": 0.2886738538742065, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.2886738657951355, "sampling/importance_sampling_ratio/max": 1.30220023393631, "sampling/importance_sampling_ratio/mean": 1.0007225215435027, "sampling/importance_sampling_ratio/min": 0.7616489440202713, "sampling/sampling_logp_difference/max": 0.3328009366989136, "sampling/sampling_logp_difference/mean": 0.02219512779265642, "step": 12890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.4, "completions/max_terminated_length": 287.4, "completions/mean_length": 194.675, "completions/mean_terminated_length": 194.675, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.1202234588563442, "epoch": 1.1859887836719682, "frac_reward_zero_std": 0.6, "grad_norm": 0.12449105083942413, "kl": 0.010085426864679902, "learning_rate": 3.5397735178497912e-06, "loss": 0.0009, "num_tokens": 56250714.0, "reward": -0.55, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.315497922897339, "sampling/importance_sampling_ratio/mean": 1.0006853222846985, "sampling/importance_sampling_ratio/min": 0.7857313215732574, "sampling/sampling_logp_difference/max": 0.291825670003891, "sampling/sampling_logp_difference/mean": 0.021933120675385, "step": 12900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 209.3875, "completions/mean_terminated_length": 209.3875, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.0882079489529133, "epoch": 1.1869081548221017, "frac_reward_zero_std": 0.5, "grad_norm": 5.1212663650512695, "kl": 0.012128527427557855, "learning_rate": 3.5374688911080716e-06, "loss": 0.001, "num_tokens": 56295025.0, "reward": -0.325, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2436386585235595, "sampling/importance_sampling_ratio/mean": 0.9997487843036652, "sampling/importance_sampling_ratio/min": 0.7919215559959412, "sampling/sampling_logp_difference/max": 0.24786479473114015, "sampling/sampling_logp_difference/mean": 0.021630967408418654, "step": 12910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.5, "completions/max_terminated_length": 308.5, "completions/mean_length": 220.5, "completions/mean_terminated_length": 220.5, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0179756596684455, "epoch": 1.187827525972235, "frac_reward_zero_std": 0.6, "grad_norm": 5.730569362640381, "kl": 0.013060616853181273, "learning_rate": 3.535163198802638e-06, "loss": 0.0005, "num_tokens": 56338025.0, "reward": -0.425, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.3406694531440735, "sampling/importance_sampling_ratio/mean": 0.9998735010623931, "sampling/importance_sampling_ratio/min": 0.7653741598129272, "sampling/sampling_logp_difference/max": 0.33348612785339354, "sampling/sampling_logp_difference/mean": 0.020756789855659007, "step": 12920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.0, "completions/max_terminated_length": 307.0, "completions/mean_length": 213.1625, "completions/mean_terminated_length": 213.1625, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1193585865199567, "epoch": 1.1887468971223683, "frac_reward_zero_std": 0.4, "grad_norm": 5.6087470054626465, "kl": 0.011479452016646973, "learning_rate": 3.53285644330162e-06, "loss": 0.0006, "num_tokens": 56384598.0, "reward": -0.18125, "reward_std": 0.49661222100257874, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.49661223888397216, "sampling/importance_sampling_ratio/max": 1.2675758600234985, "sampling/importance_sampling_ratio/mean": 0.9999707221984864, "sampling/importance_sampling_ratio/min": 0.7794164597988129, "sampling/sampling_logp_difference/max": 0.26604504585266114, "sampling/sampling_logp_difference/mean": 0.022023872844874857, "step": 12930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.1, "completions/max_terminated_length": 294.1, "completions/mean_length": 208.925, "completions/mean_terminated_length": 208.925, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0842120319604873, "epoch": 1.1896662682725017, "frac_reward_zero_std": 0.5, "grad_norm": 0.08173853904008865, "kl": 0.009703094451106153, "learning_rate": 3.5305486269742415e-06, "loss": 0.0003, "num_tokens": 56428040.0, "reward": 0.425, "reward_std": 0.49153932929039, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.49153935313224795, "sampling/importance_sampling_ratio/max": 1.2821467518806458, "sampling/importance_sampling_ratio/mean": 0.9998727977275849, "sampling/importance_sampling_ratio/min": 0.7930573582649231, "sampling/sampling_logp_difference/max": 0.2691413402557373, "sampling/sampling_logp_difference/mean": 0.021701275184750558, "step": 12940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 342.5, "completions/max_terminated_length": 336.3, "completions/mean_length": 249.2875, "completions/mean_terminated_length": 242.2, "completions/min_length": 154.8, "completions/min_terminated_length": 154.8, "entropy": 1.1467635199427604, "epoch": 1.1905856394226348, "frac_reward_zero_std": 0.3, "grad_norm": 5.701486110687256, "kl": 0.013500099704833702, "learning_rate": 3.528239752190815e-06, "loss": 0.0008, "num_tokens": 56477183.0, "reward": -0.25, "reward_std": 0.6143951833248138, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6143951833248138, "sampling/importance_sampling_ratio/max": 1.2518826484680177, "sampling/importance_sampling_ratio/mean": 0.9997780978679657, "sampling/importance_sampling_ratio/min": 0.7861500382423401, "sampling/sampling_logp_difference/max": 0.2730622529983521, "sampling/sampling_logp_difference/mean": 0.022403847239911557, "step": 12950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 237.1, "completions/max_terminated_length": 237.1, "completions/mean_length": 178.1, "completions/mean_terminated_length": 178.1, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 0.9781753465533256, "epoch": 1.1915050105727683, "frac_reward_zero_std": 0.9, "grad_norm": 0.08748628944158554, "kl": 0.010804156010271982, "learning_rate": 3.5259298213227387e-06, "loss": 0.0006, "num_tokens": 56517559.0, "reward": 0.5, "reward_std": 0.10690449476242066, "rewards/reward_fn/mean": 0.5, "rewards/reward_fn/std": 0.10690449476242066, "sampling/importance_sampling_ratio/max": 1.2581507563591003, "sampling/importance_sampling_ratio/mean": 0.9998533844947814, "sampling/importance_sampling_ratio/min": 0.7812200903892517, "sampling/sampling_logp_difference/max": 0.2647660970687866, "sampling/sampling_logp_difference/mean": 0.021105890162289143, "step": 12960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.7, "completions/max_terminated_length": 298.7, "completions/mean_length": 210.4875, "completions/mean_terminated_length": 210.4875, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0590761229395866, "epoch": 1.1924243817229014, "frac_reward_zero_std": 0.5, "grad_norm": 0.12348658591508865, "kl": 0.011865573294926435, "learning_rate": 3.5236188367424982e-06, "loss": 0.0008, "num_tokens": 56558606.0, "reward": -0.15, "reward_std": 0.4847656965255737, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.48476572036743165, "sampling/importance_sampling_ratio/max": 1.30410293340683, "sampling/importance_sampling_ratio/mean": 1.0003032565116883, "sampling/importance_sampling_ratio/min": 0.7627254247665405, "sampling/sampling_logp_difference/max": 0.30228362083435056, "sampling/sampling_logp_difference/mean": 0.021882767789065838, "step": 12970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.1, "completions/max_terminated_length": 295.6, "completions/mean_length": 211.125, "completions/mean_terminated_length": 208.3589294433594, "completions/min_length": 152.6, "completions/min_terminated_length": 152.6, "entropy": 1.0660047613084316, "epoch": 1.1933437528730348, "frac_reward_zero_std": 0.3, "grad_norm": 5.614049434661865, "kl": 0.011804240354103967, "learning_rate": 3.5213068008236597e-06, "loss": 0.0001, "num_tokens": 56604704.0, "reward": -0.13125, "reward_std": 0.5403548881411553, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.5403549000620842, "sampling/importance_sampling_ratio/max": 1.316912090778351, "sampling/importance_sampling_ratio/mean": 0.9999988436698913, "sampling/importance_sampling_ratio/min": 0.7092660427093506, "sampling/sampling_logp_difference/max": 0.3903221607208252, "sampling/sampling_logp_difference/mean": 0.021143856458365917, "step": 12980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.7, "completions/max_terminated_length": 300.7, "completions/mean_length": 206.35, "completions/mean_terminated_length": 206.35, "completions/min_length": 126.3, "completions/min_terminated_length": 126.3, "entropy": 1.0811785131692886, "epoch": 1.194263124023168, "frac_reward_zero_std": 0.7, "grad_norm": 0.1149553507566452, "kl": 0.012454828730551527, "learning_rate": 3.5189937159408684e-06, "loss": 0.0006, "num_tokens": 56648612.0, "reward": -0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2545925378799438, "sampling/importance_sampling_ratio/mean": 0.9996000230312347, "sampling/importance_sampling_ratio/min": 0.7880428671836853, "sampling/sampling_logp_difference/max": 0.24327645301818848, "sampling/sampling_logp_difference/mean": 0.02138494700193405, "step": 12990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.1, "completions/max_terminated_length": 303.1, "completions/mean_length": 205.1875, "completions/mean_terminated_length": 205.1875, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.0962326496839523, "epoch": 1.1951824951733014, "frac_reward_zero_std": 0.4, "grad_norm": 7.641502857208252, "kl": 0.011787807793007232, "learning_rate": 3.516679584469848e-06, "loss": 0.0007, "num_tokens": 56691491.0, "reward": 0.075, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2532150864601135, "sampling/importance_sampling_ratio/mean": 1.0003086566925048, "sampling/importance_sampling_ratio/min": 0.7796613872051239, "sampling/sampling_logp_difference/max": 0.27288773059844973, "sampling/sampling_logp_difference/mean": 0.02160818502306938, "step": 13000 }, { "epoch": 1.1951824951733014, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 304.08726003490403, "eval_completions/max_terminated_length": 301.52705061082025, "eval_completions/mean_length": 212.64725130890054, "eval_completions/mean_terminated_length": 211.93098172758678, "eval_completions/min_length": 139.38219895287958, "eval_completions/min_terminated_length": 139.38219895287958, "eval_entropy": 1.0961968853003483, "eval_frac_reward_zero_std": 0.4816753926701571, "eval_kl": 0.011702674543436285, "eval_loss": 0.0005838062497787178, "eval_num_tokens": 56691491.0, "eval_reward": -0.11202006980802792, "eval_reward_std": 0.45468040770261076, "eval_rewards/reward_fn/mean": -0.11202006980802792, "eval_rewards/reward_fn/std": 0.45468042309787676, "eval_runtime": 1765.4962, "eval_samples_per_second": 0.325, "eval_sampling/importance_sampling_ratio/max": 1.2708042381529616, "eval_sampling/importance_sampling_ratio/mean": 1.0000330085945794, "eval_sampling/importance_sampling_ratio/min": 0.775110278480132, "eval_sampling/sampling_logp_difference/max": 0.28805416934257194, "eval_sampling/sampling_logp_difference/mean": 0.02112978326894642, "eval_steps_per_second": 0.041, "step": 13000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.1, "completions/max_terminated_length": 305.1, "completions/mean_length": 200.8875, "completions/mean_terminated_length": 200.8875, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.0016167685389519, "epoch": 1.1961018663234348, "frac_reward_zero_std": 0.4, "grad_norm": 0.08278321474790573, "kl": 0.013985663116909564, "learning_rate": 3.514364408787398e-06, "loss": 0.0006, "num_tokens": 56732394.0, "reward": 0.26875, "reward_std": 0.5133711859583855, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.5133712217211723, "sampling/importance_sampling_ratio/max": 1.3263503432273864, "sampling/importance_sampling_ratio/mean": 1.0001817286014556, "sampling/importance_sampling_ratio/min": 0.7698430120944977, "sampling/sampling_logp_difference/max": 0.30820298194885254, "sampling/sampling_logp_difference/mean": 0.02068722452968359, "step": 13010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 218.3375, "completions/mean_terminated_length": 218.3375, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.102985506504774, "epoch": 1.197021237473568, "frac_reward_zero_std": 0.3, "grad_norm": 5.40641975402832, "kl": 0.011007527337642386, "learning_rate": 3.512048191271389e-06, "loss": 0.0006, "num_tokens": 56777269.0, "reward": 0.125, "reward_std": 0.6480583846569061, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6480584084987641, "sampling/importance_sampling_ratio/max": 1.3089889645576478, "sampling/importance_sampling_ratio/mean": 0.999907809495926, "sampling/importance_sampling_ratio/min": 0.7883631944656372, "sampling/sampling_logp_difference/max": 0.2883092284202576, "sampling/sampling_logp_difference/mean": 0.02093223575502634, "step": 13020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.8, "completions/max_terminated_length": 280.8, "completions/mean_length": 192.15, "completions/mean_terminated_length": 192.15, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.072444386780262, "epoch": 1.1979406086237014, "frac_reward_zero_std": 0.2, "grad_norm": 7.725576877593994, "kl": 0.012364886392606422, "learning_rate": 3.5097309343007617e-06, "loss": 0.0006, "num_tokens": 56820169.0, "reward": -0.175, "reward_std": 0.7187533676624298, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.7187534153461457, "sampling/importance_sampling_ratio/max": 1.3140068531036377, "sampling/importance_sampling_ratio/mean": 1.0000372648239135, "sampling/importance_sampling_ratio/min": 0.7942931711673736, "sampling/sampling_logp_difference/max": 0.27501233816146853, "sampling/sampling_logp_difference/mean": 0.021729365177452564, "step": 13030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.2, "completions/max_terminated_length": 297.2, "completions/mean_length": 205.4125, "completions/mean_terminated_length": 205.4125, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0618013143539429, "epoch": 1.1988599797738346, "frac_reward_zero_std": 0.8, "grad_norm": 0.1087055578827858, "kl": 0.011147426371462644, "learning_rate": 3.5074126402555246e-06, "loss": 0.0005, "num_tokens": 56862514.0, "reward": -0.325, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.2976880550384522, "sampling/importance_sampling_ratio/mean": 0.9999469220638275, "sampling/importance_sampling_ratio/min": 0.7667073965072632, "sampling/sampling_logp_difference/max": 0.31477442681789397, "sampling/sampling_logp_difference/mean": 0.021407883055508135, "step": 13040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 184.0125, "completions/mean_terminated_length": 184.0125, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0156176008284092, "epoch": 1.199779350923968, "frac_reward_zero_std": 0.5, "grad_norm": 0.1507747620344162, "kl": 0.010279237711802124, "learning_rate": 3.505093311516753e-06, "loss": 0.0003, "num_tokens": 56902651.0, "reward": 0.125, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2408055424690247, "sampling/importance_sampling_ratio/mean": 1.0001389622688293, "sampling/importance_sampling_ratio/min": 0.7777035593986511, "sampling/sampling_logp_difference/max": 0.25713381767272947, "sampling/sampling_logp_difference/mean": 0.021222737058997153, "step": 13050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 192.6625, "completions/mean_terminated_length": 192.6625, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 1.1020600371062756, "epoch": 1.2006987220741012, "frac_reward_zero_std": 0.5, "grad_norm": 0.11264205724000931, "kl": 0.012825869489461184, "learning_rate": 3.502772950466582e-06, "loss": 0.0007, "num_tokens": 56944400.0, "reward": -0.25, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2903294324874879, "sampling/importance_sampling_ratio/mean": 0.9997969508171082, "sampling/importance_sampling_ratio/min": 0.752705180644989, "sampling/sampling_logp_difference/max": 0.3029646396636963, "sampling/sampling_logp_difference/mean": 0.02314635906368494, "step": 13060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.6, "completions/max_terminated_length": 286.6, "completions/mean_length": 204.25, "completions/mean_terminated_length": 204.25, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.045848973840475, "epoch": 1.2016180932242346, "frac_reward_zero_std": 0.4, "grad_norm": 5.664388179779053, "kl": 0.010946528418571688, "learning_rate": 3.500451559488208e-06, "loss": 0.0008, "num_tokens": 56989940.0, "reward": -0.175, "reward_std": 0.5841213405132294, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5841213643550873, "sampling/importance_sampling_ratio/max": 1.271582794189453, "sampling/importance_sampling_ratio/mean": 0.9997782111167908, "sampling/importance_sampling_ratio/min": 0.7790528357028961, "sampling/sampling_logp_difference/max": 0.28257017135620116, "sampling/sampling_logp_difference/mean": 0.02207141313701868, "step": 13070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.8, "completions/max_terminated_length": 278.8, "completions/mean_length": 197.7375, "completions/mean_terminated_length": 197.7375, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 0.9965252969413996, "epoch": 1.2025374643743678, "frac_reward_zero_std": 0.7, "grad_norm": 0.12151244282722473, "kl": 0.010050613412749954, "learning_rate": 3.498129140965888e-06, "loss": 0.0002, "num_tokens": 57032791.0, "reward": 0.1, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.3676928639411927, "sampling/importance_sampling_ratio/mean": 1.000283819437027, "sampling/importance_sampling_ratio/min": 0.7643963515758514, "sampling/sampling_logp_difference/max": 0.3400022268295288, "sampling/sampling_logp_difference/mean": 0.021105833165347577, "step": 13080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 196.3, "completions/mean_terminated_length": 196.3, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0339159116148948, "epoch": 1.2034568355245012, "frac_reward_zero_std": 0.7, "grad_norm": 3.891164779663086, "kl": 0.011045815038960427, "learning_rate": 3.495805697284929e-06, "loss": 0.0007, "num_tokens": 57076463.0, "reward": 0.15, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.276653003692627, "sampling/importance_sampling_ratio/mean": 1.0007806658744811, "sampling/importance_sampling_ratio/min": 0.7947619318962097, "sampling/sampling_logp_difference/max": 0.25692063570022583, "sampling/sampling_logp_difference/mean": 0.02087047304958105, "step": 13090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.7, "completions/max_terminated_length": 319.7, "completions/mean_length": 227.775, "completions/mean_terminated_length": 227.775, "completions/min_length": 152.8, "completions/min_terminated_length": 152.8, "entropy": 1.1305967919528483, "epoch": 1.2043762066746346, "frac_reward_zero_std": 0.4, "grad_norm": 0.1049635037779808, "kl": 0.013015161006478593, "learning_rate": 3.4934812308316957e-06, "loss": 0.0004, "num_tokens": 57120893.0, "reward": -0.075, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.35944344997406, "sampling/importance_sampling_ratio/mean": 0.999826318025589, "sampling/importance_sampling_ratio/min": 0.7605683267116546, "sampling/sampling_logp_difference/max": 0.3489307641983032, "sampling/sampling_logp_difference/mean": 0.022713737189769746, "step": 13100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 208.85, "completions/mean_terminated_length": 208.85, "completions/min_length": 141.4, "completions/min_terminated_length": 141.4, "entropy": 1.040209709852934, "epoch": 1.2052955778247678, "frac_reward_zero_std": 0.2, "grad_norm": 5.148966312408447, "kl": 0.01040358521277085, "learning_rate": 3.491155743993601e-06, "loss": 0.0007, "num_tokens": 57165745.0, "reward": -0.10625, "reward_std": 0.7583023726940155, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.7583024263381958, "sampling/importance_sampling_ratio/max": 1.2545533061027527, "sampling/importance_sampling_ratio/mean": 1.0001945614814758, "sampling/importance_sampling_ratio/min": 0.746969872713089, "sampling/sampling_logp_difference/max": 0.3132442355155945, "sampling/sampling_logp_difference/mean": 0.0213812455534935, "step": 13110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 234.4, "completions/max_terminated_length": 234.4, "completions/mean_length": 175.55, "completions/mean_terminated_length": 175.55, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0477112270891666, "epoch": 1.2062149489749012, "frac_reward_zero_std": 0.7, "grad_norm": 8.5221586227417, "kl": 0.011736217365250923, "learning_rate": 3.4888292391591065e-06, "loss": 0.0007, "num_tokens": 57202885.0, "reward": -0.175, "reward_std": 0.30299633741378784, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3029963493347168, "sampling/importance_sampling_ratio/max": 1.2447654366493226, "sampling/importance_sampling_ratio/mean": 1.000334495306015, "sampling/importance_sampling_ratio/min": 0.7442826807498932, "sampling/sampling_logp_difference/max": 0.3187698483467102, "sampling/sampling_logp_difference/mean": 0.021178030781447886, "step": 13120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.9, "completions/max_terminated_length": 308.9, "completions/mean_length": 219.1, "completions/mean_terminated_length": 219.1, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.1049752525985241, "epoch": 1.2071343201250344, "frac_reward_zero_std": 0.3, "grad_norm": 6.1880974769592285, "kl": 0.013186289492296055, "learning_rate": 3.486501718717719e-06, "loss": 0.0009, "num_tokens": 57245949.0, "reward": -0.375, "reward_std": 0.6001615285873413, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.6001615405082703, "sampling/importance_sampling_ratio/max": 1.3721838355064393, "sampling/importance_sampling_ratio/mean": 0.9995830535888672, "sampling/importance_sampling_ratio/min": 0.7415451735258103, "sampling/sampling_logp_difference/max": 0.35662396550178527, "sampling/sampling_logp_difference/mean": 0.022245214134454728, "step": 13130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 206.825, "completions/mean_terminated_length": 206.825, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.108198868483305, "epoch": 1.2080536912751678, "frac_reward_zero_std": 0.4, "grad_norm": 6.436650276184082, "kl": 0.011979929107474164, "learning_rate": 3.484173185059989e-06, "loss": 0.0006, "num_tokens": 57296303.0, "reward": -0.275, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.3819093346595763, "sampling/importance_sampling_ratio/mean": 0.9997508287429809, "sampling/importance_sampling_ratio/min": 0.7807163894176483, "sampling/sampling_logp_difference/max": 0.34348788261413576, "sampling/sampling_logp_difference/mean": 0.021449166908860208, "step": 13140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.7, "completions/max_terminated_length": 283.7, "completions/mean_length": 200.3125, "completions/mean_terminated_length": 200.3125, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.9558900523930788, "epoch": 1.208973062425301, "frac_reward_zero_std": 0.5, "grad_norm": 20.703554153442383, "kl": 0.025188092995085752, "learning_rate": 3.481843640577506e-06, "loss": 0.0012, "num_tokens": 57335520.0, "reward": -0.075, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2403390407562256, "sampling/importance_sampling_ratio/mean": 1.0004243314266206, "sampling/importance_sampling_ratio/min": 0.7680588722229004, "sampling/sampling_logp_difference/max": 0.2706081449985504, "sampling/sampling_logp_difference/mean": 0.020155103597790002, "step": 13150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.4, "completions/max_terminated_length": 249.4, "completions/mean_length": 177.95, "completions/mean_terminated_length": 177.95, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 0.982709313929081, "epoch": 1.2098924335754344, "frac_reward_zero_std": 0.6, "grad_norm": 0.1335449069738388, "kl": 0.011607114344951697, "learning_rate": 3.4795130876629003e-06, "loss": 0.0005, "num_tokens": 57373012.0, "reward": -0.40625, "reward_std": 0.34549421072006226, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.3454942166805267, "sampling/importance_sampling_ratio/max": 1.292046344280243, "sampling/importance_sampling_ratio/mean": 1.0002207159996033, "sampling/importance_sampling_ratio/min": 0.7770138621330261, "sampling/sampling_logp_difference/max": 0.2804917573928833, "sampling/sampling_logp_difference/mean": 0.02112022750079632, "step": 13160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 360.9, "completions/max_terminated_length": 360.9, "completions/mean_length": 239.125, "completions/mean_terminated_length": 239.125, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.118070115149021, "epoch": 1.2108118047255676, "frac_reward_zero_std": 0.2, "grad_norm": 0.1750139743089676, "kl": 0.01285756605793722, "learning_rate": 3.477181528709836e-06, "loss": 0.0004, "num_tokens": 57420950.0, "reward": -0.325, "reward_std": 0.6968977272510528, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.6968977510929107, "sampling/importance_sampling_ratio/max": 1.2901989340782165, "sampling/importance_sampling_ratio/mean": 0.9995056748390198, "sampling/importance_sampling_ratio/min": 0.7425417184829712, "sampling/sampling_logp_difference/max": 0.3121076762676239, "sampling/sampling_logp_difference/mean": 0.022666409611701965, "step": 13170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 192.8375, "completions/mean_terminated_length": 192.8375, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.0132109191268683, "epoch": 1.211731175875701, "frac_reward_zero_std": 0.5, "grad_norm": 5.872958183288574, "kl": 0.011774955177679658, "learning_rate": 3.4748489661130104e-06, "loss": 0.0004, "num_tokens": 57462185.0, "reward": -0.275, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.329495084285736, "sampling/importance_sampling_ratio/mean": 1.0004606604576112, "sampling/importance_sampling_ratio/min": 0.7833117485046387, "sampling/sampling_logp_difference/max": 0.29913721680641175, "sampling/sampling_logp_difference/mean": 0.021312739141285418, "step": 13180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.4, "completions/max_terminated_length": 296.4, "completions/mean_length": 203.3125, "completions/mean_terminated_length": 203.3125, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0852484077215194, "epoch": 1.2126505470258344, "frac_reward_zero_std": 0.3, "grad_norm": 6.667899131774902, "kl": 0.011438735766569152, "learning_rate": 3.472515402268154e-06, "loss": 0.0008, "num_tokens": 57508834.0, "reward": 0.075, "reward_std": 0.6001615285873413, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.6001615405082703, "sampling/importance_sampling_ratio/max": 1.2599624872207642, "sampling/importance_sampling_ratio/mean": 0.9996525287628174, "sampling/importance_sampling_ratio/min": 0.7511632382869721, "sampling/sampling_logp_difference/max": 0.2918216049671173, "sampling/sampling_logp_difference/mean": 0.021917281672358513, "step": 13190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 204.45, "completions/mean_terminated_length": 204.45, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.0530879355967044, "epoch": 1.2135699181759676, "frac_reward_zero_std": 0.5, "grad_norm": 6.173785209655762, "kl": 0.012459032266633585, "learning_rate": 3.4701808395720216e-06, "loss": 0.0006, "num_tokens": 57550982.0, "reward": 0.075, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2438038825988769, "sampling/importance_sampling_ratio/mean": 0.9997240781784058, "sampling/importance_sampling_ratio/min": 0.738278841972351, "sampling/sampling_logp_difference/max": 0.32518017292022705, "sampling/sampling_logp_difference/mean": 0.02176930885761976, "step": 13200 }, { "epoch": 1.2135699181759676, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 298.44677137870855, "eval_completions/max_terminated_length": 295.63176265270505, "eval_completions/mean_length": 205.3913612565445, "eval_completions/mean_terminated_length": 204.70501972617905, "eval_completions/min_length": 132.72949389179755, "eval_completions/min_terminated_length": 132.72949389179755, "eval_entropy": 1.079767266129116, "eval_frac_reward_zero_std": 0.5043630017452007, "eval_kl": 0.012056015344112332, "eval_loss": 0.0006001430447213352, "eval_num_tokens": 57550982.0, "eval_reward": -0.0968586387434555, "eval_reward_std": 0.4368554967964299, "eval_rewards/reward_fn/mean": -0.0968586387434555, "eval_rewards/reward_fn/std": 0.4368555109434311, "eval_runtime": 1734.7978, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2762546196033817, "eval_sampling/importance_sampling_ratio/mean": 1.0000005440354138, "eval_sampling/importance_sampling_ratio/min": 0.7614033637246537, "eval_sampling/sampling_logp_difference/max": 0.29976442991959074, "eval_sampling/sampling_logp_difference/mean": 0.021757649205768087, "eval_steps_per_second": 0.042, "step": 13200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.4, "completions/max_terminated_length": 305.4, "completions/mean_length": 211.9, "completions/mean_terminated_length": 211.9, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0292070254683494, "epoch": 1.214489289326101, "frac_reward_zero_std": 0.6, "grad_norm": 0.12270359694957733, "kl": 0.012007759406697006, "learning_rate": 3.4678452804223984e-06, "loss": 0.0007, "num_tokens": 57594486.0, "reward": -0.3, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.3526293873786925, "sampling/importance_sampling_ratio/mean": 1.0002144038677216, "sampling/importance_sampling_ratio/min": 0.7545961320400238, "sampling/sampling_logp_difference/max": 0.36925657987594607, "sampling/sampling_logp_difference/mean": 0.021208292245864867, "step": 13210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.7, "completions/max_terminated_length": 317.7, "completions/mean_length": 210.225, "completions/mean_terminated_length": 210.225, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0304672364145517, "epoch": 1.2154086604762342, "frac_reward_zero_std": 0.5, "grad_norm": 0.13093261420726776, "kl": 0.012857493251794949, "learning_rate": 3.4655087272180895e-06, "loss": 0.0001, "num_tokens": 57639416.0, "reward": -0.05, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2917356610298156, "sampling/importance_sampling_ratio/mean": 0.9997867286205292, "sampling/importance_sampling_ratio/min": 0.7877563297748565, "sampling/sampling_logp_difference/max": 0.2684637069702148, "sampling/sampling_logp_difference/mean": 0.021774257905781268, "step": 13220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 196.225, "completions/mean_terminated_length": 196.225, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 0.975885608792305, "epoch": 1.2163280316263676, "frac_reward_zero_std": 0.8, "grad_norm": 4.782840251922607, "kl": 0.01126242927275598, "learning_rate": 3.4631711823589226e-06, "loss": 0.0005, "num_tokens": 57682154.0, "reward": -0.275, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.237126874923706, "sampling/importance_sampling_ratio/mean": 0.9995870053768158, "sampling/importance_sampling_ratio/min": 0.7813907623291015, "sampling/sampling_logp_difference/max": 0.25594363212585447, "sampling/sampling_logp_difference/mean": 0.020732624270021916, "step": 13230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.3, "completions/max_terminated_length": 325.3, "completions/mean_length": 231.45, "completions/mean_terminated_length": 231.45, "completions/min_length": 165.5, "completions/min_terminated_length": 165.5, "entropy": 1.0550205804407597, "epoch": 1.2172474027765008, "frac_reward_zero_std": 0.4, "grad_norm": 6.401443004608154, "kl": 0.011286059085978194, "learning_rate": 3.460832648245744e-06, "loss": 0.0011, "num_tokens": 57727614.0, "reward": -0.325, "reward_std": 0.5336207330226899, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5336207330226899, "sampling/importance_sampling_ratio/max": 1.2858677625656127, "sampling/importance_sampling_ratio/mean": 1.0007679164409637, "sampling/importance_sampling_ratio/min": 0.772913521528244, "sampling/sampling_logp_difference/max": 0.2799473166465759, "sampling/sampling_logp_difference/mean": 0.02116876430809498, "step": 13240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.1, "completions/max_terminated_length": 322.1, "completions/mean_length": 218.3, "completions/mean_terminated_length": 215.02142944335938, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.1041935201734305, "epoch": 1.2181667739266342, "frac_reward_zero_std": 0.1, "grad_norm": 6.699099540710449, "kl": 0.011735542357200757, "learning_rate": 3.4584931272804155e-06, "loss": 0.0007, "num_tokens": 57773902.0, "reward": -0.26875, "reward_std": 0.7228082284331322, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.7228082582354546, "sampling/importance_sampling_ratio/max": 1.2978209495544433, "sampling/importance_sampling_ratio/mean": 1.0001315414905547, "sampling/importance_sampling_ratio/min": 0.7779477655887603, "sampling/sampling_logp_difference/max": 0.29808523058891295, "sampling/sampling_logp_difference/mean": 0.022298302315175534, "step": 13250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.9, "completions/max_terminated_length": 281.9, "completions/mean_length": 194.2125, "completions/mean_terminated_length": 194.2125, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0285296067595482, "epoch": 1.2190861450767674, "frac_reward_zero_std": 0.5, "grad_norm": 5.721333026885986, "kl": 0.011141916841734201, "learning_rate": 3.4561526218658125e-06, "loss": -0.0001, "num_tokens": 57814159.0, "reward": 0.025, "reward_std": 0.5024828433990478, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5024828553199768, "sampling/importance_sampling_ratio/max": 1.3349933743476867, "sampling/importance_sampling_ratio/mean": 1.000074326992035, "sampling/importance_sampling_ratio/min": 0.7567422389984131, "sampling/sampling_logp_difference/max": 0.3109550356864929, "sampling/sampling_logp_difference/mean": 0.021602443791925906, "step": 13260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.0, "completions/max_terminated_length": 276.0, "completions/mean_length": 196.3125, "completions/mean_terminated_length": 196.3125, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0328598964959383, "epoch": 1.2200055162269008, "frac_reward_zero_std": 0.4, "grad_norm": 0.2005244344472885, "kl": 0.01288128348533064, "learning_rate": 3.4538111344058218e-06, "loss": 0.0003, "num_tokens": 57854664.0, "reward": -0.025, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.2651203393936157, "sampling/importance_sampling_ratio/mean": 0.9996863663196563, "sampling/importance_sampling_ratio/min": 0.6810633033514023, "sampling/sampling_logp_difference/max": 0.4577652752399445, "sampling/sampling_logp_difference/mean": 0.021481659449636935, "step": 13270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.5, "completions/max_terminated_length": 285.5, "completions/mean_length": 196.15, "completions/mean_terminated_length": 196.15, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0465112160891294, "epoch": 1.2209248873770342, "frac_reward_zero_std": 0.2, "grad_norm": 0.10290118306875229, "kl": 0.01103897588327527, "learning_rate": 3.4514686673053396e-06, "loss": 0.0006, "num_tokens": 57897332.0, "reward": -0.13125, "reward_std": 0.589194230735302, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.5891942426562309, "sampling/importance_sampling_ratio/max": 1.2773096680641174, "sampling/importance_sampling_ratio/mean": 0.9995383143424987, "sampling/importance_sampling_ratio/min": 0.7594920992851257, "sampling/sampling_logp_difference/max": 0.29829304218292235, "sampling/sampling_logp_difference/mean": 0.02172661982476711, "step": 13280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 211.0, "completions/mean_terminated_length": 211.0, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.060120564699173, "epoch": 1.2218442585271674, "frac_reward_zero_std": 0.8, "grad_norm": 0.10950567573308945, "kl": 0.010776519170030951, "learning_rate": 3.4491252229702648e-06, "loss": 0.0005, "num_tokens": 57940492.0, "reward": -0.725, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.725, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2704498171806335, "sampling/importance_sampling_ratio/mean": 0.9995988249778748, "sampling/importance_sampling_ratio/min": 0.7785082697868347, "sampling/sampling_logp_difference/max": 0.2773622155189514, "sampling/sampling_logp_difference/mean": 0.02191921081393957, "step": 13290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.1, "completions/max_terminated_length": 279.1, "completions/mean_length": 184.3625, "completions/mean_terminated_length": 184.3625, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 0.9745671775192022, "epoch": 1.2227636296773008, "frac_reward_zero_std": 0.5, "grad_norm": 6.778991222381592, "kl": 0.011408400890650228, "learning_rate": 3.446780803807506e-06, "loss": 0.0003, "num_tokens": 57981753.0, "reward": 0.15, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.2501779556274415, "sampling/importance_sampling_ratio/mean": 1.0007660806179046, "sampling/importance_sampling_ratio/min": 0.7878679990768432, "sampling/sampling_logp_difference/max": 0.2690280079841614, "sampling/sampling_logp_difference/mean": 0.020466745458543302, "step": 13300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 203.0875, "completions/mean_terminated_length": 203.0875, "completions/min_length": 115.2, "completions/min_terminated_length": 115.2, "entropy": 1.0526458732783794, "epoch": 1.223683000827434, "frac_reward_zero_std": 0.4, "grad_norm": 6.062877655029297, "kl": 0.011629727011313661, "learning_rate": 3.444435412224967e-06, "loss": 0.0005, "num_tokens": 58024208.0, "reward": 0.3, "reward_std": 0.5916701912879944, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.5916702151298523, "sampling/importance_sampling_ratio/max": 1.2757673978805542, "sampling/importance_sampling_ratio/mean": 1.0001576006412507, "sampling/importance_sampling_ratio/min": 0.751650047302246, "sampling/sampling_logp_difference/max": 0.30136586129665377, "sampling/sampling_logp_difference/mean": 0.021606842428445815, "step": 13310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.4, "completions/max_terminated_length": 292.4, "completions/mean_length": 203.5875, "completions/mean_terminated_length": 203.5875, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.0223957620561124, "epoch": 1.2246023719775674, "frac_reward_zero_std": 0.4, "grad_norm": 7.341691970825195, "kl": 0.011185273446608334, "learning_rate": 3.442089050631554e-06, "loss": 0.0006, "num_tokens": 58066239.0, "reward": 0.275, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.3123634457588196, "sampling/importance_sampling_ratio/mean": 1.000015425682068, "sampling/importance_sampling_ratio/min": 0.7610883772373199, "sampling/sampling_logp_difference/max": 0.3059302806854248, "sampling/sampling_logp_difference/mean": 0.02182705942541361, "step": 13320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.7, "completions/max_terminated_length": 323.7, "completions/mean_length": 214.9875, "completions/mean_terminated_length": 214.9875, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.1071106016635894, "epoch": 1.2255217431277006, "frac_reward_zero_std": 0.6, "grad_norm": 8.074707984924316, "kl": 0.012672591092996299, "learning_rate": 3.439741721437169e-06, "loss": 0.0005, "num_tokens": 58110486.0, "reward": -0.15, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2642085671424865, "sampling/importance_sampling_ratio/mean": 0.9999412894248962, "sampling/importance_sampling_ratio/min": 0.7671201825141907, "sampling/sampling_logp_difference/max": 0.27371405363082885, "sampling/sampling_logp_difference/mean": 0.02199447024613619, "step": 13330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.9, "completions/max_terminated_length": 284.9, "completions/mean_length": 217.7375, "completions/mean_terminated_length": 217.7375, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.0246843434870243, "epoch": 1.226441114277834, "frac_reward_zero_std": 0.5, "grad_norm": 0.14391978085041046, "kl": 0.011191445408621803, "learning_rate": 3.4373934270527052e-06, "loss": 0.0002, "num_tokens": 58157705.0, "reward": 0.125, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2513033509254456, "sampling/importance_sampling_ratio/mean": 1.000104296207428, "sampling/importance_sampling_ratio/min": 0.7426950871944428, "sampling/sampling_logp_difference/max": 0.31975862979888914, "sampling/sampling_logp_difference/mean": 0.020920468680560588, "step": 13340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.8, "completions/max_terminated_length": 304.8, "completions/mean_length": 210.7875, "completions/mean_terminated_length": 210.7875, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.1156291835010053, "epoch": 1.2273604854279672, "frac_reward_zero_std": 0.4, "grad_norm": 0.12212592363357544, "kl": 0.014824754209257662, "learning_rate": 3.435044169890052e-06, "loss": 0.0009, "num_tokens": 58202528.0, "reward": -0.15625, "reward_std": 0.54008709192276, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.540087103843689, "sampling/importance_sampling_ratio/max": 1.283536684513092, "sampling/importance_sampling_ratio/mean": 0.9998489916324615, "sampling/importance_sampling_ratio/min": 0.7914196431636811, "sampling/sampling_logp_difference/max": 0.26156499683856965, "sampling/sampling_logp_difference/mean": 0.02221636101603508, "step": 13350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.9, "completions/max_terminated_length": 306.9, "completions/mean_length": 214.575, "completions/mean_terminated_length": 214.575, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "entropy": 1.1335675314068794, "epoch": 1.2282798565781006, "frac_reward_zero_std": 0.3, "grad_norm": 7.017544269561768, "kl": 0.012479108147090301, "learning_rate": 3.4326939523620823e-06, "loss": 0.001, "num_tokens": 58245838.0, "reward": 0.175, "reward_std": 0.6186381995677948, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6186382234096527, "sampling/importance_sampling_ratio/max": 1.3060958743095399, "sampling/importance_sampling_ratio/mean": 1.0003341853618621, "sampling/importance_sampling_ratio/min": 0.7721905529499054, "sampling/sampling_logp_difference/max": 0.3011674404144287, "sampling/sampling_logp_difference/mean": 0.022261269949376584, "step": 13360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.7, "completions/max_terminated_length": 297.7, "completions/mean_length": 206.3375, "completions/mean_terminated_length": 206.3375, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1016542509198188, "epoch": 1.229199227728234, "frac_reward_zero_std": 0.4, "grad_norm": 0.09034109115600586, "kl": 0.011986213608179242, "learning_rate": 3.4303427768826593e-06, "loss": 0.0004, "num_tokens": 58288441.0, "reward": 0.225, "reward_std": 0.5984438240528107, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5984438478946685, "sampling/importance_sampling_ratio/max": 1.2849693655967713, "sampling/importance_sampling_ratio/mean": 0.9995818495750427, "sampling/importance_sampling_ratio/min": 0.7564325273036957, "sampling/sampling_logp_difference/max": 0.3111949980258942, "sampling/sampling_logp_difference/mean": 0.02148675434291363, "step": 13370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.2, "completions/max_terminated_length": 322.2, "completions/mean_length": 225.4375, "completions/mean_terminated_length": 225.4375, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.109952037408948, "epoch": 1.2301185988783672, "frac_reward_zero_std": 0.5, "grad_norm": 0.10746071487665176, "kl": 0.012367018021177501, "learning_rate": 3.4279906458666285e-06, "loss": 0.0007, "num_tokens": 58332924.0, "reward": 0.0, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2799421072006225, "sampling/importance_sampling_ratio/mean": 1.0001028954982758, "sampling/importance_sampling_ratio/min": 0.743025952577591, "sampling/sampling_logp_difference/max": 0.32220596075057983, "sampling/sampling_logp_difference/mean": 0.021914581768214703, "step": 13380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.8, "completions/max_terminated_length": 307.8, "completions/mean_length": 205.5375, "completions/mean_terminated_length": 205.5375, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0676206089556217, "epoch": 1.2310379700285006, "frac_reward_zero_std": 0.5, "grad_norm": 0.09487441182136536, "kl": 0.011361061441130004, "learning_rate": 3.425637561729817e-06, "loss": 0.0005, "num_tokens": 58372655.0, "reward": -0.075, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2423848390579224, "sampling/importance_sampling_ratio/mean": 0.9996363162994385, "sampling/importance_sampling_ratio/min": 0.7858622550964356, "sampling/sampling_logp_difference/max": 0.2508970081806183, "sampling/sampling_logp_difference/mean": 0.021972773410379887, "step": 13390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 210.675, "completions/mean_terminated_length": 210.675, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.0463161878287792, "epoch": 1.2319573411786338, "frac_reward_zero_std": 0.3, "grad_norm": 5.465270042419434, "kl": 0.010803931381087751, "learning_rate": 3.4232835268890298e-06, "loss": 0.0002, "num_tokens": 58412981.0, "reward": 0.5, "reward_std": 0.6371305644512176, "rewards/reward_fn/mean": 0.5, "rewards/reward_fn/std": 0.6371305763721467, "sampling/importance_sampling_ratio/max": 1.2586640000343323, "sampling/importance_sampling_ratio/mean": 0.9994501948356629, "sampling/importance_sampling_ratio/min": 0.7388533055782318, "sampling/sampling_logp_difference/max": 0.3152337551116943, "sampling/sampling_logp_difference/mean": 0.021609633602201937, "step": 13400 }, { "epoch": 1.2319573411786338, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 292.5671902268761, "eval_completions/max_terminated_length": 291.59511343804536, "eval_completions/mean_length": 205.5490837696335, "eval_completions/mean_terminated_length": 205.28786469837343, "eval_completions/min_length": 133.52705061082025, "eval_completions/min_terminated_length": 133.52705061082025, "eval_entropy": 1.0805754384341248, "eval_frac_reward_zero_std": 0.5095986038394416, "eval_kl": 0.011788596303233093, "eval_loss": 0.0005789236165583134, "eval_num_tokens": 58412981.0, "eval_reward": -0.10722076788830716, "eval_reward_std": 0.43119728963100473, "eval_rewards/reward_fn/mean": -0.10722076788830716, "eval_rewards/reward_fn/std": 0.4311973040900721, "eval_runtime": 1706.8654, "eval_samples_per_second": 0.336, "eval_sampling/importance_sampling_ratio/max": 1.272912514147334, "eval_sampling/importance_sampling_ratio/mean": 1.0000074297761834, "eval_sampling/importance_sampling_ratio/min": 0.7670406915026393, "eval_sampling/sampling_logp_difference/max": 0.29116689571119314, "eval_sampling/sampling_logp_difference/mean": 0.0214676700148855, "eval_steps_per_second": 0.042, "step": 13400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.9, "completions/max_terminated_length": 255.9, "completions/mean_length": 192.0875, "completions/mean_terminated_length": 192.0875, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.0937306866049767, "epoch": 1.2328767123287672, "frac_reward_zero_std": 0.3, "grad_norm": 5.4080586433410645, "kl": 0.012279455340467394, "learning_rate": 3.4209285437620514e-06, "loss": 0.0006, "num_tokens": 58452652.0, "reward": -0.125, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.250808584690094, "sampling/importance_sampling_ratio/mean": 1.000521993637085, "sampling/importance_sampling_ratio/min": 0.7713961780071259, "sampling/sampling_logp_difference/max": 0.2655188739299774, "sampling/sampling_logp_difference/mean": 0.02182045914232731, "step": 13410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.2, "completions/max_terminated_length": 298.2, "completions/mean_length": 207.9875, "completions/mean_terminated_length": 207.9875, "completions/min_length": 124.7, "completions/min_terminated_length": 124.7, "entropy": 1.000043273717165, "epoch": 1.2337960834789004, "frac_reward_zero_std": 0.4, "grad_norm": 5.738818168640137, "kl": 0.01311063887551427, "learning_rate": 3.418572614767637e-06, "loss": 0.0007, "num_tokens": 58496555.0, "reward": -0.025, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2564791321754456, "sampling/importance_sampling_ratio/mean": 0.9998393774032592, "sampling/importance_sampling_ratio/min": 0.7659902274608612, "sampling/sampling_logp_difference/max": 0.29012939929962156, "sampling/sampling_logp_difference/mean": 0.020610517263412474, "step": 13420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.2, "completions/max_terminated_length": 324.4, "completions/mean_length": 210.875, "completions/mean_terminated_length": 208.47142944335937, "completions/min_length": 114.1, "completions/min_terminated_length": 114.1, "entropy": 1.007377913966775, "epoch": 1.2347154546290338, "frac_reward_zero_std": 0.5, "grad_norm": 9.932769775390625, "kl": 0.014260686101624741, "learning_rate": 3.4162157423255136e-06, "loss": 0.0007, "num_tokens": 58537641.0, "reward": -0.3375, "reward_std": 0.4384414732456207, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.4384414851665497, "sampling/importance_sampling_ratio/max": 1.2892139554023743, "sampling/importance_sampling_ratio/mean": 0.999925720691681, "sampling/importance_sampling_ratio/min": 0.7383851110935211, "sampling/sampling_logp_difference/max": 0.329064679145813, "sampling/sampling_logp_difference/mean": 0.021169681567698717, "step": 13430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 301.1, "completions/max_terminated_length": 280.3, "completions/mean_length": 203.9125, "completions/mean_terminated_length": 200.1500015258789, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.0802625127136707, "epoch": 1.235634825779167, "frac_reward_zero_std": 0.4, "grad_norm": 0.13196736574172974, "kl": 0.011842112010344862, "learning_rate": 3.41385792885638e-06, "loss": 0.0008, "num_tokens": 58579866.0, "reward": 0.19375, "reward_std": 0.5328060388565063, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.5328060567378998, "sampling/importance_sampling_ratio/max": 1.2814983010292054, "sampling/importance_sampling_ratio/mean": 1.0000597655773162, "sampling/importance_sampling_ratio/min": 0.7771244764328002, "sampling/sampling_logp_difference/max": 0.2726619720458984, "sampling/sampling_logp_difference/mean": 0.021908232755959035, "step": 13440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 207.1, "completions/mean_terminated_length": 207.1, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.1097841635346413, "epoch": 1.2365541969293004, "frac_reward_zero_std": 0.4, "grad_norm": 5.372663497924805, "kl": 0.013776939746458083, "learning_rate": 3.4114991767818985e-06, "loss": 0.0006, "num_tokens": 58629338.0, "reward": 0.1, "reward_std": 0.5547168493270874, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5547168493270874, "sampling/importance_sampling_ratio/max": 1.261541211605072, "sampling/importance_sampling_ratio/mean": 0.9998250246047974, "sampling/importance_sampling_ratio/min": 0.7677361786365509, "sampling/sampling_logp_difference/max": 0.2685872673988342, "sampling/sampling_logp_difference/mean": 0.022268942184746267, "step": 13450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.4, "completions/max_terminated_length": 305.4, "completions/mean_length": 213.2625, "completions/mean_terminated_length": 213.2625, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.1089450806379317, "epoch": 1.2374735680794338, "frac_reward_zero_std": 0.5, "grad_norm": 0.09168729931116104, "kl": 0.012623401221935638, "learning_rate": 3.4091394885246965e-06, "loss": 0.001, "num_tokens": 58674463.0, "reward": -0.1, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.354660987854004, "sampling/importance_sampling_ratio/mean": 1.0003811240196228, "sampling/importance_sampling_ratio/min": 0.792609441280365, "sampling/sampling_logp_difference/max": 0.3269010603427887, "sampling/sampling_logp_difference/mean": 0.0221543088555336, "step": 13460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.9, "completions/max_terminated_length": 295.9, "completions/mean_length": 192.95, "completions/mean_terminated_length": 192.95, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.018105897307396, "epoch": 1.238392939229567, "frac_reward_zero_std": 0.6, "grad_norm": 0.13058315217494965, "kl": 0.014370241377037019, "learning_rate": 3.406778866508364e-06, "loss": 0.0007, "num_tokens": 58713667.0, "reward": 0.275, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2577629923820495, "sampling/importance_sampling_ratio/mean": 1.0000057101249695, "sampling/importance_sampling_ratio/min": 0.7380602836608887, "sampling/sampling_logp_difference/max": 0.3371522068977356, "sampling/sampling_logp_difference/mean": 0.021540416032075883, "step": 13470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 201.2, "completions/mean_terminated_length": 201.2, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.087583463639021, "epoch": 1.2393123103797004, "frac_reward_zero_std": 0.7, "grad_norm": 0.17346560955047607, "kl": 0.011513007606845349, "learning_rate": 3.4044173131574477e-06, "loss": 0.0006, "num_tokens": 58756827.0, "reward": -0.3, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.3261451482772828, "sampling/importance_sampling_ratio/mean": 1.0000224649906158, "sampling/importance_sampling_ratio/min": 0.7785903096199036, "sampling/sampling_logp_difference/max": 0.3472015857696533, "sampling/sampling_logp_difference/mean": 0.02191253434866667, "step": 13480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 214.5375, "completions/mean_terminated_length": 214.5375, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.1455257460474968, "epoch": 1.2402316815298335, "frac_reward_zero_std": 0.2, "grad_norm": 5.0957536697387695, "kl": 0.012164858914911748, "learning_rate": 3.4020548308974533e-06, "loss": 0.0008, "num_tokens": 58797822.0, "reward": -0.375, "reward_std": 0.7296968817710876, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.7296969175338746, "sampling/importance_sampling_ratio/max": 1.2944772601127625, "sampling/importance_sampling_ratio/mean": 0.9997569799423218, "sampling/importance_sampling_ratio/min": 0.7564833402633667, "sampling/sampling_logp_difference/max": 0.3230409622192383, "sampling/sampling_logp_difference/mean": 0.022704833559691905, "step": 13490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.5, "completions/max_terminated_length": 255.5, "completions/mean_length": 174.6875, "completions/mean_terminated_length": 174.6875, "completions/min_length": 107.4, "completions/min_terminated_length": 107.4, "entropy": 1.128016009181738, "epoch": 1.241151052679967, "frac_reward_zero_std": 0.8, "grad_norm": 6.797439098358154, "kl": 0.011718755977926775, "learning_rate": 3.3996914221548396e-06, "loss": 0.0008, "num_tokens": 58835957.0, "reward": 0.175, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.269640851020813, "sampling/importance_sampling_ratio/mean": 0.9996892511844635, "sampling/importance_sampling_ratio/min": 0.75516077876091, "sampling/sampling_logp_difference/max": 0.30136393308639525, "sampling/sampling_logp_difference/mean": 0.023690856620669366, "step": 13500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.0, "completions/max_terminated_length": 275.0, "completions/mean_length": 198.375, "completions/mean_terminated_length": 198.375, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.0335066631436347, "epoch": 1.2420704238301001, "frac_reward_zero_std": 0.6, "grad_norm": 5.632632732391357, "kl": 0.013707448579953051, "learning_rate": 3.397327089357015e-06, "loss": 0.001, "num_tokens": 58878483.0, "reward": -0.15, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.288282859325409, "sampling/importance_sampling_ratio/mean": 0.9999646842479706, "sampling/importance_sampling_ratio/min": 0.7841484546661377, "sampling/sampling_logp_difference/max": 0.2636910676956177, "sampling/sampling_logp_difference/mean": 0.021632946096360683, "step": 13510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.6, "completions/max_terminated_length": 324.6, "completions/mean_length": 223.35, "completions/mean_terminated_length": 223.35, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.122538446635008, "epoch": 1.2429897949802335, "frac_reward_zero_std": 0.6, "grad_norm": 0.08908328413963318, "kl": 0.014242640545126051, "learning_rate": 3.394961834932341e-06, "loss": 0.0005, "num_tokens": 58924639.0, "reward": 0.1, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.238014805316925, "sampling/importance_sampling_ratio/mean": 1.0001881420612335, "sampling/importance_sampling_ratio/min": 0.7788609325885772, "sampling/sampling_logp_difference/max": 0.25112074613571167, "sampling/sampling_logp_difference/mean": 0.021927749551832677, "step": 13520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 192.05, "completions/mean_terminated_length": 192.05, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 1.0893173068761826, "epoch": 1.2439091661303667, "frac_reward_zero_std": 0.8, "grad_norm": 0.14776189625263214, "kl": 0.011684635892743244, "learning_rate": 3.3925956613101222e-06, "loss": 0.0006, "num_tokens": 58964771.0, "reward": 0.15, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.240661871433258, "sampling/importance_sampling_ratio/mean": 0.9999294996261596, "sampling/importance_sampling_ratio/min": 0.757619434595108, "sampling/sampling_logp_difference/max": 0.289424341917038, "sampling/sampling_logp_difference/mean": 0.022797429747879506, "step": 13530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 202.5, "completions/mean_terminated_length": 202.5, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.153013388812542, "epoch": 1.2448285372805001, "frac_reward_zero_std": 0.3, "grad_norm": 6.062677383422852, "kl": 0.011064263148000464, "learning_rate": 3.390228570920609e-06, "loss": 0.0008, "num_tokens": 59009291.0, "reward": -0.10625, "reward_std": 0.6013532161712647, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6013532459735871, "sampling/importance_sampling_ratio/max": 1.2459774494171143, "sampling/importance_sampling_ratio/mean": 0.9994210660457611, "sampling/importance_sampling_ratio/min": 0.7523209631443024, "sampling/sampling_logp_difference/max": 0.29025859832763673, "sampling/sampling_logp_difference/mean": 0.023247643187642098, "step": 13540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 297.6, "completions/max_terminated_length": 292.1, "completions/mean_length": 212.0, "completions/mean_terminated_length": 209.43035888671875, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0032671704888343, "epoch": 1.2457479084306335, "frac_reward_zero_std": 0.5, "grad_norm": 0.12896759808063507, "kl": 0.01128785191103816, "learning_rate": 3.3878605661949926e-06, "loss": 0.0006, "num_tokens": 59051659.0, "reward": 0.16875, "reward_std": 0.5012218177318573, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.5012218296527863, "sampling/importance_sampling_ratio/max": 1.2464654088020324, "sampling/importance_sampling_ratio/mean": 0.9996826767921447, "sampling/importance_sampling_ratio/min": 0.7497195661067962, "sampling/sampling_logp_difference/max": 0.2967799246311188, "sampling/sampling_logp_difference/mean": 0.020724620670080185, "step": 13550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.9, "completions/max_terminated_length": 291.6, "completions/mean_length": 213.475, "completions/mean_terminated_length": 211.06785888671874, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.095436968654394, "epoch": 1.2466672795807667, "frac_reward_zero_std": 0.2, "grad_norm": 5.063879013061523, "kl": 0.011680925940163433, "learning_rate": 3.3854916495654044e-06, "loss": 0.0006, "num_tokens": 59098425.0, "reward": 0.24375, "reward_std": 0.8338168740272522, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.8338169097900391, "sampling/importance_sampling_ratio/max": 1.271662211418152, "sampling/importance_sampling_ratio/mean": 1.0003268480300904, "sampling/importance_sampling_ratio/min": 0.7453538835048675, "sampling/sampling_logp_difference/max": 0.32066367864608764, "sampling/sampling_logp_difference/mean": 0.021845745854079723, "step": 13560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.7, "completions/max_terminated_length": 314.7, "completions/mean_length": 221.825, "completions/mean_terminated_length": 221.825, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.128221009671688, "epoch": 1.2475866507309001, "frac_reward_zero_std": 0.5, "grad_norm": 6.132857799530029, "kl": 0.01223227196605876, "learning_rate": 3.383121823464911e-06, "loss": 0.0007, "num_tokens": 59142459.0, "reward": -0.15, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2397900938987731, "sampling/importance_sampling_ratio/mean": 0.9997495949268341, "sampling/importance_sampling_ratio/min": 0.7567213654518128, "sampling/sampling_logp_difference/max": 0.28461931347846986, "sampling/sampling_logp_difference/mean": 0.02241149786859751, "step": 13570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.7, "completions/max_terminated_length": 306.7, "completions/mean_length": 221.9625, "completions/mean_terminated_length": 221.9625, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.084916463494301, "epoch": 1.2485060218810333, "frac_reward_zero_std": 0.5, "grad_norm": 0.12524886429309845, "kl": 0.013065524731064215, "learning_rate": 3.3807510903275143e-06, "loss": 0.0005, "num_tokens": 59188496.0, "reward": 0.375, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.281197154521942, "sampling/importance_sampling_ratio/mean": 0.999637758731842, "sampling/importance_sampling_ratio/min": 0.7747971296310425, "sampling/sampling_logp_difference/max": 0.27586205005645753, "sampling/sampling_logp_difference/mean": 0.021579699032008647, "step": 13580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.9, "completions/max_terminated_length": 290.9, "completions/mean_length": 208.3875, "completions/mean_terminated_length": 208.3875, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.0837876476347446, "epoch": 1.2494253930311667, "frac_reward_zero_std": 0.2, "grad_norm": 7.161287784576416, "kl": 0.011995491379639133, "learning_rate": 3.3783794525881468e-06, "loss": -0.0, "num_tokens": 59230727.0, "reward": 0.275, "reward_std": 0.6968977272510528, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.6968977510929107, "sampling/importance_sampling_ratio/max": 1.237980353832245, "sampling/importance_sampling_ratio/mean": 0.9998725712299347, "sampling/importance_sampling_ratio/min": 0.7734759688377381, "sampling/sampling_logp_difference/max": 0.2728630483150482, "sampling/sampling_logp_difference/mean": 0.02246280312538147, "step": 13590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 202.375, "completions/mean_terminated_length": 202.375, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.0262967243790626, "epoch": 1.2503447641813, "frac_reward_zero_std": 0.7, "grad_norm": 4.768826007843018, "kl": 0.011890007427427917, "learning_rate": 3.3760069126826705e-06, "loss": 0.0007, "num_tokens": 59274157.0, "reward": -0.1, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.3098264932632446, "sampling/importance_sampling_ratio/mean": 0.9996854305267334, "sampling/importance_sampling_ratio/min": 0.7701345920562744, "sampling/sampling_logp_difference/max": 0.282219660282135, "sampling/sampling_logp_difference/mean": 0.021375838853418826, "step": 13600 }, { "epoch": 1.2503447641813, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 308.5567190226876, "eval_completions/max_terminated_length": 306.4659685863874, "eval_completions/mean_length": 215.75610820244327, "eval_completions/mean_terminated_length": 215.22581668341138, "eval_completions/min_length": 139.9476439790576, "eval_completions/min_terminated_length": 139.9476439790576, "eval_entropy": 1.0951936481614803, "eval_frac_reward_zero_std": 0.4712041884816754, "eval_kl": 0.011507800643707758, "eval_loss": 0.0005701505579054356, "eval_num_tokens": 59274157.0, "eval_reward": -0.11234729493891797, "eval_reward_std": 0.4677366456228193, "eval_rewards/reward_fn/mean": -0.11234729493891797, "eval_rewards/reward_fn/std": 0.46773666216232807, "eval_runtime": 1788.501, "eval_samples_per_second": 0.32, "eval_sampling/importance_sampling_ratio/max": 1.2744373124723452, "eval_sampling/importance_sampling_ratio/mean": 0.9999882406173161, "eval_sampling/importance_sampling_ratio/min": 0.7575482065036867, "eval_sampling/sampling_logp_difference/max": 0.30515404794540707, "eval_sampling/sampling_logp_difference/mean": 0.022186610101678297, "eval_steps_per_second": 0.04, "step": 13600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 354.0, "completions/max_terminated_length": 335.8, "completions/mean_length": 236.4375, "completions/mean_terminated_length": 233.51785888671876, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.1469553127884864, "epoch": 1.2512641353314333, "frac_reward_zero_std": 0.4, "grad_norm": 4.067615032196045, "kl": 0.012300558749120682, "learning_rate": 3.3736334730478755e-06, "loss": 0.0007, "num_tokens": 59322984.0, "reward": -0.58125, "reward_std": 0.4805720314383507, "rewards/reward_fn/mean": -0.58125, "rewards/reward_fn/std": 0.4805720552802086, "sampling/importance_sampling_ratio/max": 1.352760946750641, "sampling/importance_sampling_ratio/mean": 1.0000710248947144, "sampling/importance_sampling_ratio/min": 0.7440151154994965, "sampling/sampling_logp_difference/max": 0.36594257354736326, "sampling/sampling_logp_difference/mean": 0.02268607672303915, "step": 13610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 369.3, "completions/max_terminated_length": 369.3, "completions/mean_length": 247.775, "completions/mean_terminated_length": 247.775, "completions/min_length": 149.7, "completions/min_terminated_length": 149.7, "entropy": 1.0627096220850945, "epoch": 1.2521835064815665, "frac_reward_zero_std": 0.3, "grad_norm": 4.381964683532715, "kl": 0.0125552833895199, "learning_rate": 3.371259136121472e-06, "loss": 0.0009, "num_tokens": 59368734.0, "reward": -0.35, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.3469698548316955, "sampling/importance_sampling_ratio/mean": 1.0002039313316344, "sampling/importance_sampling_ratio/min": 0.7104892760515213, "sampling/sampling_logp_difference/max": 0.4441659927368164, "sampling/sampling_logp_difference/mean": 0.021929512917995452, "step": 13620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.2, "completions/max_terminated_length": 336.2, "completions/mean_length": 242.5875, "completions/mean_terminated_length": 240.26785888671876, "completions/min_length": 158.3, "completions/min_terminated_length": 158.3, "entropy": 1.1211291640996932, "epoch": 1.2531028776317, "frac_reward_zero_std": 0.6, "grad_norm": 0.0958312377333641, "kl": 0.0117451824946329, "learning_rate": 3.3688839043420964e-06, "loss": 0.0005, "num_tokens": 59416709.0, "reward": -0.23125, "reward_std": 0.4080048441886902, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.4080048561096191, "sampling/importance_sampling_ratio/max": 1.2711104869842529, "sampling/importance_sampling_ratio/mean": 0.999987781047821, "sampling/importance_sampling_ratio/min": 0.7758310616016388, "sampling/sampling_logp_difference/max": 0.2758403062820435, "sampling/sampling_logp_difference/mean": 0.02184295579791069, "step": 13630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.5, "completions/max_terminated_length": 323.5, "completions/mean_length": 217.275, "completions/mean_terminated_length": 217.275, "completions/min_length": 147.2, "completions/min_terminated_length": 147.2, "entropy": 1.0611898206174373, "epoch": 1.2540222487818333, "frac_reward_zero_std": 0.2, "grad_norm": 6.09527587890625, "kl": 0.012331625778460874, "learning_rate": 3.366507780149301e-06, "loss": 0.0009, "num_tokens": 59460507.0, "reward": 0.025, "reward_std": 0.6784210562705993, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6784210681915284, "sampling/importance_sampling_ratio/max": 1.3751269221305846, "sampling/importance_sampling_ratio/mean": 0.9999768733978271, "sampling/importance_sampling_ratio/min": 0.7359389185905456, "sampling/sampling_logp_difference/max": 0.36294074058532716, "sampling/sampling_logp_difference/mean": 0.022230802290141582, "step": 13640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.9, "completions/max_terminated_length": 294.9, "completions/mean_length": 213.0125, "completions/mean_terminated_length": 213.0125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0147599555552005, "epoch": 1.2549416199319665, "frac_reward_zero_std": 0.5, "grad_norm": 0.11178058385848999, "kl": 0.011493722486193292, "learning_rate": 3.364130765983557e-06, "loss": 0.0002, "num_tokens": 59506252.0, "reward": -0.4, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.3057227611541748, "sampling/importance_sampling_ratio/mean": 1.0004410445690155, "sampling/importance_sampling_ratio/min": 0.7890201807022095, "sampling/sampling_logp_difference/max": 0.2917902052402496, "sampling/sampling_logp_difference/mean": 0.020932627096772193, "step": 13650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 204.8125, "completions/mean_terminated_length": 204.8125, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.135323917120695, "epoch": 1.2558609910821, "frac_reward_zero_std": 0.4, "grad_norm": 0.1269131451845169, "kl": 0.010874840762699022, "learning_rate": 3.3617528642862474e-06, "loss": 0.0007, "num_tokens": 59552629.0, "reward": 0.15, "reward_std": 0.4898623704910278, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.48986239433288575, "sampling/importance_sampling_ratio/max": 1.27200528383255, "sampling/importance_sampling_ratio/mean": 1.000491887331009, "sampling/importance_sampling_ratio/min": 0.7828515589237213, "sampling/sampling_logp_difference/max": 0.26573472023010253, "sampling/sampling_logp_difference/mean": 0.022864541225135326, "step": 13660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.4, "completions/max_terminated_length": 310.8, "completions/mean_length": 220.25, "completions/mean_terminated_length": 216.84285888671874, "completions/min_length": 146.3, "completions/min_terminated_length": 146.3, "entropy": 1.070057739317417, "epoch": 1.256780362232233, "frac_reward_zero_std": 0.4, "grad_norm": 5.911267280578613, "kl": 0.010838153745862656, "learning_rate": 3.359374077499668e-06, "loss": 0.0001, "num_tokens": 59599417.0, "reward": 0.26875, "reward_std": 0.5294113755226135, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.5294114053249359, "sampling/importance_sampling_ratio/max": 1.268005633354187, "sampling/importance_sampling_ratio/mean": 1.0004335999488831, "sampling/importance_sampling_ratio/min": 0.78154137134552, "sampling/sampling_logp_difference/max": 0.2692338407039642, "sampling/sampling_logp_difference/mean": 0.022303778678178787, "step": 13670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 212.8875, "completions/mean_terminated_length": 212.8875, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0538347356021405, "epoch": 1.2576997333823665, "frac_reward_zero_std": 0.5, "grad_norm": 0.08972906321287155, "kl": 0.011987180873984472, "learning_rate": 3.3569944080670243e-06, "loss": 0.0008, "num_tokens": 59643184.0, "reward": -0.4, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.243271815776825, "sampling/importance_sampling_ratio/mean": 1.000413167476654, "sampling/importance_sampling_ratio/min": 0.760655653476715, "sampling/sampling_logp_difference/max": 0.2764920473098755, "sampling/sampling_logp_difference/mean": 0.02165785264223814, "step": 13680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.8, "completions/max_terminated_length": 323.8, "completions/mean_length": 209.225, "completions/mean_terminated_length": 209.225, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 0.9174929346889258, "epoch": 1.2586191045324997, "frac_reward_zero_std": 0.2, "grad_norm": 6.289613723754883, "kl": 0.011677893038722686, "learning_rate": 3.354613858432426e-06, "loss": 0.0005, "num_tokens": 59684850.0, "reward": 0.375, "reward_std": 0.7112202107906341, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.7112202346324921, "sampling/importance_sampling_ratio/max": 1.2506483912467956, "sampling/importance_sampling_ratio/mean": 0.9995531022548676, "sampling/importance_sampling_ratio/min": 0.7165382444858551, "sampling/sampling_logp_difference/max": 0.34672203063964846, "sampling/sampling_logp_difference/mean": 0.019923371355980636, "step": 13690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 208.2875, "completions/mean_terminated_length": 208.2875, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0042922131717205, "epoch": 1.259538475682633, "frac_reward_zero_std": 0.7, "grad_norm": 0.09787281602621078, "kl": 0.010784125319332816, "learning_rate": 3.3522324310408904e-06, "loss": 0.0001, "num_tokens": 59728809.0, "reward": -0.375, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.2925377488136292, "sampling/importance_sampling_ratio/mean": 0.9999913036823272, "sampling/importance_sampling_ratio/min": 0.7859817206859588, "sampling/sampling_logp_difference/max": 0.2767203450202942, "sampling/sampling_logp_difference/mean": 0.02068875189870596, "step": 13700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.6, "completions/max_terminated_length": 282.6, "completions/mean_length": 201.7875, "completions/mean_terminated_length": 201.7875, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.0898744620382785, "epoch": 1.2604578468327663, "frac_reward_zero_std": 0.4, "grad_norm": 5.282708644866943, "kl": 0.010867033718386664, "learning_rate": 3.3498501283383327e-06, "loss": 0.0003, "num_tokens": 59771776.0, "reward": 0.1, "reward_std": 0.4898623704910278, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.48986239433288575, "sampling/importance_sampling_ratio/max": 1.2596891045570373, "sampling/importance_sampling_ratio/mean": 1.0003415167331695, "sampling/importance_sampling_ratio/min": 0.782443743944168, "sampling/sampling_logp_difference/max": 0.2553379476070404, "sampling/sampling_logp_difference/mean": 0.021893630921840667, "step": 13710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.1, "completions/max_terminated_length": 311.1, "completions/mean_length": 198.025, "completions/mean_terminated_length": 198.025, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.0915186010301112, "epoch": 1.2613772179828997, "frac_reward_zero_std": 0.5, "grad_norm": 6.531297206878662, "kl": 0.01157874656491913, "learning_rate": 3.3474669527715705e-06, "loss": 0.0002, "num_tokens": 59812330.0, "reward": -0.2, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.2219540476799011, "sampling/importance_sampling_ratio/mean": 0.9999062895774842, "sampling/importance_sampling_ratio/min": 0.7672264575958252, "sampling/sampling_logp_difference/max": 0.26716679334640503, "sampling/sampling_logp_difference/mean": 0.02115933708846569, "step": 13720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 213.2375, "completions/mean_terminated_length": 213.2375, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.0846717827022077, "epoch": 1.262296589133033, "frac_reward_zero_std": 0.5, "grad_norm": 0.23767361044883728, "kl": 0.011790023889625445, "learning_rate": 3.3450829067883132e-06, "loss": 0.0007, "num_tokens": 59855397.0, "reward": -0.025, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.249717366695404, "sampling/importance_sampling_ratio/mean": 1.0001376688480377, "sampling/importance_sampling_ratio/min": 0.7691656827926636, "sampling/sampling_logp_difference/max": 0.2765718221664429, "sampling/sampling_logp_difference/mean": 0.021959360502660275, "step": 13730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.1, "completions/max_terminated_length": 315.1, "completions/mean_length": 223.95, "completions/mean_terminated_length": 223.95, "completions/min_length": 152.8, "completions/min_terminated_length": 152.8, "entropy": 1.0946502663195132, "epoch": 1.2632159602831663, "frac_reward_zero_std": 0.4, "grad_norm": 4.308483123779297, "kl": 0.01040725020575337, "learning_rate": 3.3426979928371706e-06, "loss": 0.0006, "num_tokens": 59899585.0, "reward": -0.2, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2793826937675477, "sampling/importance_sampling_ratio/mean": 1.0001733958721162, "sampling/importance_sampling_ratio/min": 0.7588054120540619, "sampling/sampling_logp_difference/max": 0.30750308036804197, "sampling/sampling_logp_difference/mean": 0.02172518353909254, "step": 13740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.9, "completions/max_terminated_length": 274.9, "completions/mean_length": 187.05, "completions/mean_terminated_length": 187.05, "completions/min_length": 117.6, "completions/min_terminated_length": 117.6, "entropy": 1.10036740899086, "epoch": 1.2641353314332997, "frac_reward_zero_std": 0.5, "grad_norm": 0.08569607883691788, "kl": 0.010542294254992157, "learning_rate": 3.340312213367638e-06, "loss": 0.0004, "num_tokens": 59939205.0, "reward": 0.1, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.2280383944511413, "sampling/importance_sampling_ratio/mean": 1.0003519892692565, "sampling/importance_sampling_ratio/min": 0.7857196033000946, "sampling/sampling_logp_difference/max": 0.25516090393066404, "sampling/sampling_logp_difference/mean": 0.022711453214287757, "step": 13750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.5, "completions/max_terminated_length": 320.5, "completions/mean_length": 223.05, "completions/mean_terminated_length": 223.05, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0753794647753239, "epoch": 1.2650547025834329, "frac_reward_zero_std": 0.5, "grad_norm": 0.14315083622932434, "kl": 0.0138775369792711, "learning_rate": 3.3379255708301034e-06, "loss": 0.0008, "num_tokens": 59983825.0, "reward": -0.475, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.2707252264022828, "sampling/importance_sampling_ratio/mean": 1.0001060903072356, "sampling/importance_sampling_ratio/min": 0.7590583086013794, "sampling/sampling_logp_difference/max": 0.28936522603034975, "sampling/sampling_logp_difference/mean": 0.021992787159979342, "step": 13760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.8, "completions/max_terminated_length": 296.8, "completions/mean_length": 204.325, "completions/mean_terminated_length": 204.325, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 0.9996506817638874, "epoch": 1.2659740737335663, "frac_reward_zero_std": 0.6, "grad_norm": 0.08471813797950745, "kl": 0.012510418382589705, "learning_rate": 3.3355380676758393e-06, "loss": 0.0007, "num_tokens": 60028611.0, "reward": -0.025, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.2431680321693421, "sampling/importance_sampling_ratio/mean": 0.9998033285140991, "sampling/importance_sampling_ratio/min": 0.757933521270752, "sampling/sampling_logp_difference/max": 0.30170794725418093, "sampling/sampling_logp_difference/mean": 0.021241624653339387, "step": 13770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 221.1375, "completions/mean_terminated_length": 221.1375, "completions/min_length": 149.5, "completions/min_terminated_length": 149.5, "entropy": 1.0615092292428017, "epoch": 1.2668934448836995, "frac_reward_zero_std": 0.7, "grad_norm": 0.10346423089504242, "kl": 0.01170123288175091, "learning_rate": 3.3331497063570032e-06, "loss": 0.0006, "num_tokens": 60075838.0, "reward": -0.025, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2854142665863038, "sampling/importance_sampling_ratio/mean": 1.0007150530815125, "sampling/importance_sampling_ratio/min": 0.7546527087688446, "sampling/sampling_logp_difference/max": 0.29825439453125, "sampling/sampling_logp_difference/mean": 0.021793097630143164, "step": 13780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.075, "completions/max_length": 299.8, "completions/max_terminated_length": 299.8, "completions/mean_length": 237.05, "completions/mean_terminated_length": 235.0625, "completions/min_length": 179.1, "completions/min_terminated_length": 179.1, "entropy": 1.060751572623849, "epoch": 1.267812816033833, "frac_reward_zero_std": 0.5, "grad_norm": 2.5905961990356445, "kl": 0.01362378048361279, "learning_rate": 3.3307604893266334e-06, "loss": 0.0008, "num_tokens": 60123138.0, "reward": -0.3125, "reward_std": 0.43304633498191836, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.43304634690284727, "sampling/importance_sampling_ratio/max": 1.2541781425476075, "sampling/importance_sampling_ratio/mean": 1.0000711500644683, "sampling/importance_sampling_ratio/min": 0.763622236251831, "sampling/sampling_logp_difference/max": 0.28664498329162597, "sampling/sampling_logp_difference/mean": 0.02093506706878543, "step": 13790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.3, "completions/max_terminated_length": 311.3, "completions/mean_length": 215.3125, "completions/mean_terminated_length": 215.3125, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.070097205787897, "epoch": 1.268732187183966, "frac_reward_zero_std": 0.5, "grad_norm": 0.11988935619592667, "kl": 0.010266180170583538, "learning_rate": 3.3283704190386477e-06, "loss": 0.0007, "num_tokens": 60167275.0, "reward": 0.24375, "reward_std": 0.4413444399833679, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.4413444399833679, "sampling/importance_sampling_ratio/max": 1.2578479051589966, "sampling/importance_sampling_ratio/mean": 1.000446182489395, "sampling/importance_sampling_ratio/min": 0.7663450419902802, "sampling/sampling_logp_difference/max": 0.2851829528808594, "sampling/sampling_logp_difference/mean": 0.021840295940637588, "step": 13800 }, { "epoch": 1.268732187183966, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 298.71204188481676, "eval_completions/max_terminated_length": 297.1273996509599, "eval_completions/mean_length": 210.2539267015707, "eval_completions/mean_terminated_length": 209.77852581350382, "eval_completions/min_length": 137.24432809773123, "eval_completions/min_terminated_length": 137.24432809773123, "eval_entropy": 1.0773127597560017, "eval_frac_reward_zero_std": 0.48342059336823734, "eval_kl": 0.011127440206849554, "eval_loss": 0.000531416735611856, "eval_num_tokens": 60167275.0, "eval_reward": -0.10405759162303665, "eval_reward_std": 0.4611572347334334, "eval_rewards/reward_fn/mean": -0.10405759162303665, "eval_rewards/reward_fn/std": 0.46115724784021395, "eval_runtime": 1739.0416, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.271415042211456, "eval_sampling/importance_sampling_ratio/mean": 1.0000393544608177, "eval_sampling/importance_sampling_ratio/min": 0.7678306781496677, "eval_sampling/sampling_logp_difference/max": 0.2915417479178876, "eval_sampling/sampling_logp_difference/mean": 0.021398891985780488, "eval_steps_per_second": 0.041, "step": 13800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 335.6, "completions/max_terminated_length": 321.4, "completions/mean_length": 232.5625, "completions/mean_terminated_length": 223.7559555053711, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.0431741245090962, "epoch": 1.2696515583340995, "frac_reward_zero_std": 0.4, "grad_norm": 6.2760748863220215, "kl": 0.010889346909243614, "learning_rate": 3.3259794979478392e-06, "loss": 0.0005, "num_tokens": 60211232.0, "reward": 0.00625, "reward_std": 0.5616438746452331, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.5616438746452331, "sampling/importance_sampling_ratio/max": 1.2358537077903748, "sampling/importance_sampling_ratio/mean": 1.0002201855182649, "sampling/importance_sampling_ratio/min": 0.7866417646408081, "sampling/sampling_logp_difference/max": 0.2448671817779541, "sampling/sampling_logp_difference/mean": 0.02145417369902134, "step": 13810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 198.95, "completions/mean_terminated_length": 198.95, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.0163672544062137, "epoch": 1.270570929484233, "frac_reward_zero_std": 0.5, "grad_norm": 0.09999719262123108, "kl": 0.011739226226927713, "learning_rate": 3.3235877285098768e-06, "loss": 0.0005, "num_tokens": 60253572.0, "reward": 0.1, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2962898373603822, "sampling/importance_sampling_ratio/mean": 0.9996142745018005, "sampling/importance_sampling_ratio/min": 0.7699313402175904, "sampling/sampling_logp_difference/max": 0.31555152535438535, "sampling/sampling_logp_difference/mean": 0.02086798772215843, "step": 13820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.2, "completions/max_terminated_length": 317.2, "completions/mean_length": 225.1875, "completions/mean_terminated_length": 225.1875, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 1.0183143254369498, "epoch": 1.271490300634366, "frac_reward_zero_std": 0.4, "grad_norm": 5.095686435699463, "kl": 0.011168564693070948, "learning_rate": 3.321195113181298e-06, "loss": 0.0007, "num_tokens": 60298155.0, "reward": 0.0, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.2847809791564941, "sampling/importance_sampling_ratio/mean": 1.000015413761139, "sampling/importance_sampling_ratio/min": 0.7733495771884918, "sampling/sampling_logp_difference/max": 0.2853808283805847, "sampling/sampling_logp_difference/mean": 0.0208575289696455, "step": 13830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.4, "completions/max_terminated_length": 284.4, "completions/mean_length": 190.2, "completions/mean_terminated_length": 190.2, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.0496782913804055, "epoch": 1.2724096717844995, "frac_reward_zero_std": 0.4, "grad_norm": 5.250753402709961, "kl": 0.012114070466486737, "learning_rate": 3.318801654419511e-06, "loss": 0.0005, "num_tokens": 60337955.0, "reward": -0.3, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.3011623859405517, "sampling/importance_sampling_ratio/mean": 0.9998733043670655, "sampling/importance_sampling_ratio/min": 0.7886810302734375, "sampling/sampling_logp_difference/max": 0.2785904884338379, "sampling/sampling_logp_difference/mean": 0.02145045418292284, "step": 13840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 212.975, "completions/mean_terminated_length": 212.975, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.116989615559578, "epoch": 1.2733290429346327, "frac_reward_zero_std": 0.5, "grad_norm": 6.4846601486206055, "kl": 0.012697600747924299, "learning_rate": 3.3164073546827906e-06, "loss": 0.0005, "num_tokens": 60382473.0, "reward": -0.375, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2550538301467895, "sampling/importance_sampling_ratio/mean": 0.9998733043670655, "sampling/importance_sampling_ratio/min": 0.7392546653747558, "sampling/sampling_logp_difference/max": 0.3260577321052551, "sampling/sampling_logp_difference/mean": 0.022472891211509704, "step": 13850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.1, "completions/max_terminated_length": 295.1, "completions/mean_length": 208.45, "completions/mean_terminated_length": 208.45, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 0.9971697323024273, "epoch": 1.274248414084766, "frac_reward_zero_std": 0.2, "grad_norm": 0.2938764989376068, "kl": 0.011485464312136174, "learning_rate": 3.3140122164302735e-06, "loss": 0.0007, "num_tokens": 60426909.0, "reward": 0.45, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.2366588354110717, "sampling/importance_sampling_ratio/mean": 0.9998960614204406, "sampling/importance_sampling_ratio/min": 0.7965594112873078, "sampling/sampling_logp_difference/max": 0.23663683533668517, "sampling/sampling_logp_difference/mean": 0.02070887703448534, "step": 13860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.8, "completions/max_terminated_length": 281.8, "completions/mean_length": 199.0625, "completions/mean_terminated_length": 199.0625, "completions/min_length": 113.8, "completions/min_terminated_length": 113.8, "entropy": 1.0584336303174495, "epoch": 1.2751677852348993, "frac_reward_zero_std": 0.5, "grad_norm": 6.932671070098877, "kl": 0.01366493059322238, "learning_rate": 3.311616242121959e-06, "loss": 0.0005, "num_tokens": 60467970.0, "reward": 0.125, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2886322498321534, "sampling/importance_sampling_ratio/mean": 1.0002249836921693, "sampling/importance_sampling_ratio/min": 0.754968348145485, "sampling/sampling_logp_difference/max": 0.328838586807251, "sampling/sampling_logp_difference/mean": 0.021528098732233047, "step": 13870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.3, "completions/max_terminated_length": 305.9, "completions/mean_length": 217.75, "completions/mean_terminated_length": 215.3125, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0837241098284722, "epoch": 1.2760871563850327, "frac_reward_zero_std": 0.6, "grad_norm": 0.31722763180732727, "kl": 0.011626588128274306, "learning_rate": 3.3092194342187043e-06, "loss": 0.0009, "num_tokens": 60514126.0, "reward": -0.55625, "reward_std": 0.2516810342669487, "rewards/reward_fn/mean": -0.55625, "rewards/reward_fn/std": 0.2516810342669487, "sampling/importance_sampling_ratio/max": 1.4136049032211304, "sampling/importance_sampling_ratio/mean": 1.0003038346767426, "sampling/importance_sampling_ratio/min": 0.7610296845436096, "sampling/sampling_logp_difference/max": 0.3787039279937744, "sampling/sampling_logp_difference/mean": 0.022233299724757672, "step": 13880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.7, "completions/max_terminated_length": 332.7, "completions/mean_length": 226.275, "completions/mean_terminated_length": 226.275, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 1.1098318077623843, "epoch": 1.2770065275351659, "frac_reward_zero_std": 0.3, "grad_norm": 5.35006856918335, "kl": 0.011762978782644495, "learning_rate": 3.3068217951822245e-06, "loss": 0.0001, "num_tokens": 60562708.0, "reward": -0.2, "reward_std": 0.6800980150699616, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6800980269908905, "sampling/importance_sampling_ratio/max": 1.3525440573692322, "sampling/importance_sampling_ratio/mean": 1.0000444591045379, "sampling/importance_sampling_ratio/min": 0.7743641912937165, "sampling/sampling_logp_difference/max": 0.32447699308395384, "sampling/sampling_logp_difference/mean": 0.02286960296332836, "step": 13890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 203.525, "completions/mean_terminated_length": 203.525, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.0347427174448967, "epoch": 1.2779258986852993, "frac_reward_zero_std": 0.2, "grad_norm": 0.13321539759635925, "kl": 0.011401356651913375, "learning_rate": 3.304423327475086e-06, "loss": 0.0001, "num_tokens": 60604662.0, "reward": -0.15, "reward_std": 0.7658906996250152, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.7658907353878022, "sampling/importance_sampling_ratio/max": 1.2804095506668092, "sampling/importance_sampling_ratio/mean": 1.0002930462360382, "sampling/importance_sampling_ratio/min": 0.7705044031143189, "sampling/sampling_logp_difference/max": 0.30423173904418943, "sampling/sampling_logp_difference/mean": 0.02122398801147938, "step": 13900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.1, "completions/max_terminated_length": 335.1, "completions/mean_length": 200.7875, "completions/mean_terminated_length": 200.7875, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 0.9987304754555225, "epoch": 1.2788452698354327, "frac_reward_zero_std": 0.6, "grad_norm": 0.10112756490707397, "kl": 0.01132296079886146, "learning_rate": 3.302024033560708e-06, "loss": 0.0007, "num_tokens": 60646093.0, "reward": -0.075, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.250422716140747, "sampling/importance_sampling_ratio/mean": 0.9998579502105713, "sampling/importance_sampling_ratio/min": 0.7880058526992798, "sampling/sampling_logp_difference/max": 0.25142712593078614, "sampling/sampling_logp_difference/mean": 0.02031352687627077, "step": 13910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.9, "completions/max_terminated_length": 260.9, "completions/mean_length": 188.2875, "completions/mean_terminated_length": 188.2875, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.0443702194839717, "epoch": 1.2797646409855659, "frac_reward_zero_std": 0.6, "grad_norm": 6.143082141876221, "kl": 0.012493279960472137, "learning_rate": 3.299623915903357e-06, "loss": 0.0005, "num_tokens": 60688644.0, "reward": 0.4, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.297843050956726, "sampling/importance_sampling_ratio/mean": 0.9998142063617707, "sampling/importance_sampling_ratio/min": 0.7854148924350739, "sampling/sampling_logp_difference/max": 0.2784585952758789, "sampling/sampling_logp_difference/mean": 0.021057771518826485, "step": 13920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 355.0, "completions/max_terminated_length": 345.0, "completions/mean_length": 260.4625, "completions/mean_terminated_length": 258.06964416503905, "completions/min_length": 164.5, "completions/min_terminated_length": 164.5, "entropy": 1.0945907536894084, "epoch": 1.280684012135699, "frac_reward_zero_std": 0.1, "grad_norm": 3.3669416904449463, "kl": 0.012461850588442758, "learning_rate": 3.2972229769681475e-06, "loss": 0.0003, "num_tokens": 60737929.0, "reward": 0.19375, "reward_std": 0.8587941229343414, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.8587941348552703, "sampling/importance_sampling_ratio/max": 1.3181120157241821, "sampling/importance_sampling_ratio/mean": 0.9997206687927246, "sampling/importance_sampling_ratio/min": 0.7373691618442535, "sampling/sampling_logp_difference/max": 0.3360304653644562, "sampling/sampling_logp_difference/mean": 0.021761367935687302, "step": 13930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.5, "completions/max_terminated_length": 282.5, "completions/mean_length": 203.2125, "completions/mean_terminated_length": 203.2125, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.050247573107481, "epoch": 1.2816033832858325, "frac_reward_zero_std": 0.4, "grad_norm": 0.10736138373613358, "kl": 0.011127402813872322, "learning_rate": 3.2948212192210347e-06, "loss": 0.0003, "num_tokens": 60780730.0, "reward": 0.55, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": 0.55, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2536615133285522, "sampling/importance_sampling_ratio/mean": 1.0002457618713378, "sampling/importance_sampling_ratio/min": 0.796945083141327, "sampling/sampling_logp_difference/max": 0.24934355020523072, "sampling/sampling_logp_difference/mean": 0.021067624539136888, "step": 13940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.4, "completions/max_terminated_length": 291.4, "completions/mean_length": 214.2625, "completions/mean_terminated_length": 214.2625, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.062100102007389, "epoch": 1.2825227544359659, "frac_reward_zero_std": 0.5, "grad_norm": 0.0868634283542633, "kl": 0.01183248435263522, "learning_rate": 3.292418645128817e-06, "loss": 0.0004, "num_tokens": 60824287.0, "reward": 0.225, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2548524618148804, "sampling/importance_sampling_ratio/mean": 0.9998877286911011, "sampling/importance_sampling_ratio/min": 0.7227929770946503, "sampling/sampling_logp_difference/max": 0.3389148056507111, "sampling/sampling_logp_difference/mean": 0.02083495110273361, "step": 13950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.3, "completions/max_terminated_length": 296.3, "completions/mean_length": 206.35, "completions/mean_terminated_length": 206.35, "completions/min_length": 151.8, "completions/min_terminated_length": 151.8, "entropy": 1.1004817977547645, "epoch": 1.283442125586099, "frac_reward_zero_std": 0.2, "grad_norm": 5.168828964233398, "kl": 0.010929867607774212, "learning_rate": 3.2900152571591317e-06, "loss": 0.0006, "num_tokens": 60869787.0, "reward": -0.125, "reward_std": 0.7187847554683685, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.7187847554683685, "sampling/importance_sampling_ratio/max": 1.2682936310768127, "sampling/importance_sampling_ratio/mean": 1.0005610227584838, "sampling/importance_sampling_ratio/min": 0.7747234165668487, "sampling/sampling_logp_difference/max": 0.2696775674819946, "sampling/sampling_logp_difference/mean": 0.021471554785966872, "step": 13960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.5, "completions/max_terminated_length": 328.5, "completions/mean_length": 228.175, "completions/mean_terminated_length": 228.175, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.1824248261749744, "epoch": 1.2843614967362325, "frac_reward_zero_std": 0.3, "grad_norm": 0.12412134557962418, "kl": 0.013051330973394216, "learning_rate": 3.287611057780449e-06, "loss": 0.0007, "num_tokens": 60915057.0, "reward": -0.225, "reward_std": 0.6152592301368713, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6152592420578002, "sampling/importance_sampling_ratio/max": 1.2765207886695862, "sampling/importance_sampling_ratio/mean": 0.9999041557312012, "sampling/importance_sampling_ratio/min": 0.7412395417690277, "sampling/sampling_logp_difference/max": 0.33352811336517335, "sampling/sampling_logp_difference/mean": 0.02291590943932533, "step": 13970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 205.95, "completions/mean_terminated_length": 205.95, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0559897392988205, "epoch": 1.2852808678863656, "frac_reward_zero_std": 0.5, "grad_norm": 6.403100967407227, "kl": 0.012645882059587166, "learning_rate": 3.2852060494620752e-06, "loss": 0.0007, "num_tokens": 60959869.0, "reward": -0.25, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.399783980846405, "sampling/importance_sampling_ratio/mean": 1.0004522204399109, "sampling/importance_sampling_ratio/min": 0.7398989856243133, "sampling/sampling_logp_difference/max": 0.43043878078460696, "sampling/sampling_logp_difference/mean": 0.02215727586299181, "step": 13980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.6, "completions/max_terminated_length": 311.6, "completions/mean_length": 206.1125, "completions/mean_terminated_length": 206.1125, "completions/min_length": 119.6, "completions/min_terminated_length": 119.6, "entropy": 0.9989783201366663, "epoch": 1.286200239036499, "frac_reward_zero_std": 0.4, "grad_norm": 0.19533534348011017, "kl": 0.011434272755286657, "learning_rate": 3.2828002346741463e-06, "loss": 0.0006, "num_tokens": 61003278.0, "reward": -0.25, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2387512564659118, "sampling/importance_sampling_ratio/mean": 1.0002170503139496, "sampling/importance_sampling_ratio/min": 0.7356505990028381, "sampling/sampling_logp_difference/max": 0.31349279284477233, "sampling/sampling_logp_difference/mean": 0.020968426670879125, "step": 13990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.5, "completions/max_terminated_length": 302.5, "completions/mean_length": 211.0625, "completions/mean_terminated_length": 211.0625, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0923313036561013, "epoch": 1.2871196101866325, "frac_reward_zero_std": 0.7, "grad_norm": 0.0992208942770958, "kl": 0.013289419643115252, "learning_rate": 3.2803936158876276e-06, "loss": 0.0008, "num_tokens": 61047563.0, "reward": 0.0, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.309867250919342, "sampling/importance_sampling_ratio/mean": 1.0000964939594268, "sampling/importance_sampling_ratio/min": 0.7660624504089355, "sampling/sampling_logp_difference/max": 0.2988543748855591, "sampling/sampling_logp_difference/mean": 0.02229737974703312, "step": 14000 }, { "epoch": 1.2871196101866325, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 286.25305410122166, "eval_completions/max_terminated_length": 285.3490401396161, "eval_completions/mean_length": 201.18259162303664, "eval_completions/mean_terminated_length": 200.8798097673719, "eval_completions/min_length": 128.36823734729495, "eval_completions/min_terminated_length": 128.36823734729495, "eval_entropy": 1.0583787708694399, "eval_frac_reward_zero_std": 0.4537521815008726, "eval_kl": 0.011944737512923869, "eval_loss": 0.0005730666453018785, "eval_num_tokens": 61047563.0, "eval_reward": -0.11300174520069808, "eval_reward_std": 0.48118967358770587, "eval_rewards/reward_fn/mean": -0.11300174520069808, "eval_rewards/reward_fn/std": 0.48118969012721463, "eval_runtime": 1675.8505, "eval_samples_per_second": 0.342, "eval_sampling/importance_sampling_ratio/max": 1.280812550381632, "eval_sampling/importance_sampling_ratio/mean": 1.000029692371062, "eval_sampling/importance_sampling_ratio/min": 0.76722762248695, "eval_sampling/sampling_logp_difference/max": 0.29693282129044724, "eval_sampling/sampling_logp_difference/mean": 0.021418172903488963, "eval_steps_per_second": 0.043, "step": 14000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.5, "completions/max_terminated_length": 284.5, "completions/mean_length": 196.75, "completions/mean_terminated_length": 196.75, "completions/min_length": 121.9, "completions/min_terminated_length": 121.9, "entropy": 0.9726397871971131, "epoch": 1.2880389813367656, "frac_reward_zero_std": 0.7, "grad_norm": 0.09803581982851028, "kl": 0.010991890079458244, "learning_rate": 3.2779861955743093e-06, "loss": 0.0004, "num_tokens": 61091231.0, "reward": 0.45, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2299365639686584, "sampling/importance_sampling_ratio/mean": 0.999958050251007, "sampling/importance_sampling_ratio/min": 0.7679587602615356, "sampling/sampling_logp_difference/max": 0.27473343908786774, "sampling/sampling_logp_difference/mean": 0.020129251666367053, "step": 14010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.2, "completions/max_terminated_length": 318.2, "completions/mean_length": 212.1375, "completions/mean_terminated_length": 212.1375, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0555764749646186, "epoch": 1.2889583524868988, "frac_reward_zero_std": 0.3, "grad_norm": 5.194159507751465, "kl": 0.012685057986527682, "learning_rate": 3.2755779762068045e-06, "loss": 0.0003, "num_tokens": 61134082.0, "reward": -0.125, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.285775077342987, "sampling/importance_sampling_ratio/mean": 1.0004034876823424, "sampling/importance_sampling_ratio/min": 0.76815767288208, "sampling/sampling_logp_difference/max": 0.3136365294456482, "sampling/sampling_logp_difference/mean": 0.022021930664777756, "step": 14020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.3, "completions/max_terminated_length": 274.3, "completions/mean_length": 192.8625, "completions/mean_terminated_length": 192.8625, "completions/min_length": 115.3, "completions/min_terminated_length": 115.3, "entropy": 1.0151037946343422, "epoch": 1.2898777236370322, "frac_reward_zero_std": 0.4, "grad_norm": 5.013052463531494, "kl": 0.015582594700390473, "learning_rate": 3.2731689602585482e-06, "loss": 0.0005, "num_tokens": 61174263.0, "reward": 0.1, "reward_std": 0.5117493987083435, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5117493987083435, "sampling/importance_sampling_ratio/max": 1.252225649356842, "sampling/importance_sampling_ratio/mean": 0.9999263048171997, "sampling/importance_sampling_ratio/min": 0.7717524409294129, "sampling/sampling_logp_difference/max": 0.29316273927688596, "sampling/sampling_logp_difference/mean": 0.020998944714665412, "step": 14030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.3, "completions/max_terminated_length": 309.3, "completions/mean_length": 215.75, "completions/mean_terminated_length": 215.75, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.0836028821766377, "epoch": 1.2907970947871656, "frac_reward_zero_std": 0.7, "grad_norm": 5.232485294342041, "kl": 0.012705548806115985, "learning_rate": 3.2707591502037927e-06, "loss": 0.0003, "num_tokens": 61217819.0, "reward": -0.225, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2769073128700257, "sampling/importance_sampling_ratio/mean": 0.9998096227645874, "sampling/importance_sampling_ratio/min": 0.7469082176685333, "sampling/sampling_logp_difference/max": 0.30882670879364016, "sampling/sampling_logp_difference/mean": 0.02190206293016672, "step": 14040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 209.95, "completions/mean_terminated_length": 209.95, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0774663031101226, "epoch": 1.2917164659372988, "frac_reward_zero_std": 0.3, "grad_norm": 0.10130154341459274, "kl": 0.014014481916092336, "learning_rate": 3.268348548517607e-06, "loss": 0.0004, "num_tokens": 61261055.0, "reward": -0.1, "reward_std": 0.6657755315303803, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6657755434513092, "sampling/importance_sampling_ratio/max": 1.2823560118675232, "sampling/importance_sampling_ratio/mean": 0.9995956897735596, "sampling/importance_sampling_ratio/min": 0.763829517364502, "sampling/sampling_logp_difference/max": 0.28889893293380736, "sampling/sampling_logp_difference/mean": 0.02232759166508913, "step": 14050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.3, "completions/max_terminated_length": 286.8, "completions/mean_length": 208.225, "completions/mean_terminated_length": 205.59285888671874, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.0918292284011841, "epoch": 1.2926358370874322, "frac_reward_zero_std": 0.5, "grad_norm": 6.250483989715576, "kl": 0.011526072712149471, "learning_rate": 3.265937157675869e-06, "loss": 0.0004, "num_tokens": 61305961.0, "reward": -0.18125, "reward_std": 0.36951334923505785, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.36951336115598676, "sampling/importance_sampling_ratio/max": 1.2422160148620605, "sampling/importance_sampling_ratio/mean": 0.999760377407074, "sampling/importance_sampling_ratio/min": 0.6662306934595108, "sampling/sampling_logp_difference/max": 0.46547071933746337, "sampling/sampling_logp_difference/mean": 0.021979919634759426, "step": 14060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 218.9375, "completions/mean_terminated_length": 218.9375, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.0881498441100121, "epoch": 1.2935552082375654, "frac_reward_zero_std": 0.6, "grad_norm": 0.08359672129154205, "kl": 0.010431270755361766, "learning_rate": 3.263524980155275e-06, "loss": 0.0004, "num_tokens": 61350252.0, "reward": -0.25, "reward_std": 0.420828652381897, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.42082867622375486, "sampling/importance_sampling_ratio/max": 1.2653282165527344, "sampling/importance_sampling_ratio/mean": 1.0001354932785034, "sampling/importance_sampling_ratio/min": 0.7865015983581543, "sampling/sampling_logp_difference/max": 0.26205642223358155, "sampling/sampling_logp_difference/mean": 0.02180274501442909, "step": 14070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.2, "completions/max_terminated_length": 305.2, "completions/mean_length": 209.5375, "completions/mean_terminated_length": 209.5375, "completions/min_length": 115.3, "completions/min_terminated_length": 115.3, "entropy": 1.109402571618557, "epoch": 1.2944745793876988, "frac_reward_zero_std": 0.3, "grad_norm": 0.09521181881427765, "kl": 0.01112961363978684, "learning_rate": 3.261112018433321e-06, "loss": 0.0004, "num_tokens": 61394007.0, "reward": -0.025, "reward_std": 0.538717406988144, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.538717406988144, "sampling/importance_sampling_ratio/max": 1.2824077606201172, "sampling/importance_sampling_ratio/mean": 1.0002397358417512, "sampling/importance_sampling_ratio/min": 0.7823758661746979, "sampling/sampling_logp_difference/max": 0.26647999286651614, "sampling/sampling_logp_difference/mean": 0.02152075469493866, "step": 14080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 304.3, "completions/max_terminated_length": 302.2, "completions/mean_length": 216.1625, "completions/mean_terminated_length": 212.95416870117188, "completions/min_length": 147.7, "completions/min_terminated_length": 147.7, "entropy": 0.9286431200802326, "epoch": 1.2953939505378322, "frac_reward_zero_std": 0.3, "grad_norm": 0.10013268142938614, "kl": 0.011100528421229683, "learning_rate": 3.2586982749883135e-06, "loss": 0.0004, "num_tokens": 61440076.0, "reward": -0.1375, "reward_std": 0.5941853106021882, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.5941853225231171, "sampling/importance_sampling_ratio/max": 1.2656769394874572, "sampling/importance_sampling_ratio/mean": 0.9997518420219421, "sampling/importance_sampling_ratio/min": 0.7546243309974671, "sampling/sampling_logp_difference/max": 0.2924291014671326, "sampling/sampling_logp_difference/mean": 0.020414523780345917, "step": 14090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 201.8875, "completions/mean_terminated_length": 201.8875, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.0942590348422527, "epoch": 1.2963133216879654, "frac_reward_zero_std": 0.6, "grad_norm": 4.7359819412231445, "kl": 0.013214588252594694, "learning_rate": 3.2562837522993618e-06, "loss": 0.0008, "num_tokens": 61481563.0, "reward": -0.175, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.26415011882782, "sampling/importance_sampling_ratio/mean": 0.999830013513565, "sampling/importance_sampling_ratio/min": 0.8019089221954345, "sampling/sampling_logp_difference/max": 0.2435823917388916, "sampling/sampling_logp_difference/mean": 0.02183500789105892, "step": 14100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.2, "completions/max_terminated_length": 326.2, "completions/mean_length": 229.425, "completions/mean_terminated_length": 229.425, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.0760547511279583, "epoch": 1.2972326928380986, "frac_reward_zero_std": 0.5, "grad_norm": 5.597666263580322, "kl": 0.011034123250283301, "learning_rate": 3.2538684528463716e-06, "loss": 0.0004, "num_tokens": 61530285.0, "reward": -0.275, "reward_std": 0.4738378763198853, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4738378882408142, "sampling/importance_sampling_ratio/max": 1.285448181629181, "sampling/importance_sampling_ratio/mean": 1.000157457590103, "sampling/importance_sampling_ratio/min": 0.7734461963176728, "sampling/sampling_logp_difference/max": 0.29336836338043215, "sampling/sampling_logp_difference/mean": 0.021742811240255833, "step": 14110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 204.3375, "completions/mean_terminated_length": 204.3375, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0071913473308087, "epoch": 1.298152063988232, "frac_reward_zero_std": 0.2, "grad_norm": 5.7514967918396, "kl": 0.010570807894691826, "learning_rate": 3.2514523791100506e-06, "loss": 0.0001, "num_tokens": 61574048.0, "reward": -0.25, "reward_std": 0.7221480309963226, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.7221480667591095, "sampling/importance_sampling_ratio/max": 1.2465546369552611, "sampling/importance_sampling_ratio/mean": 0.9993069589138031, "sampling/importance_sampling_ratio/min": 0.7708948791027069, "sampling/sampling_logp_difference/max": 0.27606927156448363, "sampling/sampling_logp_difference/mean": 0.020013948902487755, "step": 14120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.6, "completions/max_terminated_length": 270.6, "completions/mean_length": 206.9125, "completions/mean_terminated_length": 206.9125, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0637610368430614, "epoch": 1.2990714351383654, "frac_reward_zero_std": 0.4, "grad_norm": 6.901784420013428, "kl": 0.011994443903677165, "learning_rate": 3.2490355335719003e-06, "loss": 0.0002, "num_tokens": 61620401.0, "reward": -0.275, "reward_std": 0.5588553428649903, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5588553786277771, "sampling/importance_sampling_ratio/max": 1.2869035243988036, "sampling/importance_sampling_ratio/mean": 0.9998305201530456, "sampling/importance_sampling_ratio/min": 0.7702841639518738, "sampling/sampling_logp_difference/max": 0.2844775319099426, "sampling/sampling_logp_difference/mean": 0.021297093480825424, "step": 14130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 217.2, "completions/mean_terminated_length": 217.2, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.1217822790145875, "epoch": 1.2999908062884986, "frac_reward_zero_std": 0.2, "grad_norm": 5.882911205291748, "kl": 0.012687637261115015, "learning_rate": 3.2466179187142143e-06, "loss": 0.0009, "num_tokens": 61664425.0, "reward": -0.225, "reward_std": 0.6203559041023254, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6203559160232544, "sampling/importance_sampling_ratio/max": 1.337649440765381, "sampling/importance_sampling_ratio/mean": 1.0001010417938232, "sampling/importance_sampling_ratio/min": 0.715613104403019, "sampling/sampling_logp_difference/max": 0.41859147548675535, "sampling/sampling_logp_difference/mean": 0.0223718686029315, "step": 14140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 204.5625, "completions/mean_terminated_length": 204.5625, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0657050423324108, "epoch": 1.300910177438632, "frac_reward_zero_std": 0.2, "grad_norm": 5.645789623260498, "kl": 0.012376716057769955, "learning_rate": 3.244199537020077e-06, "loss": 0.0007, "num_tokens": 61708646.0, "reward": 0.025, "reward_std": 0.6968977272510528, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6968977510929107, "sampling/importance_sampling_ratio/max": 1.2944947361946106, "sampling/importance_sampling_ratio/mean": 0.9998242259025574, "sampling/importance_sampling_ratio/min": 0.7646070778369903, "sampling/sampling_logp_difference/max": 0.31702356934547427, "sampling/sampling_logp_difference/mean": 0.021232466213405132, "step": 14150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 209.1625, "completions/mean_terminated_length": 209.1625, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0345599450170995, "epoch": 1.3018295485887652, "frac_reward_zero_std": 0.6, "grad_norm": 0.23806661367416382, "kl": 0.011553524993360043, "learning_rate": 3.241780390973361e-06, "loss": 0.0007, "num_tokens": 61753939.0, "reward": 0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.3556976437568664, "sampling/importance_sampling_ratio/mean": 0.9998816668987274, "sampling/importance_sampling_ratio/min": 0.7997465610504151, "sampling/sampling_logp_difference/max": 0.31545245051383974, "sampling/sampling_logp_difference/mean": 0.021120969392359256, "step": 14160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.4, "completions/max_terminated_length": 284.4, "completions/mean_length": 197.8125, "completions/mean_terminated_length": 197.8125, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 0.9995936632156373, "epoch": 1.3027489197388986, "frac_reward_zero_std": 0.5, "grad_norm": 4.848412990570068, "kl": 0.010355523059843108, "learning_rate": 3.239360483058721e-06, "loss": 0.0005, "num_tokens": 61793996.0, "reward": -0.25, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2516098856925963, "sampling/importance_sampling_ratio/mean": 1.000349622964859, "sampling/importance_sampling_ratio/min": 0.7602176785469055, "sampling/sampling_logp_difference/max": 0.2794570684432983, "sampling/sampling_logp_difference/mean": 0.020623466186225414, "step": 14170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.2, "completions/max_terminated_length": 288.2, "completions/mean_length": 203.1125, "completions/mean_terminated_length": 203.1125, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.0190670561045407, "epoch": 1.303668290889032, "frac_reward_zero_std": 0.6, "grad_norm": 7.143975257873535, "kl": 0.0109583280980587, "learning_rate": 3.2369398157615995e-06, "loss": 0.0008, "num_tokens": 61833581.0, "reward": -0.2, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2673624992370605, "sampling/importance_sampling_ratio/mean": 0.9998266220092773, "sampling/importance_sampling_ratio/min": 0.786111307144165, "sampling/sampling_logp_difference/max": 0.2674436688423157, "sampling/sampling_logp_difference/mean": 0.02068785037845373, "step": 14180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 200.5125, "completions/mean_terminated_length": 200.5125, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0907985579222441, "epoch": 1.3045876620391652, "frac_reward_zero_std": 0.5, "grad_norm": 0.556806206703186, "kl": 0.01169002842507325, "learning_rate": 3.2345183915682134e-06, "loss": 0.0008, "num_tokens": 61874606.0, "reward": -0.175, "reward_std": 0.4410387217998505, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4410387217998505, "sampling/importance_sampling_ratio/max": 1.247003984451294, "sampling/importance_sampling_ratio/mean": 0.999652522802353, "sampling/importance_sampling_ratio/min": 0.79026158452034, "sampling/sampling_logp_difference/max": 0.25003105998039243, "sampling/sampling_logp_difference/mean": 0.021546063758432866, "step": 14190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 219.175, "completions/mean_terminated_length": 219.175, "completions/min_length": 141.1, "completions/min_terminated_length": 141.1, "entropy": 1.1570598036050797, "epoch": 1.3055070331892984, "frac_reward_zero_std": 0.3, "grad_norm": 6.271591663360596, "kl": 0.011117229436058551, "learning_rate": 3.2320962129655597e-06, "loss": 0.0006, "num_tokens": 61921628.0, "reward": -0.2, "reward_std": 0.651453047990799, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6514530599117279, "sampling/importance_sampling_ratio/max": 1.248835039138794, "sampling/importance_sampling_ratio/mean": 0.999636447429657, "sampling/importance_sampling_ratio/min": 0.7708378672599793, "sampling/sampling_logp_difference/max": 0.27125184535980223, "sampling/sampling_logp_difference/mean": 0.021883097104728223, "step": 14200 }, { "epoch": 1.3055070331892984, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 299.2111692844677, "eval_completions/max_terminated_length": 295.9668411867365, "eval_completions/mean_length": 208.7532722513089, "eval_completions/mean_terminated_length": 207.94325003365987, "eval_completions/min_length": 136.08376963350784, "eval_completions/min_terminated_length": 136.08376963350784, "eval_entropy": 1.0811373974744354, "eval_frac_reward_zero_std": 0.4694589877835951, "eval_kl": 0.011531258105132957, "eval_loss": 0.0005945896846242249, "eval_num_tokens": 61921628.0, "eval_reward": -0.11147469458987784, "eval_reward_std": 0.45287854124322197, "eval_rewards/reward_fn/mean": -0.11147469458987784, "eval_rewards/reward_fn/std": 0.4528785539339142, "eval_runtime": 1739.0595, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.2726136256678982, "eval_sampling/importance_sampling_ratio/mean": 0.9999434135555181, "eval_sampling/importance_sampling_ratio/min": 0.7711658476222872, "eval_sampling/sampling_logp_difference/max": 0.2932136486858182, "eval_sampling/sampling_logp_difference/mean": 0.02114746678315012, "eval_steps_per_second": 0.041, "step": 14200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.7, "completions/max_terminated_length": 329.7, "completions/mean_length": 217.7875, "completions/mean_terminated_length": 217.7875, "completions/min_length": 141.7, "completions/min_terminated_length": 141.7, "entropy": 1.0903087370097637, "epoch": 1.3064264043394318, "frac_reward_zero_std": 0.4, "grad_norm": 7.073495388031006, "kl": 0.012449744943296537, "learning_rate": 3.2296732824414114e-06, "loss": 0.0002, "num_tokens": 61967075.0, "reward": 0.375, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2308412671089173, "sampling/importance_sampling_ratio/mean": 0.9997945308685303, "sampling/importance_sampling_ratio/min": 0.7368205070495606, "sampling/sampling_logp_difference/max": 0.31938259601593016, "sampling/sampling_logp_difference/mean": 0.02098805196583271, "step": 14210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.6, "completions/max_terminated_length": 276.6, "completions/mean_length": 205.7125, "completions/mean_terminated_length": 205.7125, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.07438894957304, "epoch": 1.3073457754895652, "frac_reward_zero_std": 0.4, "grad_norm": 6.577508449554443, "kl": 0.010672560764942319, "learning_rate": 3.2272496024843113e-06, "loss": 0.0004, "num_tokens": 62008300.0, "reward": -0.03125, "reward_std": 0.5542153477668762, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5542153716087341, "sampling/importance_sampling_ratio/max": 1.2848419666290283, "sampling/importance_sampling_ratio/mean": 1.0000403642654419, "sampling/importance_sampling_ratio/min": 0.7668918251991272, "sampling/sampling_logp_difference/max": 0.2986613869667053, "sampling/sampling_logp_difference/mean": 0.02157081216573715, "step": 14220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 194.55, "completions/mean_terminated_length": 194.55, "completions/min_length": 116.6, "completions/min_terminated_length": 116.6, "entropy": 1.0595195814967155, "epoch": 1.3082651466396984, "frac_reward_zero_std": 0.8, "grad_norm": 0.274580717086792, "kl": 0.012063133588526398, "learning_rate": 3.2248251755835735e-06, "loss": 0.0005, "num_tokens": 62048024.0, "reward": -0.05, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2731210589408875, "sampling/importance_sampling_ratio/mean": 1.000289499759674, "sampling/importance_sampling_ratio/min": 0.7510496079921722, "sampling/sampling_logp_difference/max": 0.30295934677124026, "sampling/sampling_logp_difference/mean": 0.022165285982191564, "step": 14230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.8, "completions/max_terminated_length": 290.8, "completions/mean_length": 205.2125, "completions/mean_terminated_length": 205.2125, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0766675002872943, "epoch": 1.3091845177898318, "frac_reward_zero_std": 0.5, "grad_norm": 5.417413711547852, "kl": 0.010661420662654564, "learning_rate": 3.2224000042292785e-06, "loss": 0.0002, "num_tokens": 62089809.0, "reward": 0.1, "reward_std": 0.4881446659564972, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.48814470171928404, "sampling/importance_sampling_ratio/max": 1.2281319141387939, "sampling/importance_sampling_ratio/mean": 0.9996682107448578, "sampling/importance_sampling_ratio/min": 0.8007112920284272, "sampling/sampling_logp_difference/max": 0.23069701194763184, "sampling/sampling_logp_difference/mean": 0.021652900613844395, "step": 14240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.0, "completions/max_terminated_length": 283.2, "completions/mean_length": 203.2875, "completions/mean_terminated_length": 199.1250015258789, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.0820331588387488, "epoch": 1.310103888939965, "frac_reward_zero_std": 0.3, "grad_norm": 5.543617248535156, "kl": 0.011681779479840771, "learning_rate": 3.2199740909122713e-06, "loss": 0.0007, "num_tokens": 62131752.0, "reward": 0.31875, "reward_std": 0.6035167157649994, "rewards/reward_fn/mean": 0.31875, "rewards/reward_fn/std": 0.6035167336463928, "sampling/importance_sampling_ratio/max": 1.2495511770248413, "sampling/importance_sampling_ratio/mean": 0.9995479822158814, "sampling/importance_sampling_ratio/min": 0.7926664054393768, "sampling/sampling_logp_difference/max": 0.2543695569038391, "sampling/sampling_logp_difference/mean": 0.021540283411741256, "step": 14250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.2, "completions/max_terminated_length": 255.2, "completions/mean_length": 185.7, "completions/mean_terminated_length": 185.7, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.0913464926183223, "epoch": 1.3110232600900984, "frac_reward_zero_std": 0.4, "grad_norm": 4.587975025177002, "kl": 0.013440577016444877, "learning_rate": 3.2175474381241602e-06, "loss": 0.0006, "num_tokens": 62175200.0, "reward": 0.15, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.2891609072685242, "sampling/importance_sampling_ratio/mean": 1.000155156850815, "sampling/importance_sampling_ratio/min": 0.7916574656963349, "sampling/sampling_logp_difference/max": 0.27847790718078613, "sampling/sampling_logp_difference/mean": 0.022624258138239383, "step": 14260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.0, "completions/max_terminated_length": 296.0, "completions/mean_length": 211.125, "completions/mean_terminated_length": 211.125, "completions/min_length": 149.7, "completions/min_terminated_length": 149.7, "entropy": 1.0304478786885738, "epoch": 1.3119426312402318, "frac_reward_zero_std": 0.6, "grad_norm": 0.09758010506629944, "kl": 0.015143390058074146, "learning_rate": 3.2151200483573118e-06, "loss": 0.0008, "num_tokens": 62218010.0, "reward": -0.275, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2626612067222596, "sampling/importance_sampling_ratio/mean": 1.0000301063060761, "sampling/importance_sampling_ratio/min": 0.72094686627388, "sampling/sampling_logp_difference/max": 0.36471153497695924, "sampling/sampling_logp_difference/mean": 0.020853855460882188, "step": 14270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 342.5, "completions/max_terminated_length": 315.8, "completions/mean_length": 231.5, "completions/mean_terminated_length": 227.59821472167968, "completions/min_length": 149.3, "completions/min_terminated_length": 149.3, "entropy": 1.0547644719481468, "epoch": 1.312862002390365, "frac_reward_zero_std": 0.5, "grad_norm": 0.09591091424226761, "kl": 0.013719870496424846, "learning_rate": 3.2126919241048492e-06, "loss": 0.0004, "num_tokens": 62259354.0, "reward": -0.18125, "reward_std": 0.401882404088974, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.4018824100494385, "sampling/importance_sampling_ratio/max": 1.2877193212509155, "sampling/importance_sampling_ratio/mean": 1.0001699209213257, "sampling/importance_sampling_ratio/min": 0.7413520663976669, "sampling/sampling_logp_difference/max": 0.3417306661605835, "sampling/sampling_logp_difference/mean": 0.021895259991288184, "step": 14280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 222.1125, "completions/mean_terminated_length": 222.1125, "completions/min_length": 158.9, "completions/min_terminated_length": 158.9, "entropy": 1.0410067677497863, "epoch": 1.3137813735404982, "frac_reward_zero_std": 0.3, "grad_norm": 9.458206176757812, "kl": 0.011236741591710597, "learning_rate": 3.2102630678606523e-06, "loss": 0.0006, "num_tokens": 62305635.0, "reward": 0.025, "reward_std": 0.58583904504776, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.585839056968689, "sampling/importance_sampling_ratio/max": 1.3842050075531005, "sampling/importance_sampling_ratio/mean": 1.0003475904464723, "sampling/importance_sampling_ratio/min": 0.7576281309127808, "sampling/sampling_logp_difference/max": 0.4435335874557495, "sampling/sampling_logp_difference/mean": 0.021046116575598718, "step": 14290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.2, "completions/max_terminated_length": 316.9, "completions/mean_length": 246.4375, "completions/mean_terminated_length": 245.0875030517578, "completions/min_length": 168.9, "completions/min_terminated_length": 168.9, "entropy": 1.1017370097339154, "epoch": 1.3147007446906316, "frac_reward_zero_std": 0.4, "grad_norm": 0.15540646016597748, "kl": 0.011680907325353474, "learning_rate": 3.2078334821193496e-06, "loss": 0.0007, "num_tokens": 62350310.0, "reward": -0.20625, "reward_std": 0.538993501663208, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.5389935314655304, "sampling/importance_sampling_ratio/max": 1.291669499874115, "sampling/importance_sampling_ratio/mean": 0.999974662065506, "sampling/importance_sampling_ratio/min": 0.7768973708152771, "sampling/sampling_logp_difference/max": 0.29118822813034057, "sampling/sampling_logp_difference/mean": 0.021826916188001633, "step": 14300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.4, "completions/max_terminated_length": 311.4, "completions/mean_length": 216.15, "completions/mean_terminated_length": 216.15, "completions/min_length": 156.4, "completions/min_terminated_length": 156.4, "entropy": 1.0752815622836351, "epoch": 1.315620115840765, "frac_reward_zero_std": 0.3, "grad_norm": 4.2370195388793945, "kl": 0.011499467722023838, "learning_rate": 3.20540316937632e-06, "loss": 0.0008, "num_tokens": 62395698.0, "reward": -0.175, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.2959213495254516, "sampling/importance_sampling_ratio/mean": 1.000348037481308, "sampling/importance_sampling_ratio/min": 0.7803402364253997, "sampling/sampling_logp_difference/max": 0.2787236928939819, "sampling/sampling_logp_difference/mean": 0.021166897471994163, "step": 14310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 232.775, "completions/mean_terminated_length": 232.775, "completions/min_length": 149.7, "completions/min_terminated_length": 149.7, "entropy": 1.076864490658045, "epoch": 1.3165394869908982, "frac_reward_zero_std": 0.4, "grad_norm": 0.09231948852539062, "kl": 0.010953745094593615, "learning_rate": 3.2029721321276917e-06, "loss": 0.0003, "num_tokens": 62442400.0, "reward": -0.35, "reward_std": 0.5445485532283783, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5445485651493073, "sampling/importance_sampling_ratio/max": 1.2902247667312623, "sampling/importance_sampling_ratio/mean": 0.9999490082263947, "sampling/importance_sampling_ratio/min": 0.7836666762828827, "sampling/sampling_logp_difference/max": 0.28277337551116943, "sampling/sampling_logp_difference/mean": 0.021295712143182755, "step": 14320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 201.85, "completions/mean_terminated_length": 201.85, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.0534234084188938, "epoch": 1.3174588581410316, "frac_reward_zero_std": 0.5, "grad_norm": 5.603939056396484, "kl": 0.010939880914520472, "learning_rate": 3.200540372870333e-06, "loss": 0.0008, "num_tokens": 62486244.0, "reward": -0.2, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.304901158809662, "sampling/importance_sampling_ratio/mean": 1.0003602266311646, "sampling/importance_sampling_ratio/min": 0.7823372721672058, "sampling/sampling_logp_difference/max": 0.2918186902999878, "sampling/sampling_logp_difference/mean": 0.02095262799412012, "step": 14330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 200.8625, "completions/mean_terminated_length": 200.8625, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0097220785915852, "epoch": 1.3183782292911648, "frac_reward_zero_std": 0.4, "grad_norm": 5.613681316375732, "kl": 0.015020327951060609, "learning_rate": 3.1981078941018574e-06, "loss": 0.0006, "num_tokens": 62528873.0, "reward": 0.125, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.254827857017517, "sampling/importance_sampling_ratio/mean": 0.9999096393585205, "sampling/importance_sampling_ratio/min": 0.7717343389987945, "sampling/sampling_logp_difference/max": 0.285428512096405, "sampling/sampling_logp_difference/mean": 0.020681513659656047, "step": 14340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.0, "completions/max_terminated_length": 326.0, "completions/mean_length": 231.3875, "completions/mean_terminated_length": 231.3875, "completions/min_length": 170.9, "completions/min_terminated_length": 170.9, "entropy": 1.1470089316368104, "epoch": 1.3192976004412982, "frac_reward_zero_std": 0.6, "grad_norm": 4.910032749176025, "kl": 0.011947593255899846, "learning_rate": 3.1956746983206135e-06, "loss": 0.0005, "num_tokens": 62580032.0, "reward": 0.2, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.256103503704071, "sampling/importance_sampling_ratio/mean": 0.9997960507869721, "sampling/importance_sampling_ratio/min": 0.7224022969603539, "sampling/sampling_logp_difference/max": 0.38388772010803224, "sampling/sampling_logp_difference/mean": 0.0222330529242754, "step": 14350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.7, "completions/max_terminated_length": 328.7, "completions/mean_length": 229.1125, "completions/mean_terminated_length": 229.1125, "completions/min_length": 151.9, "completions/min_terminated_length": 151.9, "entropy": 1.0580030962824822, "epoch": 1.3202169715914316, "frac_reward_zero_std": 0.2, "grad_norm": 4.650418758392334, "kl": 0.012584863509982825, "learning_rate": 3.1932407880256887e-06, "loss": 0.0003, "num_tokens": 62624321.0, "reward": -0.05, "reward_std": 0.6531707525253296, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6531707525253296, "sampling/importance_sampling_ratio/max": 1.257939064502716, "sampling/importance_sampling_ratio/mean": 1.0002426326274871, "sampling/importance_sampling_ratio/min": 0.7104125365614891, "sampling/sampling_logp_difference/max": 0.4144736647605896, "sampling/sampling_logp_difference/mean": 0.02106559704989195, "step": 14360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.9, "completions/max_terminated_length": 315.9, "completions/mean_length": 210.4625, "completions/mean_terminated_length": 210.4625, "completions/min_length": 121.9, "completions/min_terminated_length": 121.9, "entropy": 1.1366039611399175, "epoch": 1.3211363427415648, "frac_reward_zero_std": 0.4, "grad_norm": 8.865973472595215, "kl": 0.011421188578242435, "learning_rate": 3.190806165716905e-06, "loss": 0.0004, "num_tokens": 62667798.0, "reward": -0.35, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.2847450375556946, "sampling/importance_sampling_ratio/mean": 1.000437581539154, "sampling/importance_sampling_ratio/min": 0.7549129068851471, "sampling/sampling_logp_difference/max": 0.29540286064147947, "sampling/sampling_logp_difference/mean": 0.02215249855071306, "step": 14370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 312.1, "completions/max_terminated_length": 301.7, "completions/mean_length": 200.2375, "completions/mean_terminated_length": 192.7675018310547, "completions/min_length": 102.3, "completions/min_terminated_length": 102.3, "entropy": 1.0965386517345905, "epoch": 1.322055713891698, "frac_reward_zero_std": 0.3, "grad_norm": 5.654280662536621, "kl": 0.012664362776558847, "learning_rate": 3.188370833894815e-06, "loss": 0.0008, "num_tokens": 62708233.0, "reward": -0.025, "reward_std": 0.6341940701007843, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6341940879821777, "sampling/importance_sampling_ratio/max": 1.3124969124794006, "sampling/importance_sampling_ratio/mean": 0.999909770488739, "sampling/importance_sampling_ratio/min": 0.7159189194440841, "sampling/sampling_logp_difference/max": 0.4187240242958069, "sampling/sampling_logp_difference/mean": 0.02257648780941963, "step": 14380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 353.3, "completions/max_terminated_length": 333.2, "completions/mean_length": 228.4, "completions/mean_terminated_length": 225.13392944335936, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0502663679420947, "epoch": 1.3229750850418314, "frac_reward_zero_std": 0.6, "grad_norm": 0.10182666033506393, "kl": 0.012932244956027716, "learning_rate": 3.185934795060697e-06, "loss": 0.0004, "num_tokens": 62755993.0, "reward": -0.08125, "reward_std": 0.36736554503440855, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.36736555099487306, "sampling/importance_sampling_ratio/max": 1.277195692062378, "sampling/importance_sampling_ratio/mean": 0.9996980369091034, "sampling/importance_sampling_ratio/min": 0.7547052681446076, "sampling/sampling_logp_difference/max": 0.3007274389266968, "sampling/sampling_logp_difference/mean": 0.02122058942914009, "step": 14390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.4, "completions/max_terminated_length": 334.4, "completions/mean_length": 210.7625, "completions/mean_terminated_length": 210.7625, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.071439293026924, "epoch": 1.3238944561919648, "frac_reward_zero_std": 0.6, "grad_norm": 5.990674018859863, "kl": 0.012388462404487655, "learning_rate": 3.1834980517165614e-06, "loss": 0.0007, "num_tokens": 62798654.0, "reward": -0.075, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2471033811569214, "sampling/importance_sampling_ratio/mean": 0.9999261140823364, "sampling/importance_sampling_ratio/min": 0.7784325778484344, "sampling/sampling_logp_difference/max": 0.2807926177978516, "sampling/sampling_logp_difference/mean": 0.021344858035445214, "step": 14400 }, { "epoch": 1.3238944561919648, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 305.42582897033157, "eval_completions/max_terminated_length": 302.9825479930192, "eval_completions/mean_length": 209.48625654450262, "eval_completions/mean_terminated_length": 208.80030561783343, "eval_completions/min_length": 132.92321116928446, "eval_completions/min_terminated_length": 132.92321116928446, "eval_entropy": 1.0913914220495373, "eval_frac_reward_zero_std": 0.4851657940663176, "eval_kl": 0.01177816199103003, "eval_loss": 0.0006227299454621971, "eval_num_tokens": 62798654.0, "eval_reward": -0.10972949389179755, "eval_reward_std": 0.45642491961856996, "eval_rewards/reward_fn/mean": -0.10972949389179755, "eval_rewards/reward_fn/std": 0.4564249344937256, "eval_runtime": 1771.3089, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2796190592214907, "eval_sampling/importance_sampling_ratio/mean": 1.000061584080701, "eval_sampling/importance_sampling_ratio/min": 0.7690541311240321, "eval_sampling/sampling_logp_difference/max": 0.29563965764137257, "eval_sampling/sampling_logp_difference/mean": 0.021371343472150087, "eval_steps_per_second": 0.041, "step": 14400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.3, "completions/max_terminated_length": 303.3, "completions/mean_length": 217.3, "completions/mean_terminated_length": 217.3, "completions/min_length": 148.7, "completions/min_terminated_length": 148.7, "entropy": 1.0140181809663773, "epoch": 1.324813827342098, "frac_reward_zero_std": 0.1, "grad_norm": 5.929305553436279, "kl": 0.011770680180052295, "learning_rate": 3.1810606063651372e-06, "loss": 0.0007, "num_tokens": 62842942.0, "reward": 0.025, "reward_std": 0.78532555103302, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.785325562953949, "sampling/importance_sampling_ratio/max": 1.3586744666099548, "sampling/importance_sampling_ratio/mean": 1.0003777861595153, "sampling/importance_sampling_ratio/min": 0.7917648375034332, "sampling/sampling_logp_difference/max": 0.32024872303009033, "sampling/sampling_logp_difference/mean": 0.020951391011476517, "step": 14410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 202.1125, "completions/mean_terminated_length": 202.1125, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.0899024672806263, "epoch": 1.3257331984922314, "frac_reward_zero_std": 0.1, "grad_norm": 6.067655563354492, "kl": 0.014416837022872641, "learning_rate": 3.178622461509877e-06, "loss": 0.0006, "num_tokens": 62883207.0, "reward": 0.225, "reward_std": 0.8761898577213287, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.8761898815631867, "sampling/importance_sampling_ratio/max": 1.3501186609268188, "sampling/importance_sampling_ratio/mean": 1.0002654254436494, "sampling/importance_sampling_ratio/min": 0.7680778920650482, "sampling/sampling_logp_difference/max": 0.33954448699951173, "sampling/sampling_logp_difference/mean": 0.021849639154970645, "step": 14420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 203.2625, "completions/mean_terminated_length": 203.2625, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.123717726022005, "epoch": 1.3266525696423646, "frac_reward_zero_std": 0.5, "grad_norm": 5.146963596343994, "kl": 0.01228854501969181, "learning_rate": 3.176183619654951e-06, "loss": 0.001, "num_tokens": 62928156.0, "reward": 0.125, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2559892773628234, "sampling/importance_sampling_ratio/mean": 0.9995297253131866, "sampling/importance_sampling_ratio/min": 0.7190466165542603, "sampling/sampling_logp_difference/max": 0.3345257997512817, "sampling/sampling_logp_difference/mean": 0.02233971282839775, "step": 14430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.5, "completions/max_terminated_length": 314.5, "completions/mean_length": 203.75, "completions/mean_terminated_length": 203.75, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 1.1062382958829402, "epoch": 1.327571940792498, "frac_reward_zero_std": 0.4, "grad_norm": 8.945001602172852, "kl": 0.014091018319595605, "learning_rate": 3.1737440833052437e-06, "loss": 0.001, "num_tokens": 62968696.0, "reward": -0.08125, "reward_std": 0.5448542714118958, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5448542833328247, "sampling/importance_sampling_ratio/max": 1.2738448143005372, "sampling/importance_sampling_ratio/mean": 0.9996927082538605, "sampling/importance_sampling_ratio/min": 0.7587066948413849, "sampling/sampling_logp_difference/max": 0.31525429487228396, "sampling/sampling_logp_difference/mean": 0.02233581580221653, "step": 14440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.0, "completions/max_terminated_length": 300.0, "completions/mean_length": 199.825, "completions/mean_terminated_length": 199.825, "completions/min_length": 123.6, "completions/min_terminated_length": 123.6, "entropy": 1.1508255250751973, "epoch": 1.3284913119426314, "frac_reward_zero_std": 0.5, "grad_norm": 5.2638092041015625, "kl": 0.011217901448253543, "learning_rate": 3.1713038549663564e-06, "loss": 0.0004, "num_tokens": 63011258.0, "reward": 0.1, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2979517579078674, "sampling/importance_sampling_ratio/mean": 1.0000681936740876, "sampling/importance_sampling_ratio/min": 0.7974397003650665, "sampling/sampling_logp_difference/max": 0.276339590549469, "sampling/sampling_logp_difference/mean": 0.02198922522366047, "step": 14450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.9, "completions/max_terminated_length": 284.9, "completions/mean_length": 180.8875, "completions/mean_terminated_length": 180.8875, "completions/min_length": 100.7, "completions/min_terminated_length": 100.7, "entropy": 0.8611604876816272, "epoch": 1.3294106830927646, "frac_reward_zero_std": 0.5, "grad_norm": 0.16677837073802948, "kl": 0.01363138488959521, "learning_rate": 3.1688629371445982e-06, "loss": 0.0007, "num_tokens": 63049233.0, "reward": 0.1, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.249489676952362, "sampling/importance_sampling_ratio/mean": 1.0002849340438842, "sampling/importance_sampling_ratio/min": 0.77080796957016, "sampling/sampling_logp_difference/max": 0.2799820899963379, "sampling/sampling_logp_difference/mean": 0.018710826244205236, "step": 14460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.0, "completions/max_terminated_length": 288.8, "completions/mean_length": 193.9875, "completions/mean_terminated_length": 189.90357208251953, "completions/min_length": 110.9, "completions/min_terminated_length": 110.9, "entropy": 1.0475920952856541, "epoch": 1.3303300542428977, "frac_reward_zero_std": 0.3, "grad_norm": 0.12670843303203583, "kl": 0.010985177499242127, "learning_rate": 3.1664213323469876e-06, "loss": 0.0006, "num_tokens": 63090936.0, "reward": 0.19375, "reward_std": 0.6155649483203888, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.6155649602413178, "sampling/importance_sampling_ratio/max": 1.2539703369140625, "sampling/importance_sampling_ratio/mean": 1.0004190683364869, "sampling/importance_sampling_ratio/min": 0.8118577361106872, "sampling/sampling_logp_difference/max": 0.25086426734924316, "sampling/sampling_logp_difference/mean": 0.021410688012838363, "step": 14470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 192.1, "completions/mean_terminated_length": 192.1, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 0.9883208237588406, "epoch": 1.3312494253930311, "frac_reward_zero_std": 0.7, "grad_norm": 0.09566178917884827, "kl": 0.0118029738398036, "learning_rate": 3.163979043081249e-06, "loss": 0.0, "num_tokens": 63129864.0, "reward": 0.475, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2462343215942382, "sampling/importance_sampling_ratio/mean": 1.000136399269104, "sampling/importance_sampling_ratio/min": 0.7585723280906678, "sampling/sampling_logp_difference/max": 0.28792176246643064, "sampling/sampling_logp_difference/mean": 0.020272728987038136, "step": 14480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.0, "completions/max_terminated_length": 245.0, "completions/mean_length": 177.9875, "completions/mean_terminated_length": 177.9875, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.0478151381015777, "epoch": 1.3321687965431646, "frac_reward_zero_std": 0.3, "grad_norm": 6.4561309814453125, "kl": 0.011273219215217978, "learning_rate": 3.161536071855808e-06, "loss": 0.0009, "num_tokens": 63170167.0, "reward": -0.2, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.3237234354019165, "sampling/importance_sampling_ratio/mean": 1.0002945601940154, "sampling/importance_sampling_ratio/min": 0.7924952387809754, "sampling/sampling_logp_difference/max": 0.31205384731292723, "sampling/sampling_logp_difference/mean": 0.02158548105508089, "step": 14490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.8, "completions/max_terminated_length": 269.8, "completions/mean_length": 189.4125, "completions/mean_terminated_length": 189.4125, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.033185612410307, "epoch": 1.3330881676932977, "frac_reward_zero_std": 0.4, "grad_norm": 5.317416191101074, "kl": 0.013318761240225286, "learning_rate": 3.159092421179793e-06, "loss": 0.0008, "num_tokens": 63210936.0, "reward": 0.3, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.3593466758728028, "sampling/importance_sampling_ratio/mean": 0.9995911478996277, "sampling/importance_sampling_ratio/min": 0.7349961042404175, "sampling/sampling_logp_difference/max": 0.3925474166870117, "sampling/sampling_logp_difference/mean": 0.021977507323026658, "step": 14500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.9, "completions/max_terminated_length": 300.9, "completions/mean_length": 211.6125, "completions/mean_terminated_length": 211.6125, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0217561833560467, "epoch": 1.3340075388434312, "frac_reward_zero_std": 0.7, "grad_norm": 0.0949326679110527, "kl": 0.012108816724503414, "learning_rate": 3.1566480935630286e-06, "loss": 0.0005, "num_tokens": 63256065.0, "reward": 0.3, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2663405895233155, "sampling/importance_sampling_ratio/mean": 1.0004772782325744, "sampling/importance_sampling_ratio/min": 0.7731914699077607, "sampling/sampling_logp_difference/max": 0.2687946557998657, "sampling/sampling_logp_difference/mean": 0.020708634331822396, "step": 14510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 332.6, "completions/max_terminated_length": 313.4, "completions/mean_length": 223.0875, "completions/mean_terminated_length": 219.8982147216797, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.106456422060728, "epoch": 1.3349269099935643, "frac_reward_zero_std": 0.3, "grad_norm": 4.669721603393555, "kl": 0.016984482057159765, "learning_rate": 3.154203091516035e-06, "loss": 0.0008, "num_tokens": 63298936.0, "reward": -0.35, "reward_std": 0.6043314099311828, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.6043314099311828, "sampling/importance_sampling_ratio/max": 1.3217129230499267, "sampling/importance_sampling_ratio/mean": 0.9996466219425202, "sampling/importance_sampling_ratio/min": 0.7904131770133972, "sampling/sampling_logp_difference/max": 0.332375293970108, "sampling/sampling_logp_difference/mean": 0.021818508207798005, "step": 14520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 210.8125, "completions/mean_terminated_length": 210.8125, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0853372931480407, "epoch": 1.3358462811436977, "frac_reward_zero_std": 0.5, "grad_norm": 6.267653942108154, "kl": 0.011310172459343449, "learning_rate": 3.151757417550026e-06, "loss": 0.0006, "num_tokens": 63341305.0, "reward": 0.175, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.28044650554657, "sampling/importance_sampling_ratio/mean": 1.0002282917499543, "sampling/importance_sampling_ratio/min": 0.7821170389652252, "sampling/sampling_logp_difference/max": 0.2633053779602051, "sampling/sampling_logp_difference/mean": 0.021527918800711632, "step": 14530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 211.9375, "completions/mean_terminated_length": 211.9375, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.0606992825865746, "epoch": 1.3367656522938312, "frac_reward_zero_std": 0.3, "grad_norm": 5.771407604217529, "kl": 0.010904866911005229, "learning_rate": 3.149311074176905e-06, "loss": 0.0005, "num_tokens": 63386956.0, "reward": -0.575, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2633566021919251, "sampling/importance_sampling_ratio/mean": 1.000327754020691, "sampling/importance_sampling_ratio/min": 0.7890272557735443, "sampling/sampling_logp_difference/max": 0.26050821542739866, "sampling/sampling_logp_difference/mean": 0.021691012568771838, "step": 14540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.7, "completions/max_terminated_length": 333.7, "completions/mean_length": 224.1375, "completions/mean_terminated_length": 224.1375, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0326946273446083, "epoch": 1.3376850234439643, "frac_reward_zero_std": 0.5, "grad_norm": 0.08880075812339783, "kl": 0.013903027167543769, "learning_rate": 3.1468640639092607e-06, "loss": 0.0009, "num_tokens": 63427639.0, "reward": 0.05, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.3014374017715453, "sampling/importance_sampling_ratio/mean": 0.9998917520046234, "sampling/importance_sampling_ratio/min": 0.7685854434967041, "sampling/sampling_logp_difference/max": 0.3122477114200592, "sampling/sampling_logp_difference/mean": 0.02050762940198183, "step": 14550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.8, "completions/max_terminated_length": 277.8, "completions/mean_length": 195.1625, "completions/mean_terminated_length": 195.1625, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.0365350849926471, "epoch": 1.3386043945940975, "frac_reward_zero_std": 0.3, "grad_norm": 6.149552822113037, "kl": 0.013045036920811982, "learning_rate": 3.144416389260372e-06, "loss": 0.0008, "num_tokens": 63467620.0, "reward": 0.05, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2669656038284303, "sampling/importance_sampling_ratio/mean": 0.9996731996536254, "sampling/importance_sampling_ratio/min": 0.7963317632675171, "sampling/sampling_logp_difference/max": 0.24952008724212646, "sampling/sampling_logp_difference/mean": 0.021022761426866054, "step": 14560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 208.0125, "completions/mean_terminated_length": 208.0125, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.0596942558884621, "epoch": 1.339523765744231, "frac_reward_zero_std": 0.6, "grad_norm": 0.15163029730319977, "kl": 0.011306585132842883, "learning_rate": 3.141968052744194e-06, "loss": 0.0004, "num_tokens": 63511685.0, "reward": 0.45, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2713839530944824, "sampling/importance_sampling_ratio/mean": 0.9999207496643067, "sampling/importance_sampling_ratio/min": 0.769489872455597, "sampling/sampling_logp_difference/max": 0.2941024422645569, "sampling/sampling_logp_difference/mean": 0.021470477059483527, "step": 14570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 194.75, "completions/mean_terminated_length": 194.75, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.1168655514717103, "epoch": 1.3404431368943643, "frac_reward_zero_std": 0.6, "grad_norm": 0.10008212924003601, "kl": 0.01183816441334784, "learning_rate": 3.1395190568753665e-06, "loss": 0.0006, "num_tokens": 63554025.0, "reward": -0.55, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2836973190307617, "sampling/importance_sampling_ratio/mean": 0.9998385429382324, "sampling/importance_sampling_ratio/min": 0.7883006751537323, "sampling/sampling_logp_difference/max": 0.26813330352306364, "sampling/sampling_logp_difference/mean": 0.0224718427285552, "step": 14580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 222.225, "completions/mean_terminated_length": 222.225, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.121667604148388, "epoch": 1.3413625080444975, "frac_reward_zero_std": 0.6, "grad_norm": 0.11574570834636688, "kl": 0.011753227229928599, "learning_rate": 3.137069404169203e-06, "loss": 0.0007, "num_tokens": 63599403.0, "reward": -0.675, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.675, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.2532809376716614, "sampling/importance_sampling_ratio/mean": 0.999408507347107, "sampling/importance_sampling_ratio/min": 0.7624008417129516, "sampling/sampling_logp_difference/max": 0.28479661643505094, "sampling/sampling_logp_difference/mean": 0.021180910430848597, "step": 14590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.1, "completions/max_terminated_length": 289.6, "completions/mean_length": 209.6375, "completions/mean_terminated_length": 205.69285736083984, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.088520810753107, "epoch": 1.342281879194631, "frac_reward_zero_std": 0.5, "grad_norm": 5.968191623687744, "kl": 0.010464814203442075, "learning_rate": 3.1346190971416944e-06, "loss": 0.0005, "num_tokens": 63644574.0, "reward": -0.30625, "reward_std": 0.47167748808860777, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.47167750597000124, "sampling/importance_sampling_ratio/max": 1.313251554965973, "sampling/importance_sampling_ratio/mean": 1.000597846508026, "sampling/importance_sampling_ratio/min": 0.8074646949768066, "sampling/sampling_logp_difference/max": 0.2736684322357178, "sampling/sampling_logp_difference/mean": 0.021345413103699685, "step": 14600 }, { "epoch": 1.342281879194631, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 297.64397905759165, "eval_completions/max_terminated_length": 296.05584642233856, "eval_completions/mean_length": 207.19480802792322, "eval_completions/mean_terminated_length": 206.82339203253673, "eval_completions/min_length": 133.54275741710296, "eval_completions/min_terminated_length": 133.54275741710296, "eval_entropy": 1.0741599658812944, "eval_frac_reward_zero_std": 0.4712041884816754, "eval_kl": 0.012433500575262527, "eval_loss": 0.0006397510296665132, "eval_num_tokens": 63644574.0, "eval_reward": -0.12118237347294938, "eval_reward_std": 0.46692687917962333, "eval_rewards/reward_fn/mean": -0.12118237347294938, "eval_rewards/reward_fn/std": 0.46692689353466865, "eval_runtime": 1730.8984, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2810541555727548, "eval_sampling/importance_sampling_ratio/mean": 1.0000465432178705, "eval_sampling/importance_sampling_ratio/min": 0.7654450816113704, "eval_sampling/sampling_logp_difference/max": 0.2976292290092555, "eval_sampling/sampling_logp_difference/mean": 0.02143383528192058, "eval_steps_per_second": 0.042, "step": 14600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.7, "completions/max_terminated_length": 304.7, "completions/mean_length": 212.4625, "completions/mean_terminated_length": 212.4625, "completions/min_length": 129.7, "completions/min_terminated_length": 129.7, "entropy": 1.0435392811894417, "epoch": 1.3432012503447641, "frac_reward_zero_std": 0.4, "grad_norm": 0.08600655943155289, "kl": 0.013560819334816187, "learning_rate": 3.1321681383095025e-06, "loss": 0.0005, "num_tokens": 63686419.0, "reward": -0.3, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.3558909893035889, "sampling/importance_sampling_ratio/mean": 1.0003706157207488, "sampling/importance_sampling_ratio/min": 0.7566747844219208, "sampling/sampling_logp_difference/max": 0.352947735786438, "sampling/sampling_logp_difference/mean": 0.02131558917462826, "step": 14610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.8, "completions/max_terminated_length": 276.8, "completions/mean_length": 190.2875, "completions/mean_terminated_length": 190.2875, "completions/min_length": 120.2, "completions/min_terminated_length": 120.2, "entropy": 1.1066793166100979, "epoch": 1.3441206214948975, "frac_reward_zero_std": 0.2, "grad_norm": 9.92818546295166, "kl": 0.014365515823010355, "learning_rate": 3.129716530189958e-06, "loss": 0.0006, "num_tokens": 63726618.0, "reward": -0.25, "reward_std": 0.7877463400363922, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.787746399641037, "sampling/importance_sampling_ratio/max": 1.2819572925567626, "sampling/importance_sampling_ratio/mean": 0.9999657273292542, "sampling/importance_sampling_ratio/min": 0.7961998522281647, "sampling/sampling_logp_difference/max": 0.263972333073616, "sampling/sampling_logp_difference/mean": 0.022416000440716743, "step": 14620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.6, "completions/max_terminated_length": 318.6, "completions/mean_length": 225.2, "completions/mean_terminated_length": 225.2, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.1419359877705575, "epoch": 1.345039992645031, "frac_reward_zero_std": 0.6, "grad_norm": 5.477534294128418, "kl": 0.011415231737191788, "learning_rate": 3.1272642753010586e-06, "loss": 0.0009, "num_tokens": 63772138.0, "reward": -0.55, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.287357258796692, "sampling/importance_sampling_ratio/mean": 1.0004365265369415, "sampling/importance_sampling_ratio/min": 0.7874163746833801, "sampling/sampling_logp_difference/max": 0.2754916071891785, "sampling/sampling_logp_difference/mean": 0.021946469135582448, "step": 14630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 305.4, "completions/max_terminated_length": 302.7, "completions/mean_length": 211.675, "completions/mean_terminated_length": 209.70000305175782, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 1.0682233475148677, "epoch": 1.3459593637951641, "frac_reward_zero_std": 0.6, "grad_norm": 7.061047077178955, "kl": 0.011106802296126261, "learning_rate": 3.1248113761614672e-06, "loss": 0.0008, "num_tokens": 63817496.0, "reward": 0.39375, "reward_std": 0.3661186873912811, "rewards/reward_fn/mean": 0.39375, "rewards/reward_fn/std": 0.3661187171936035, "sampling/importance_sampling_ratio/max": 1.2381195425987244, "sampling/importance_sampling_ratio/mean": 0.9999967694282532, "sampling/importance_sampling_ratio/min": 0.7671872973442078, "sampling/sampling_logp_difference/max": 0.2752948194742203, "sampling/sampling_logp_difference/mean": 0.021257864125072957, "step": 14640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 200.2625, "completions/mean_terminated_length": 200.2625, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.0910617470741273, "epoch": 1.3468787349452973, "frac_reward_zero_std": 0.3, "grad_norm": 0.0935269221663475, "kl": 0.012258883443428204, "learning_rate": 3.122357835290508e-06, "loss": 0.0008, "num_tokens": 63860141.0, "reward": -0.4, "reward_std": 0.6152435362339019, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6152435719966889, "sampling/importance_sampling_ratio/max": 1.3099632859230042, "sampling/importance_sampling_ratio/mean": 1.000043088197708, "sampling/importance_sampling_ratio/min": 0.7772800266742707, "sampling/sampling_logp_difference/max": 0.4265362977981567, "sampling/sampling_logp_difference/mean": 0.021220271289348603, "step": 14650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.4, "completions/max_terminated_length": 307.4, "completions/mean_length": 205.4, "completions/mean_terminated_length": 205.4, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.133359318971634, "epoch": 1.3477981060954307, "frac_reward_zero_std": 0.5, "grad_norm": 6.289811134338379, "kl": 0.011170743720140309, "learning_rate": 3.1199036552081636e-06, "loss": 0.0009, "num_tokens": 63905965.0, "reward": -0.425, "reward_std": 0.5243384838104248, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5243385195732116, "sampling/importance_sampling_ratio/max": 1.2875948667526245, "sampling/importance_sampling_ratio/mean": 1.0002829611301423, "sampling/importance_sampling_ratio/min": 0.7696056962013245, "sampling/sampling_logp_difference/max": 0.2948924958705902, "sampling/sampling_logp_difference/mean": 0.021601996943354608, "step": 14660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 196.725, "completions/mean_terminated_length": 196.725, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 1.074870329350233, "epoch": 1.3487174772455641, "frac_reward_zero_std": 0.5, "grad_norm": 0.17187434434890747, "kl": 0.011865350342122837, "learning_rate": 3.117448838435074e-06, "loss": 0.0003, "num_tokens": 63950999.0, "reward": 0.375, "reward_std": 0.49153932929039, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.49153935313224795, "sampling/importance_sampling_ratio/max": 1.2937021136283875, "sampling/importance_sampling_ratio/mean": 0.9996514260768891, "sampling/importance_sampling_ratio/min": 0.7608496189117432, "sampling/sampling_logp_difference/max": 0.31967306733131406, "sampling/sampling_logp_difference/mean": 0.021479593031108378, "step": 14670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.5, "completions/max_terminated_length": 253.5, "completions/mean_length": 186.1875, "completions/mean_terminated_length": 186.1875, "completions/min_length": 116.4, "completions/min_terminated_length": 116.4, "entropy": 0.9917876645922661, "epoch": 1.3496368483956973, "frac_reward_zero_std": 0.6, "grad_norm": 5.752249717712402, "kl": 0.011559840716654435, "learning_rate": 3.1149933874925332e-06, "loss": 0.0005, "num_tokens": 63993718.0, "reward": -0.3625, "reward_std": 0.35678728222846984, "rewards/reward_fn/mean": -0.3625, "rewards/reward_fn/std": 0.35678730607032777, "sampling/importance_sampling_ratio/max": 1.332026469707489, "sampling/importance_sampling_ratio/mean": 0.9995829224586487, "sampling/importance_sampling_ratio/min": 0.7587651610374451, "sampling/sampling_logp_difference/max": 0.2896604657173157, "sampling/sampling_logp_difference/mean": 0.0210218895226717, "step": 14680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.0, "completions/max_terminated_length": 272.0, "completions/mean_length": 199.1, "completions/mean_terminated_length": 199.1, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.01904142126441, "epoch": 1.3505562195458307, "frac_reward_zero_std": 0.8, "grad_norm": 5.030760765075684, "kl": 0.011448762891814113, "learning_rate": 3.1125373049024853e-06, "loss": 0.0003, "num_tokens": 64035142.0, "reward": 0.425, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2544331550598145, "sampling/importance_sampling_ratio/mean": 1.0000637829303742, "sampling/importance_sampling_ratio/min": 0.7783570110797882, "sampling/sampling_logp_difference/max": 0.2603257417678833, "sampling/sampling_logp_difference/mean": 0.02063525728881359, "step": 14690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 338.5, "completions/max_terminated_length": 329.5, "completions/mean_length": 239.475, "completions/mean_terminated_length": 234.2452423095703, "completions/min_length": 153.3, "completions/min_terminated_length": 153.3, "entropy": 0.9598729334771633, "epoch": 1.351475590695964, "frac_reward_zero_std": 0.3, "grad_norm": 6.353200435638428, "kl": 0.01119887392851524, "learning_rate": 3.1100805931875245e-06, "loss": 0.0003, "num_tokens": 64079420.0, "reward": -0.09375, "reward_std": 0.5516500234603882, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.5516500294208526, "sampling/importance_sampling_ratio/max": 1.2782689809799195, "sampling/importance_sampling_ratio/mean": 0.9997480809688568, "sampling/importance_sampling_ratio/min": 0.79712393283844, "sampling/sampling_logp_difference/max": 0.26738537549972535, "sampling/sampling_logp_difference/mean": 0.019857877492904664, "step": 14700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.3, "completions/max_terminated_length": 340.3, "completions/mean_length": 241.3625, "completions/mean_terminated_length": 241.3625, "completions/min_length": 164.2, "completions/min_terminated_length": 164.2, "entropy": 1.1329584278166294, "epoch": 1.3523949618460973, "frac_reward_zero_std": 0.2, "grad_norm": 5.254015922546387, "kl": 0.013251332024810835, "learning_rate": 3.107623254870891e-06, "loss": 0.0004, "num_tokens": 64128673.0, "reward": 0.25, "reward_std": 0.7440350592136383, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.7440350711345672, "sampling/importance_sampling_ratio/max": 1.329101598262787, "sampling/importance_sampling_ratio/mean": 1.000012743473053, "sampling/importance_sampling_ratio/min": 0.7939182817935944, "sampling/sampling_logp_difference/max": 0.2979595124721527, "sampling/sampling_logp_difference/mean": 0.02152459919452667, "step": 14710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.8, "completions/max_terminated_length": 292.8, "completions/mean_length": 200.675, "completions/mean_terminated_length": 200.675, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.035744608193636, "epoch": 1.3533143329962305, "frac_reward_zero_std": 0.5, "grad_norm": 0.10834374278783798, "kl": 0.012018676981097088, "learning_rate": 3.1051652924764664e-06, "loss": 0.0008, "num_tokens": 64169631.0, "reward": 0.05, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.2942052602767944, "sampling/importance_sampling_ratio/mean": 1.0001553475856781, "sampling/importance_sampling_ratio/min": 0.766919982433319, "sampling/sampling_logp_difference/max": 0.31087130308151245, "sampling/sampling_logp_difference/mean": 0.02167601715773344, "step": 14720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 208.875, "completions/mean_terminated_length": 208.875, "completions/min_length": 153.7, "completions/min_terminated_length": 153.7, "entropy": 1.0382295325398445, "epoch": 1.354233704146364, "frac_reward_zero_std": 0.4, "grad_norm": 0.13987404108047485, "kl": 0.013935217744437978, "learning_rate": 3.102706708528778e-06, "loss": 0.0004, "num_tokens": 64212245.0, "reward": -0.475, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.2891043543815612, "sampling/importance_sampling_ratio/mean": 0.9999121129512787, "sampling/importance_sampling_ratio/min": 0.7080472588539124, "sampling/sampling_logp_difference/max": 0.4726685076951981, "sampling/sampling_logp_difference/mean": 0.02104929406195879, "step": 14730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 330.8, "completions/max_terminated_length": 320.6, "completions/mean_length": 223.25, "completions/mean_terminated_length": 220.33035888671876, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.9926737412810326, "epoch": 1.355153075296497, "frac_reward_zero_std": 0.3, "grad_norm": 0.19368959963321686, "kl": 0.011759428976802155, "learning_rate": 3.1002475055529857e-06, "loss": 0.0011, "num_tokens": 64258777.0, "reward": 0.09375, "reward_std": 0.7000576674938201, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.7000577032566071, "sampling/importance_sampling_ratio/max": 1.4064422607421876, "sampling/importance_sampling_ratio/mean": 1.000316071510315, "sampling/importance_sampling_ratio/min": 0.7774121761322021, "sampling/sampling_logp_difference/max": 0.34409194588661196, "sampling/sampling_logp_difference/mean": 0.019927106238901614, "step": 14740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 206.6, "completions/mean_terminated_length": 206.6, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.1050823144614697, "epoch": 1.3560724464466305, "frac_reward_zero_std": 0.7, "grad_norm": 0.10051756352186203, "kl": 0.0123239551554434, "learning_rate": 3.0977876860748902e-06, "loss": 0.0006, "num_tokens": 64301017.0, "reward": -0.725, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.725, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.3455296158790588, "sampling/importance_sampling_ratio/mean": 0.9995458781719208, "sampling/importance_sampling_ratio/min": 0.7845493018627167, "sampling/sampling_logp_difference/max": 0.34309934377670287, "sampling/sampling_logp_difference/mean": 0.02155642081052065, "step": 14750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.0, "completions/max_terminated_length": 277.0, "completions/mean_length": 201.425, "completions/mean_terminated_length": 201.425, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.0334228247404098, "epoch": 1.356991817596764, "frac_reward_zero_std": 0.6, "grad_norm": 0.19301918148994446, "kl": 0.011155624088132754, "learning_rate": 3.0953272526209234e-06, "loss": 0.0008, "num_tokens": 64341939.0, "reward": 0.075, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.246075987815857, "sampling/importance_sampling_ratio/mean": 0.9998162925243378, "sampling/importance_sampling_ratio/min": 0.7818760275840759, "sampling/sampling_logp_difference/max": 0.25206747055053713, "sampling/sampling_logp_difference/mean": 0.02093894425779581, "step": 14760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 238.7, "completions/max_terminated_length": 238.7, "completions/mean_length": 173.125, "completions/mean_terminated_length": 173.125, "completions/min_length": 117.5, "completions/min_terminated_length": 117.5, "entropy": 1.0047320626676082, "epoch": 1.357911188746897, "frac_reward_zero_std": 0.6, "grad_norm": 0.07719812542200089, "kl": 0.010097684938227759, "learning_rate": 3.092866207718145e-06, "loss": 0.0007, "num_tokens": 64382541.0, "reward": -0.225, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2644615054130555, "sampling/importance_sampling_ratio/mean": 0.9996980488300323, "sampling/importance_sampling_ratio/min": 0.7892553925514221, "sampling/sampling_logp_difference/max": 0.25957087278366087, "sampling/sampling_logp_difference/mean": 0.020705018006265164, "step": 14770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 211.5875, "completions/mean_terminated_length": 211.5875, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.1031139940023422, "epoch": 1.3588305598970305, "frac_reward_zero_std": 0.5, "grad_norm": 5.456112861633301, "kl": 0.01246676406590268, "learning_rate": 3.090404553894249e-06, "loss": 0.0004, "num_tokens": 64428092.0, "reward": 0.075, "reward_std": 0.5175491571426392, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5175492167472839, "sampling/importance_sampling_ratio/max": 1.312128448486328, "sampling/importance_sampling_ratio/mean": 1.0004497408866881, "sampling/importance_sampling_ratio/min": 0.7734354615211487, "sampling/sampling_logp_difference/max": 0.29684009552001955, "sampling/sampling_logp_difference/mean": 0.02197674885392189, "step": 14780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 199.8375, "completions/mean_terminated_length": 199.8375, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.015516524016857, "epoch": 1.3597499310471637, "frac_reward_zero_std": 0.4, "grad_norm": 5.690219879150391, "kl": 0.011406485608313232, "learning_rate": 3.087942293677548e-06, "loss": 0.0005, "num_tokens": 64470751.0, "reward": 0.24375, "reward_std": 0.4696442112326622, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.46964422315359117, "sampling/importance_sampling_ratio/max": 1.3481455326080323, "sampling/importance_sampling_ratio/mean": 0.9995192170143128, "sampling/importance_sampling_ratio/min": 0.7956470131874085, "sampling/sampling_logp_difference/max": 0.3522849440574646, "sampling/sampling_logp_difference/mean": 0.020577776804566385, "step": 14790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 210.1125, "completions/mean_terminated_length": 210.1125, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0231135368347168, "epoch": 1.360669302197297, "frac_reward_zero_std": 0.6, "grad_norm": 0.0946943610906601, "kl": 0.01159496902837418, "learning_rate": 3.085479429596983e-06, "loss": 0.0008, "num_tokens": 64511544.0, "reward": -0.025, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.268881368637085, "sampling/importance_sampling_ratio/mean": 0.9997024476528168, "sampling/importance_sampling_ratio/min": 0.7589158594608307, "sampling/sampling_logp_difference/max": 0.305277144908905, "sampling/sampling_logp_difference/mean": 0.020374444127082825, "step": 14800 }, { "epoch": 1.360669302197297, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 294.3019197207679, "eval_completions/max_terminated_length": 293.39092495637, "eval_completions/mean_length": 203.6609947643979, "eval_completions/mean_terminated_length": 203.2791595192688, "eval_completions/min_length": 131.8062827225131, "eval_completions/min_terminated_length": 131.8062827225131, "eval_entropy": 1.0734555668231704, "eval_frac_reward_zero_std": 0.4537521815008726, "eval_kl": 0.01160962168488125, "eval_loss": 0.0005574455135501921, "eval_num_tokens": 64511544.0, "eval_reward": -0.10754799301919721, "eval_reward_std": 0.4828484494492646, "eval_rewards/reward_fn/mean": -0.10754799301919721, "eval_rewards/reward_fn/std": 0.48284846775714846, "eval_runtime": 1717.0323, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2784302039088171, "eval_sampling/importance_sampling_ratio/mean": 0.999955930738965, "eval_sampling/importance_sampling_ratio/min": 0.7747947540764826, "eval_sampling/sampling_logp_difference/max": 0.29209029180425206, "eval_sampling/sampling_logp_difference/mean": 0.02088044767461418, "eval_steps_per_second": 0.042, "step": 14800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.2, "completions/max_terminated_length": 335.2, "completions/mean_length": 220.1375, "completions/mean_terminated_length": 220.1375, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.038219828903675, "epoch": 1.3615886733474303, "frac_reward_zero_std": 0.4, "grad_norm": 5.206982612609863, "kl": 0.014065252972068265, "learning_rate": 3.0830159641821115e-06, "loss": 0.0005, "num_tokens": 64556899.0, "reward": -0.2, "reward_std": 0.5664041936397552, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5664042294025421, "sampling/importance_sampling_ratio/max": 1.2689083099365235, "sampling/importance_sampling_ratio/mean": 1.0005907893180848, "sampling/importance_sampling_ratio/min": 0.7961064577102661, "sampling/sampling_logp_difference/max": 0.2564717173576355, "sampling/sampling_logp_difference/mean": 0.02100973520427942, "step": 14810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 200.625, "completions/mean_terminated_length": 200.625, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 0.9060784697532653, "epoch": 1.3625080444975637, "frac_reward_zero_std": 0.6, "grad_norm": 0.2439860701560974, "kl": 0.011044072004733608, "learning_rate": 3.0805518999631107e-06, "loss": 0.0007, "num_tokens": 64598501.0, "reward": -0.25, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.349075198173523, "sampling/importance_sampling_ratio/mean": 1.0005803406238556, "sampling/importance_sampling_ratio/min": 0.7701168537139893, "sampling/sampling_logp_difference/max": 0.3566216826438904, "sampling/sampling_logp_difference/mean": 0.019053915981203318, "step": 14820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.9, "completions/max_terminated_length": 302.9, "completions/mean_length": 212.0125, "completions/mean_terminated_length": 212.0125, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 0.9644978161901235, "epoch": 1.3634274156476969, "frac_reward_zero_std": 0.4, "grad_norm": 0.10736332088708878, "kl": 0.012445054668933152, "learning_rate": 3.078087239470771e-06, "loss": 0.0008, "num_tokens": 64644302.0, "reward": 0.29375, "reward_std": 0.5512827098369598, "rewards/reward_fn/mean": 0.29375, "rewards/reward_fn/std": 0.5512827396392822, "sampling/importance_sampling_ratio/max": 1.2353989481925964, "sampling/importance_sampling_ratio/mean": 0.9995728135108948, "sampling/importance_sampling_ratio/min": 0.7397984266281128, "sampling/sampling_logp_difference/max": 0.3066884160041809, "sampling/sampling_logp_difference/mean": 0.01988979456946254, "step": 14830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 204.15, "completions/mean_terminated_length": 204.15, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.0639636285603047, "epoch": 1.3643467867978303, "frac_reward_zero_std": 0.6, "grad_norm": 7.350782871246338, "kl": 0.013323101837886497, "learning_rate": 3.075621985236498e-06, "loss": 0.0008, "num_tokens": 64687946.0, "reward": -0.15, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2776615142822265, "sampling/importance_sampling_ratio/mean": 0.999524575471878, "sampling/importance_sampling_ratio/min": 0.7891557216644287, "sampling/sampling_logp_difference/max": 0.26484601497650145, "sampling/sampling_logp_difference/mean": 0.02159636691212654, "step": 14840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 322.4, "completions/max_terminated_length": 303.9, "completions/mean_length": 220.3875, "completions/mean_terminated_length": 213.68214569091796, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.1196102887392043, "epoch": 1.3652661579479637, "frac_reward_zero_std": 0.3, "grad_norm": 4.806997776031494, "kl": 0.01087840086547658, "learning_rate": 3.073156139792303e-06, "loss": 0.0009, "num_tokens": 64733353.0, "reward": -0.2875, "reward_std": 0.6703459024429321, "rewards/reward_fn/mean": -0.2875, "rewards/reward_fn/std": 0.6703459262847901, "sampling/importance_sampling_ratio/max": 1.290158712863922, "sampling/importance_sampling_ratio/mean": 1.0002580106258392, "sampling/importance_sampling_ratio/min": 0.7911995112895965, "sampling/sampling_logp_difference/max": 0.27145912647247317, "sampling/sampling_logp_difference/mean": 0.021420991420745848, "step": 14850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.2, "completions/max_terminated_length": 318.2, "completions/mean_length": 219.8375, "completions/mean_terminated_length": 219.8375, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.1029271826148033, "epoch": 1.3661855290980969, "frac_reward_zero_std": 0.4, "grad_norm": 4.593227386474609, "kl": 0.012332745001185686, "learning_rate": 3.0706897056708098e-06, "loss": 0.0007, "num_tokens": 64781932.0, "reward": -0.175, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2491510391235352, "sampling/importance_sampling_ratio/mean": 0.9993609845638275, "sampling/importance_sampling_ratio/min": 0.7797531664371491, "sampling/sampling_logp_difference/max": 0.2589693903923035, "sampling/sampling_logp_difference/mean": 0.02112381123006344, "step": 14860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.9, "completions/max_terminated_length": 291.0, "completions/mean_length": 209.9, "completions/mean_terminated_length": 206.38571472167968, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.1054976254701614, "epoch": 1.3671049002482303, "frac_reward_zero_std": 0.5, "grad_norm": 0.1311163753271103, "kl": 0.013048137217992917, "learning_rate": 3.0682226854052423e-06, "loss": 0.0002, "num_tokens": 64825572.0, "reward": 0.26875, "reward_std": 0.42757850140333176, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.4275785133242607, "sampling/importance_sampling_ratio/max": 1.2473117232322692, "sampling/importance_sampling_ratio/mean": 0.999579519033432, "sampling/importance_sampling_ratio/min": 0.7682502925395965, "sampling/sampling_logp_difference/max": 0.2693618357181549, "sampling/sampling_logp_difference/mean": 0.022080698050558567, "step": 14870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.8, "completions/max_terminated_length": 301.4, "completions/mean_length": 212.9, "completions/mean_terminated_length": 209.33035888671876, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0213394112885, "epoch": 1.3680242713983635, "frac_reward_zero_std": 0.3, "grad_norm": 0.10430674999952316, "kl": 0.012451780447736382, "learning_rate": 3.065755081529427e-06, "loss": 0.0005, "num_tokens": 64866660.0, "reward": -0.18125, "reward_std": 0.5218782171607017, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.5218782171607017, "sampling/importance_sampling_ratio/max": 1.2835566878318787, "sampling/importance_sampling_ratio/mean": 1.0006489038467408, "sampling/importance_sampling_ratio/min": 0.7741367518901825, "sampling/sampling_logp_difference/max": 0.2874425172805786, "sampling/sampling_logp_difference/mean": 0.020930181629955767, "step": 14880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.5, "completions/max_terminated_length": 299.4, "completions/mean_length": 218.55, "completions/mean_terminated_length": 215.4875, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.0745818309485913, "epoch": 1.3689436425484969, "frac_reward_zero_std": 0.3, "grad_norm": 7.385521411895752, "kl": 0.011893004487501458, "learning_rate": 3.0632868965777936e-06, "loss": 0.0005, "num_tokens": 64912712.0, "reward": 0.04375, "reward_std": 0.5512827083468437, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.5512827321887016, "sampling/importance_sampling_ratio/max": 1.239936363697052, "sampling/importance_sampling_ratio/mean": 0.9999590992927552, "sampling/importance_sampling_ratio/min": 0.8018389940261841, "sampling/sampling_logp_difference/max": 0.24020422697067262, "sampling/sampling_logp_difference/mean": 0.02119307275861502, "step": 14890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.1, "completions/max_terminated_length": 274.1, "completions/mean_length": 194.5875, "completions/mean_terminated_length": 194.5875, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.0563636720180511, "epoch": 1.36986301369863, "frac_reward_zero_std": 0.5, "grad_norm": 0.16293366253376007, "kl": 0.011089549178723245, "learning_rate": 3.0608181330853636e-06, "loss": 0.0006, "num_tokens": 64953599.0, "reward": -0.225, "reward_std": 0.48816035985946654, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4881603717803955, "sampling/importance_sampling_ratio/max": 1.3216880917549134, "sampling/importance_sampling_ratio/mean": 0.9997837781906128, "sampling/importance_sampling_ratio/min": 0.7670328557491303, "sampling/sampling_logp_difference/max": 0.3300620555877686, "sampling/sampling_logp_difference/mean": 0.020971585996448994, "step": 14900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 206.4875, "completions/mean_terminated_length": 206.4875, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 0.9784760717302561, "epoch": 1.3707823848487635, "frac_reward_zero_std": 0.2, "grad_norm": 0.13881756365299225, "kl": 0.011616722046164795, "learning_rate": 3.058348793587756e-06, "loss": 0.0004, "num_tokens": 64996174.0, "reward": -0.1, "reward_std": 0.7474140286445617, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.7474140524864197, "sampling/importance_sampling_ratio/max": 1.272713339328766, "sampling/importance_sampling_ratio/mean": 0.9998764574527741, "sampling/importance_sampling_ratio/min": 0.7309127032756806, "sampling/sampling_logp_difference/max": 0.33983484506607053, "sampling/sampling_logp_difference/mean": 0.02018727008253336, "step": 14910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 204.975, "completions/mean_terminated_length": 204.975, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.0697114445269107, "epoch": 1.3717017559988967, "frac_reward_zero_std": 0.5, "grad_norm": 0.10180740058422089, "kl": 0.010129856603452937, "learning_rate": 3.0558788806211804e-06, "loss": 0.0002, "num_tokens": 65037196.0, "reward": -0.48125, "reward_std": 0.3909388899803162, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.3909389078617096, "sampling/importance_sampling_ratio/max": 1.2665279984474183, "sampling/importance_sampling_ratio/mean": 0.9995775699615479, "sampling/importance_sampling_ratio/min": 0.7279279679059982, "sampling/sampling_logp_difference/max": 0.35470221042633054, "sampling/sampling_logp_difference/mean": 0.022469082660973073, "step": 14920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.7, "completions/max_terminated_length": 268.7, "completions/mean_length": 187.8875, "completions/mean_terminated_length": 187.8875, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.0310532055795192, "epoch": 1.37262112714903, "frac_reward_zero_std": 0.4, "grad_norm": 5.7266106605529785, "kl": 0.0122289365332108, "learning_rate": 3.0534083967224346e-06, "loss": 0.0009, "num_tokens": 65076563.0, "reward": 0.34375, "reward_std": 0.5651431679725647, "rewards/reward_fn/mean": 0.34375, "rewards/reward_fn/std": 0.5651432037353515, "sampling/importance_sampling_ratio/max": 1.2341317176818847, "sampling/importance_sampling_ratio/mean": 0.9996717810630799, "sampling/importance_sampling_ratio/min": 0.794535893201828, "sampling/sampling_logp_difference/max": 0.2371915578842163, "sampling/sampling_logp_difference/mean": 0.021193229779601096, "step": 14930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.9, "completions/max_terminated_length": 332.9, "completions/mean_length": 225.7, "completions/mean_terminated_length": 225.7, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.0674464970827102, "epoch": 1.3735404982991635, "frac_reward_zero_std": 0.5, "grad_norm": 0.08992277830839157, "kl": 0.010958339151693508, "learning_rate": 3.050937344428905e-06, "loss": 0.0006, "num_tokens": 65122123.0, "reward": 0.025, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2559701204299927, "sampling/importance_sampling_ratio/mean": 1.0004233837127685, "sampling/importance_sampling_ratio/min": 0.7859943926334381, "sampling/sampling_logp_difference/max": 0.2539111733436584, "sampling/sampling_logp_difference/mean": 0.0210202781483531, "step": 14940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 207.05, "completions/mean_terminated_length": 207.05, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.035588999465108, "epoch": 1.3744598694492967, "frac_reward_zero_std": 0.5, "grad_norm": 0.0894203707575798, "kl": 0.012043275381438434, "learning_rate": 3.04846572627856e-06, "loss": 0.0008, "num_tokens": 65163151.0, "reward": -0.025, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.3633171319961548, "sampling/importance_sampling_ratio/mean": 1.0000123858451844, "sampling/importance_sampling_ratio/min": 0.7733891308307648, "sampling/sampling_logp_difference/max": 0.34886903762817384, "sampling/sampling_logp_difference/mean": 0.020856335759162903, "step": 14950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 208.0375, "completions/mean_terminated_length": 208.0375, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.1371171660721302, "epoch": 1.37537924059943, "frac_reward_zero_std": 0.6, "grad_norm": 5.128088474273682, "kl": 0.011442554800305515, "learning_rate": 3.0459935448099486e-06, "loss": 0.0005, "num_tokens": 65207434.0, "reward": 0.075, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2368045568466186, "sampling/importance_sampling_ratio/mean": 1.000522994995117, "sampling/importance_sampling_ratio/min": 0.7923564076423645, "sampling/sampling_logp_difference/max": 0.23821945190429689, "sampling/sampling_logp_difference/mean": 0.022178053483366968, "step": 14960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 195.7625, "completions/mean_terminated_length": 195.7625, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.0925792813301087, "epoch": 1.3762986117495632, "frac_reward_zero_std": 0.5, "grad_norm": 5.2203569412231445, "kl": 0.011380323680350557, "learning_rate": 3.0435208025622e-06, "loss": 0.0008, "num_tokens": 65248567.0, "reward": -0.2, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.297448754310608, "sampling/importance_sampling_ratio/mean": 0.9997813940048218, "sampling/importance_sampling_ratio/min": 0.7923070609569549, "sampling/sampling_logp_difference/max": 0.2729122877120972, "sampling/sampling_logp_difference/mean": 0.021539225801825525, "step": 14970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.1, "completions/max_terminated_length": 324.1, "completions/mean_length": 233.8125, "completions/mean_terminated_length": 233.8125, "completions/min_length": 146.3, "completions/min_terminated_length": 146.3, "entropy": 1.0994558565318584, "epoch": 1.3772179828996967, "frac_reward_zero_std": 0.5, "grad_norm": 0.15426798164844513, "kl": 0.011164097592700272, "learning_rate": 3.041047502075019e-06, "loss": 0.0005, "num_tokens": 65295912.0, "reward": -0.475, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.277714765071869, "sampling/importance_sampling_ratio/mean": 1.0000342190265656, "sampling/importance_sampling_ratio/min": 0.7782601654529572, "sampling/sampling_logp_difference/max": 0.2772952675819397, "sampling/sampling_logp_difference/mean": 0.021097232960164546, "step": 14980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.4, "completions/max_terminated_length": 335.4, "completions/mean_length": 235.0125, "completions/mean_terminated_length": 235.0125, "completions/min_length": 150.9, "completions/min_terminated_length": 150.9, "entropy": 1.1009477984160185, "epoch": 1.3781373540498298, "frac_reward_zero_std": 0.3, "grad_norm": 0.09947676211595535, "kl": 0.011339406011393294, "learning_rate": 3.038573645888682e-06, "loss": 0.0008, "num_tokens": 65340345.0, "reward": -0.325, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.3962857723236084, "sampling/importance_sampling_ratio/mean": 1.0004305720329285, "sampling/importance_sampling_ratio/min": 0.7900130391120911, "sampling/sampling_logp_difference/max": 0.3304335236549377, "sampling/sampling_logp_difference/mean": 0.021456700377166273, "step": 14990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.4, "completions/max_terminated_length": 332.4, "completions/mean_length": 223.4, "completions/mean_terminated_length": 223.4, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.0856854058802128, "epoch": 1.3790567251999633, "frac_reward_zero_std": 0.7, "grad_norm": 0.12481308728456497, "kl": 0.011275829444639385, "learning_rate": 3.03609923654404e-06, "loss": 0.0005, "num_tokens": 65384641.0, "reward": -0.275, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.3506656885147095, "sampling/importance_sampling_ratio/mean": 1.000161635875702, "sampling/importance_sampling_ratio/min": 0.7552223384380341, "sampling/sampling_logp_difference/max": 0.34714231491088865, "sampling/sampling_logp_difference/mean": 0.021445835009217262, "step": 15000 }, { "epoch": 1.3790567251999633, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0037085514834205935, "eval_completions/max_length": 306.2268760907504, "eval_completions/max_terminated_length": 303.4328097731239, "eval_completions/mean_length": 213.14725130890054, "eval_completions/mean_terminated_length": 212.3193513294373, "eval_completions/min_length": 139.7068062827225, "eval_completions/min_terminated_length": 139.7068062827225, "eval_entropy": 1.096245484254331, "eval_frac_reward_zero_std": 0.462478184991274, "eval_kl": 0.011368463899173089, "eval_loss": 0.0006348146707750857, "eval_num_tokens": 65384641.0, "eval_reward": -0.11616492146596859, "eval_reward_std": 0.4715483939928534, "eval_rewards/reward_fn/mean": -0.11616492146596859, "eval_rewards/reward_fn/std": 0.4715484104283401, "eval_runtime": 1775.547, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.276541108236263, "eval_sampling/importance_sampling_ratio/mean": 1.000005101970353, "eval_sampling/importance_sampling_ratio/min": 0.7714031933177828, "eval_sampling/sampling_logp_difference/max": 0.28980536184177763, "eval_sampling/sampling_logp_difference/mean": 0.021282870610151928, "eval_steps_per_second": 0.041, "step": 15000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 212.1875, "completions/mean_terminated_length": 212.1875, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.1102663263678552, "epoch": 1.3799760963500964, "frac_reward_zero_std": 0.5, "grad_norm": 0.09463823586702347, "kl": 0.01218221873859875, "learning_rate": 3.033624276582508e-06, "loss": 0.0008, "num_tokens": 65427600.0, "reward": -0.175, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.234564435482025, "sampling/importance_sampling_ratio/mean": 0.9998126208782196, "sampling/importance_sampling_ratio/min": 0.7545597732067109, "sampling/sampling_logp_difference/max": 0.29101842641830444, "sampling/sampling_logp_difference/mean": 0.02192401383072138, "step": 15010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.0, "completions/max_terminated_length": 300.2, "completions/mean_length": 218.5375, "completions/mean_terminated_length": 215.5875, "completions/min_length": 126.9, "completions/min_terminated_length": 126.9, "entropy": 1.0736484706401825, "epoch": 1.3808954675002298, "frac_reward_zero_std": 0.4, "grad_norm": 4.645532608032227, "kl": 0.011044263205258175, "learning_rate": 3.031148768546069e-06, "loss": 0.0007, "num_tokens": 65474163.0, "reward": -0.20625, "reward_std": 0.4839666947722435, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.48396670669317243, "sampling/importance_sampling_ratio/max": 1.3119783520698547, "sampling/importance_sampling_ratio/mean": 0.9996820330619812, "sampling/importance_sampling_ratio/min": 0.7562383890151978, "sampling/sampling_logp_difference/max": 0.2976393699645996, "sampling/sampling_logp_difference/mean": 0.021385933086276055, "step": 15020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.8, "completions/max_terminated_length": 322.8, "completions/mean_length": 215.5625, "completions/mean_terminated_length": 215.5625, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.1019026964902878, "epoch": 1.3818148386503633, "frac_reward_zero_std": 0.6, "grad_norm": 5.868115425109863, "kl": 0.011780762864509598, "learning_rate": 3.028672714977267e-06, "loss": 0.0007, "num_tokens": 65517872.0, "reward": -0.075, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2233730554580688, "sampling/importance_sampling_ratio/mean": 0.9998320162296295, "sampling/importance_sampling_ratio/min": 0.7619880199432373, "sampling/sampling_logp_difference/max": 0.2847552239894867, "sampling/sampling_logp_difference/mean": 0.022140546701848507, "step": 15030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.6, "completions/max_terminated_length": 334.3, "completions/mean_length": 248.075, "completions/mean_terminated_length": 246.53928833007814, "completions/min_length": 162.4, "completions/min_terminated_length": 162.4, "entropy": 1.0745249472558498, "epoch": 1.3827342098004964, "frac_reward_zero_std": 0.6, "grad_norm": 0.08943535387516022, "kl": 0.010593987675383687, "learning_rate": 3.026196118419209e-06, "loss": 0.0003, "num_tokens": 65567326.0, "reward": -0.25625, "reward_std": 0.3718110263347626, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.3718110382556915, "sampling/importance_sampling_ratio/max": 1.286945629119873, "sampling/importance_sampling_ratio/mean": 0.999373608827591, "sampling/importance_sampling_ratio/min": 0.771023279428482, "sampling/sampling_logp_difference/max": 0.2943978488445282, "sampling/sampling_logp_difference/mean": 0.021506847441196443, "step": 15040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 283.4, "completions/max_terminated_length": 268.1, "completions/mean_length": 202.8, "completions/mean_terminated_length": 199.92678833007812, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.1259734950959683, "epoch": 1.3836535809506298, "frac_reward_zero_std": 0.5, "grad_norm": 0.09020005911588669, "kl": 0.010984595632180572, "learning_rate": 3.023718981415558e-06, "loss": 0.0004, "num_tokens": 65609662.0, "reward": -0.13125, "reward_std": 0.40403020977973936, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.40403022766113283, "sampling/importance_sampling_ratio/max": 1.2361315727233886, "sampling/importance_sampling_ratio/mean": 0.9998522579669953, "sampling/importance_sampling_ratio/min": 0.7747304141521454, "sampling/sampling_logp_difference/max": 0.2687695324420929, "sampling/sampling_logp_difference/mean": 0.022329726628959178, "step": 15050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.4, "completions/max_terminated_length": 314.4, "completions/mean_length": 224.1375, "completions/mean_terminated_length": 224.1375, "completions/min_length": 157.9, "completions/min_terminated_length": 157.9, "entropy": 1.0806359946727753, "epoch": 1.384572952100763, "frac_reward_zero_std": 0.5, "grad_norm": 0.1320926547050476, "kl": 0.012469378736568615, "learning_rate": 3.021241306510531e-06, "loss": 0.0009, "num_tokens": 65654873.0, "reward": -0.1, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2524912476539611, "sampling/importance_sampling_ratio/mean": 1.0002158224582671, "sampling/importance_sampling_ratio/min": 0.77122802734375, "sampling/sampling_logp_difference/max": 0.2641189157962799, "sampling/sampling_logp_difference/mean": 0.021453846618533133, "step": 15060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.4, "completions/max_terminated_length": 318.4, "completions/mean_length": 216.975, "completions/mean_terminated_length": 216.975, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0248683005571366, "epoch": 1.3854923232508964, "frac_reward_zero_std": 0.1, "grad_norm": 6.416713237762451, "kl": 0.011094341706484556, "learning_rate": 3.0187630962489e-06, "loss": 0.001, "num_tokens": 65699943.0, "reward": 0.1, "reward_std": 0.792874401807785, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.792874413728714, "sampling/importance_sampling_ratio/max": 1.2369460821151734, "sampling/importance_sampling_ratio/mean": 0.9996050894260406, "sampling/importance_sampling_ratio/min": 0.7742851316928864, "sampling/sampling_logp_difference/max": 0.26497513949871065, "sampling/sampling_logp_difference/mean": 0.0206818925216794, "step": 15070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.1, "completions/max_terminated_length": 288.1, "completions/mean_length": 201.625, "completions/mean_terminated_length": 201.625, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 0.9684630341827869, "epoch": 1.3864116944010296, "frac_reward_zero_std": 0.4, "grad_norm": 9.296905517578125, "kl": 0.011851884436327963, "learning_rate": 3.0162843531759856e-06, "loss": 0.0003, "num_tokens": 65741569.0, "reward": 0.1, "reward_std": 0.5664041936397552, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5664042294025421, "sampling/importance_sampling_ratio/max": 1.3298131227493286, "sampling/importance_sampling_ratio/mean": 0.9995648324489593, "sampling/importance_sampling_ratio/min": 0.7462620615959168, "sampling/sampling_logp_difference/max": 0.3122373580932617, "sampling/sampling_logp_difference/mean": 0.020657781139016153, "step": 15080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 206.875, "completions/mean_terminated_length": 206.875, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.1344747945666314, "epoch": 1.387331065551163, "frac_reward_zero_std": 0.5, "grad_norm": 5.852377891540527, "kl": 0.01143290635664016, "learning_rate": 3.013805079837655e-06, "loss": 0.0007, "num_tokens": 65785351.0, "reward": 0.2, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.245332419872284, "sampling/importance_sampling_ratio/mean": 0.9999868214130402, "sampling/importance_sampling_ratio/min": 0.7368571937084198, "sampling/sampling_logp_difference/max": 0.32063076496124265, "sampling/sampling_logp_difference/mean": 0.023307582922279836, "step": 15090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.7, "completions/max_terminated_length": 324.7, "completions/mean_length": 232.6, "completions/mean_terminated_length": 232.6, "completions/min_length": 168.6, "completions/min_terminated_length": 168.6, "entropy": 1.1748461738228797, "epoch": 1.3882504367012962, "frac_reward_zero_std": 0.5, "grad_norm": 5.107047080993652, "kl": 0.011596916901180521, "learning_rate": 3.0113252787803205e-06, "loss": 0.0005, "num_tokens": 65832663.0, "reward": -0.25, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2786015748977662, "sampling/importance_sampling_ratio/mean": 1.0000973403453828, "sampling/importance_sampling_ratio/min": 0.7908303499221802, "sampling/sampling_logp_difference/max": 0.2707971900701523, "sampling/sampling_logp_difference/mean": 0.02177758403122425, "step": 15100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.5, "completions/max_terminated_length": 290.5, "completions/mean_length": 199.8875, "completions/mean_terminated_length": 199.8875, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.1147471778094769, "epoch": 1.3891698078514296, "frac_reward_zero_std": 0.9, "grad_norm": 0.09944900870323181, "kl": 0.011312450788682326, "learning_rate": 3.0088449525509377e-06, "loss": 0.0005, "num_tokens": 65881326.0, "reward": 0.325, "reward_std": 0.10350983142852783, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.10350984334945679, "sampling/importance_sampling_ratio/max": 1.2560358762741088, "sampling/importance_sampling_ratio/mean": 1.0002610623836516, "sampling/importance_sampling_ratio/min": 0.7915449380874634, "sampling/sampling_logp_difference/max": 0.2493079423904419, "sampling/sampling_logp_difference/mean": 0.022028707154095172, "step": 15110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 219.2625, "completions/mean_terminated_length": 219.2625, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.1293817199766636, "epoch": 1.390089179001563, "frac_reward_zero_std": 0.8, "grad_norm": 0.10192747414112091, "kl": 0.011436495062662288, "learning_rate": 3.0063641036969993e-06, "loss": 0.0008, "num_tokens": 65924627.0, "reward": -0.025, "reward_std": 0.1960918426513672, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.19609185457229614, "sampling/importance_sampling_ratio/max": 1.3267961859703064, "sampling/importance_sampling_ratio/mean": 0.9999339878559113, "sampling/importance_sampling_ratio/min": 0.7700299322605133, "sampling/sampling_logp_difference/max": 0.3248930096626282, "sampling/sampling_logp_difference/mean": 0.02186425160616636, "step": 15120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 218.2625, "completions/mean_terminated_length": 218.2625, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0409196808934211, "epoch": 1.3910085501516962, "frac_reward_zero_std": 0.5, "grad_norm": 0.2549496293067932, "kl": 0.012137866794364526, "learning_rate": 3.0038827347665356e-06, "loss": 0.0004, "num_tokens": 65966352.0, "reward": -0.15625, "reward_std": 0.43040092587471007, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.43040093779563904, "sampling/importance_sampling_ratio/max": 1.2567020893096923, "sampling/importance_sampling_ratio/mean": 1.0003823697566987, "sampling/importance_sampling_ratio/min": 0.7538852095603943, "sampling/sampling_logp_difference/max": 0.29510643482208254, "sampling/sampling_logp_difference/mean": 0.020840171724557877, "step": 15130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.7, "completions/max_terminated_length": 300.7, "completions/mean_length": 213.3625, "completions/mean_terminated_length": 213.3625, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.1295617893338203, "epoch": 1.3919279213018296, "frac_reward_zero_std": 0.4, "grad_norm": 0.08428405225276947, "kl": 0.01010127305635251, "learning_rate": 3.0014008483081136e-06, "loss": 0.0008, "num_tokens": 66010853.0, "reward": 0.05, "reward_std": 0.5117493987083435, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5117493987083435, "sampling/importance_sampling_ratio/max": 1.2948880791664124, "sampling/importance_sampling_ratio/mean": 1.0003705382347108, "sampling/importance_sampling_ratio/min": 0.7648368299007415, "sampling/sampling_logp_difference/max": 0.2906152456998825, "sampling/sampling_logp_difference/mean": 0.02233455367386341, "step": 15140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.4, "completions/max_terminated_length": 296.4, "completions/mean_length": 207.9625, "completions/mean_terminated_length": 207.9625, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.0849338203668595, "epoch": 1.3928472924519628, "frac_reward_zero_std": 0.9, "grad_norm": 0.24264010787010193, "kl": 0.010206780856242404, "learning_rate": 2.998918446870827e-06, "loss": 0.0005, "num_tokens": 66052858.0, "reward": 0.375, "reward_std": 0.07071067690849304, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.07071067690849304, "sampling/importance_sampling_ratio/max": 1.2548742651939393, "sampling/importance_sampling_ratio/mean": 0.9995277345180511, "sampling/importance_sampling_ratio/min": 0.7642188191413879, "sampling/sampling_logp_difference/max": 0.2826400905847549, "sampling/sampling_logp_difference/mean": 0.02173823695629835, "step": 15150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.8, "completions/max_terminated_length": 338.8, "completions/mean_length": 222.0625, "completions/mean_terminated_length": 222.0625, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.1634434156119824, "epoch": 1.3937666636020962, "frac_reward_zero_std": 0.6, "grad_norm": 0.12201710045337677, "kl": 0.010252642555860803, "learning_rate": 2.996435533004303e-06, "loss": 0.0005, "num_tokens": 66099599.0, "reward": -0.275, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2544161558151246, "sampling/importance_sampling_ratio/mean": 1.000007724761963, "sampling/importance_sampling_ratio/min": 0.7784915030002594, "sampling/sampling_logp_difference/max": 0.2742796540260315, "sampling/sampling_logp_difference/mean": 0.02206424977630377, "step": 15160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.8, "completions/max_terminated_length": 296.8, "completions/mean_length": 207.0375, "completions/mean_terminated_length": 207.0375, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.0825277037918568, "epoch": 1.3946860347522294, "frac_reward_zero_std": 0.7, "grad_norm": 6.552148818969727, "kl": 0.010229896719101817, "learning_rate": 2.993952109258692e-06, "loss": 0.0006, "num_tokens": 66142586.0, "reward": 0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.227911126613617, "sampling/importance_sampling_ratio/mean": 0.9996212482452392, "sampling/importance_sampling_ratio/min": 0.7702397465705871, "sampling/sampling_logp_difference/max": 0.26675302982330323, "sampling/sampling_logp_difference/mean": 0.022138164937496187, "step": 15170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 215.5125, "completions/mean_terminated_length": 215.5125, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.128676164150238, "epoch": 1.3956054059023628, "frac_reward_zero_std": 0.4, "grad_norm": 6.116549968719482, "kl": 0.01020295307971537, "learning_rate": 2.9914681781846697e-06, "loss": 0.0004, "num_tokens": 66190019.0, "reward": -0.1, "reward_std": 0.577347707748413, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.577347731590271, "sampling/importance_sampling_ratio/max": 1.238239347934723, "sampling/importance_sampling_ratio/mean": 1.000207144021988, "sampling/importance_sampling_ratio/min": 0.771949189901352, "sampling/sampling_logp_difference/max": 0.26110501289367677, "sampling/sampling_logp_difference/mean": 0.02191423811018467, "step": 15180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.5, "completions/max_terminated_length": 287.5, "completions/mean_length": 206.1875, "completions/mean_terminated_length": 206.1875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0374730341136456, "epoch": 1.396524777052496, "frac_reward_zero_std": 0.8, "grad_norm": 0.14402177929878235, "kl": 0.011650323274079711, "learning_rate": 2.9889837423334326e-06, "loss": 0.001, "num_tokens": 66232130.0, "reward": -0.4, "reward_std": 0.20701966285705567, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.20701968669891357, "sampling/importance_sampling_ratio/max": 1.2836471557617188, "sampling/importance_sampling_ratio/mean": 1.0004659712314605, "sampling/importance_sampling_ratio/min": 0.7938989460468292, "sampling/sampling_logp_difference/max": 0.2695014297962189, "sampling/sampling_logp_difference/mean": 0.021009203605353834, "step": 15190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.2, "completions/max_terminated_length": 286.2, "completions/mean_length": 209.7875, "completions/mean_terminated_length": 209.7875, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0685970149934292, "epoch": 1.3974441482026294, "frac_reward_zero_std": 0.3, "grad_norm": 5.618688583374023, "kl": 0.010165239364141598, "learning_rate": 2.9864988042566955e-06, "loss": 0.0004, "num_tokens": 66276409.0, "reward": 0.15, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.2762294292449952, "sampling/importance_sampling_ratio/mean": 1.0000479519367218, "sampling/importance_sampling_ratio/min": 0.7871960759162903, "sampling/sampling_logp_difference/max": 0.2745161771774292, "sampling/sampling_logp_difference/mean": 0.021427402272820473, "step": 15200 }, { "epoch": 1.3974441482026294, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 298.3961605584642, "eval_completions/max_terminated_length": 296.6980802792321, "eval_completions/mean_length": 209.51374345549738, "eval_completions/mean_terminated_length": 209.11508996241173, "eval_completions/min_length": 138.22513089005236, "eval_completions/min_terminated_length": 138.22513089005236, "eval_entropy": 1.095199942224729, "eval_frac_reward_zero_std": 0.48342059336823734, "eval_kl": 0.011428167493315006, "eval_loss": 0.000591080985032022, "eval_num_tokens": 66276409.0, "eval_reward": -0.1082024432809773, "eval_reward_std": 0.44687039619653013, "eval_rewards/reward_fn/mean": -0.1082024432809773, "eval_rewards/reward_fn/std": 0.44687041023950924, "eval_runtime": 1735.6032, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2715207910454502, "eval_sampling/importance_sampling_ratio/mean": 0.9999933186625936, "eval_sampling/importance_sampling_ratio/min": 0.770113911265166, "eval_sampling/sampling_logp_difference/max": 0.2910216311821763, "eval_sampling/sampling_logp_difference/mean": 0.021456518963005322, "eval_steps_per_second": 0.041, "step": 15200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.0, "completions/max_terminated_length": 279.0, "completions/mean_length": 198.175, "completions/mean_terminated_length": 198.175, "completions/min_length": 116.0, "completions/min_terminated_length": 116.0, "entropy": 1.1509669117629528, "epoch": 1.3983635193527628, "frac_reward_zero_std": 0.4, "grad_norm": 6.373007297515869, "kl": 0.011399814928881823, "learning_rate": 2.9840133665066885e-06, "loss": 0.0008, "num_tokens": 66315295.0, "reward": -0.225, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2718843817710876, "sampling/importance_sampling_ratio/mean": 0.9995755195617676, "sampling/importance_sampling_ratio/min": 0.7752128839492798, "sampling/sampling_logp_difference/max": 0.28117492198944094, "sampling/sampling_logp_difference/mean": 0.022643010690808298, "step": 15210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 289.4, "completions/max_terminated_length": 274.2, "completions/mean_length": 193.875, "completions/mean_terminated_length": 190.82678833007813, "completions/min_length": 118.7, "completions/min_terminated_length": 118.7, "entropy": 1.0028740845620632, "epoch": 1.399282890502896, "frac_reward_zero_std": 0.5, "grad_norm": 5.520230293273926, "kl": 0.009702791622839869, "learning_rate": 2.981527431636156e-06, "loss": 0.0005, "num_tokens": 66358965.0, "reward": -0.1125, "reward_std": 0.40106717199087144, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.40106717199087144, "sampling/importance_sampling_ratio/max": 1.2650699138641357, "sampling/importance_sampling_ratio/mean": 1.0000135123729705, "sampling/importance_sampling_ratio/min": 0.7423227250576019, "sampling/sampling_logp_difference/max": 0.32005069255828855, "sampling/sampling_logp_difference/mean": 0.021193243749439716, "step": 15220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.4, "completions/max_terminated_length": 292.4, "completions/mean_length": 210.6875, "completions/mean_terminated_length": 210.6875, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0041101854294538, "epoch": 1.4002022616530292, "frac_reward_zero_std": 0.5, "grad_norm": 0.09096013009548187, "kl": 0.011999423993984237, "learning_rate": 2.979041002198352e-06, "loss": 0.0003, "num_tokens": 66402604.0, "reward": 0.4, "reward_std": 0.3897472023963928, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.3897472023963928, "sampling/importance_sampling_ratio/max": 1.2581245660781861, "sampling/importance_sampling_ratio/mean": 1.000089293718338, "sampling/importance_sampling_ratio/min": 0.7943153440952301, "sampling/sampling_logp_difference/max": 0.24550795555114746, "sampling/sampling_logp_difference/mean": 0.0199631592258811, "step": 15230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 197.7625, "completions/mean_terminated_length": 197.7625, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0628660179674625, "epoch": 1.4011216328031626, "frac_reward_zero_std": 0.6, "grad_norm": 5.044815540313721, "kl": 0.009505057928618044, "learning_rate": 2.9765540807470376e-06, "loss": 0.0003, "num_tokens": 66444849.0, "reward": -0.225, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2436131238937378, "sampling/importance_sampling_ratio/mean": 0.9997122347354889, "sampling/importance_sampling_ratio/min": 0.7792357087135315, "sampling/sampling_logp_difference/max": 0.2635125398635864, "sampling/sampling_logp_difference/mean": 0.021638138592243193, "step": 15240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.8, "completions/max_terminated_length": 276.8, "completions/mean_length": 213.5875, "completions/mean_terminated_length": 213.5875, "completions/min_length": 154.3, "completions/min_terminated_length": 154.3, "entropy": 1.1337829135358333, "epoch": 1.402041003953296, "frac_reward_zero_std": 0.6, "grad_norm": 6.939918041229248, "kl": 0.009658633719664068, "learning_rate": 2.9740666698364833e-06, "loss": 0.0006, "num_tokens": 66488400.0, "reward": -0.15, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2469443202018737, "sampling/importance_sampling_ratio/mean": 1.0000805497169494, "sampling/importance_sampling_ratio/min": 0.7514084577560425, "sampling/sampling_logp_difference/max": 0.2883231997489929, "sampling/sampling_logp_difference/mean": 0.022338743880391122, "step": 15250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 198.725, "completions/mean_terminated_length": 198.725, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.0700346656143664, "epoch": 1.4029603751034292, "frac_reward_zero_std": 0.2, "grad_norm": 7.891740798950195, "kl": 0.01085825982154347, "learning_rate": 2.971578772021458e-06, "loss": 0.0008, "num_tokens": 66532714.0, "reward": -0.3, "reward_std": 0.7549471855163574, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.7549472332000733, "sampling/importance_sampling_ratio/max": 1.2847231149673461, "sampling/importance_sampling_ratio/mean": 1.0001934587955474, "sampling/importance_sampling_ratio/min": 0.7302603781223297, "sampling/sampling_logp_difference/max": 0.3214552104473114, "sampling/sampling_logp_difference/mean": 0.021924514137208462, "step": 15260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.5, "completions/max_terminated_length": 308.5, "completions/mean_length": 213.9875, "completions/mean_terminated_length": 213.9875, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.0859810769557954, "epoch": 1.4038797462535626, "frac_reward_zero_std": 0.3, "grad_norm": 6.547486782073975, "kl": 0.013088479801081122, "learning_rate": 2.969090389857231e-06, "loss": 0.0012, "num_tokens": 66574945.0, "reward": -0.275, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.295585572719574, "sampling/importance_sampling_ratio/mean": 0.999593871831894, "sampling/importance_sampling_ratio/min": 0.745374059677124, "sampling/sampling_logp_difference/max": 0.3407749354839325, "sampling/sampling_logp_difference/mean": 0.02110893800854683, "step": 15270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.0, "completions/max_terminated_length": 348.0, "completions/mean_length": 239.7, "completions/mean_terminated_length": 239.7, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.125454806536436, "epoch": 1.4047991174036958, "frac_reward_zero_std": 0.3, "grad_norm": 0.1255192905664444, "kl": 0.011065636499552056, "learning_rate": 2.9666015258995723e-06, "loss": 0.0008, "num_tokens": 66620401.0, "reward": -0.1625, "reward_std": 0.5249656066298485, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.5249656066298485, "sampling/importance_sampling_ratio/max": 1.2437970876693725, "sampling/importance_sampling_ratio/mean": 1.0007884621620178, "sampling/importance_sampling_ratio/min": 0.7742029666900635, "sampling/sampling_logp_difference/max": 0.26372194290161133, "sampling/sampling_logp_difference/mean": 0.021619313023984434, "step": 15280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.4, "completions/max_terminated_length": 312.4, "completions/mean_length": 205.4, "completions/mean_terminated_length": 205.4, "completions/min_length": 118.5, "completions/min_terminated_length": 118.5, "entropy": 1.0576473377645015, "epoch": 1.4057184885538292, "frac_reward_zero_std": 0.5, "grad_norm": 0.10906802117824554, "kl": 0.01120063167763874, "learning_rate": 2.9641121827047438e-06, "loss": 0.001, "num_tokens": 66664257.0, "reward": -0.225, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.33797687292099, "sampling/importance_sampling_ratio/mean": 1.0000042617321014, "sampling/importance_sampling_ratio/min": 0.8069852113723754, "sampling/sampling_logp_difference/max": 0.2865617275238037, "sampling/sampling_logp_difference/mean": 0.021036380529403688, "step": 15290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 207.3125, "completions/mean_terminated_length": 207.3125, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 0.998207052052021, "epoch": 1.4066378597039626, "frac_reward_zero_std": 0.1, "grad_norm": 5.688663482666016, "kl": 0.010114700207486749, "learning_rate": 2.9616223628295e-06, "loss": 0.0007, "num_tokens": 66705914.0, "reward": 0.075, "reward_std": 0.8038022220134735, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.8038022458553314, "sampling/importance_sampling_ratio/max": 1.2423041820526124, "sampling/importance_sampling_ratio/mean": 1.0000197649002076, "sampling/importance_sampling_ratio/min": 0.7736358344554901, "sampling/sampling_logp_difference/max": 0.269136381149292, "sampling/sampling_logp_difference/mean": 0.02040556874126196, "step": 15300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.9, "completions/max_terminated_length": 286.9, "completions/mean_length": 201.225, "completions/mean_terminated_length": 201.225, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 0.9758158221840858, "epoch": 1.4075572308540958, "frac_reward_zero_std": 0.6, "grad_norm": 15.870637893676758, "kl": 0.010723400238202885, "learning_rate": 2.959132068831087e-06, "loss": 0.0004, "num_tokens": 66746700.0, "reward": -0.025, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.32798433303833, "sampling/importance_sampling_ratio/mean": 1.0005160093307495, "sampling/importance_sampling_ratio/min": 0.7777515470981597, "sampling/sampling_logp_difference/max": 0.3330970525741577, "sampling/sampling_logp_difference/mean": 0.020634661428630352, "step": 15310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.6, "completions/max_terminated_length": 245.6, "completions/mean_length": 179.4625, "completions/mean_terminated_length": 179.4625, "completions/min_length": 117.3, "completions/min_terminated_length": 117.3, "entropy": 0.9837579041719436, "epoch": 1.408476602004229, "frac_reward_zero_std": 0.4, "grad_norm": 5.2101569175720215, "kl": 0.012606199225410819, "learning_rate": 2.956641303267234e-06, "loss": 0.0006, "num_tokens": 66786025.0, "reward": 0.425, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2683553576469422, "sampling/importance_sampling_ratio/mean": 0.9996763706207276, "sampling/importance_sampling_ratio/min": 0.7727964639663696, "sampling/sampling_logp_difference/max": 0.2852674126625061, "sampling/sampling_logp_difference/mean": 0.021081124898046255, "step": 15320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.4, "completions/max_terminated_length": 296.4, "completions/mean_length": 217.7, "completions/mean_terminated_length": 217.7, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.1179760947823525, "epoch": 1.4093959731543624, "frac_reward_zero_std": 0.4, "grad_norm": 5.2623748779296875, "kl": 0.012386842095293105, "learning_rate": 2.9541500686961596e-06, "loss": 0.0005, "num_tokens": 66834449.0, "reward": -0.325, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.2685464024543762, "sampling/importance_sampling_ratio/mean": 0.9999126613140106, "sampling/importance_sampling_ratio/min": 0.7292554438114166, "sampling/sampling_logp_difference/max": 0.3292599439620972, "sampling/sampling_logp_difference/mean": 0.021044795215129853, "step": 15330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.3, "completions/max_terminated_length": 315.3, "completions/mean_length": 211.225, "completions/mean_terminated_length": 211.225, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.0609584882855416, "epoch": 1.4103153443044958, "frac_reward_zero_std": 0.8, "grad_norm": 0.2944463789463043, "kl": 0.01167300105735194, "learning_rate": 2.9516583676765594e-06, "loss": 0.0008, "num_tokens": 66878363.0, "reward": -0.35, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.327163529396057, "sampling/importance_sampling_ratio/mean": 0.9995452046394349, "sampling/importance_sampling_ratio/min": 0.7343191295862198, "sampling/sampling_logp_difference/max": 0.3892100751399994, "sampling/sampling_logp_difference/mean": 0.02113805841654539, "step": 15340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.0, "completions/max_terminated_length": 277.0, "completions/mean_length": 206.6875, "completions/mean_terminated_length": 206.6875, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.1097784489393234, "epoch": 1.411234715454629, "frac_reward_zero_std": 0.4, "grad_norm": 5.938485622406006, "kl": 0.010992450261255726, "learning_rate": 2.9491662027676106e-06, "loss": 0.0003, "num_tokens": 66919698.0, "reward": -0.425, "reward_std": 0.5656446695327759, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5656446814537048, "sampling/importance_sampling_ratio/max": 1.2155608177185058, "sampling/importance_sampling_ratio/mean": 0.9994419276714325, "sampling/importance_sampling_ratio/min": 0.7673089385032654, "sampling/sampling_logp_difference/max": 0.2667893648147583, "sampling/sampling_logp_difference/mean": 0.022085232473909856, "step": 15350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 199.95, "completions/mean_terminated_length": 199.95, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.0737278074026109, "epoch": 1.4121540866047624, "frac_reward_zero_std": 0.3, "grad_norm": 5.487006187438965, "kl": 0.010296465511783027, "learning_rate": 2.9466735765289665e-06, "loss": 0.0001, "num_tokens": 66963878.0, "reward": 0.35, "reward_std": 0.5639677107334137, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.5639677226543427, "sampling/importance_sampling_ratio/max": 1.2547307848930358, "sampling/importance_sampling_ratio/mean": 0.9997939050197602, "sampling/importance_sampling_ratio/min": 0.7731052696704864, "sampling/sampling_logp_difference/max": 0.27008030414581297, "sampling/sampling_logp_difference/mean": 0.021892915666103362, "step": 15360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 213.7875, "completions/mean_terminated_length": 213.7875, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.088694803416729, "epoch": 1.4130734577548956, "frac_reward_zero_std": 0.5, "grad_norm": 0.20823058485984802, "kl": 0.0109740431769751, "learning_rate": 2.9441804915207543e-06, "loss": 0.0006, "num_tokens": 67011277.0, "reward": -0.33125, "reward_std": 0.40231250375509264, "rewards/reward_fn/mean": -0.33125, "rewards/reward_fn/std": 0.4023125275969505, "sampling/importance_sampling_ratio/max": 1.263831114768982, "sampling/importance_sampling_ratio/mean": 0.9997029423713684, "sampling/importance_sampling_ratio/min": 0.7871111631393433, "sampling/sampling_logp_difference/max": 0.26021270751953124, "sampling/sampling_logp_difference/mean": 0.022306659258902073, "step": 15370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 214.2625, "completions/mean_terminated_length": 214.2625, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 1.1253449939191342, "epoch": 1.413992828905029, "frac_reward_zero_std": 0.3, "grad_norm": 6.092264175415039, "kl": 0.011904302280163393, "learning_rate": 2.9416869503035716e-06, "loss": 0.0009, "num_tokens": 67057970.0, "reward": -0.45625, "reward_std": 0.5471285209059715, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.5471285328269004, "sampling/importance_sampling_ratio/max": 1.3010489225387574, "sampling/importance_sampling_ratio/mean": 0.9997627019882203, "sampling/importance_sampling_ratio/min": 0.7393942594528198, "sampling/sampling_logp_difference/max": 0.34592153429985045, "sampling/sampling_logp_difference/mean": 0.023168392293155194, "step": 15380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.6, "completions/max_terminated_length": 326.6, "completions/mean_length": 212.6, "completions/mean_terminated_length": 212.6, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.0391605239361525, "epoch": 1.4149122000551624, "frac_reward_zero_std": 0.6, "grad_norm": 0.08734679967164993, "kl": 0.011163545708404854, "learning_rate": 2.939192955438486e-06, "loss": 0.0005, "num_tokens": 67102514.0, "reward": -0.3, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.279094660282135, "sampling/importance_sampling_ratio/mean": 1.000207883119583, "sampling/importance_sampling_ratio/min": 0.7723838567733765, "sampling/sampling_logp_difference/max": 0.2868635892868042, "sampling/sampling_logp_difference/mean": 0.021180032938718795, "step": 15390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.5, "completions/max_terminated_length": 273.5, "completions/mean_length": 186.575, "completions/mean_terminated_length": 186.575, "completions/min_length": 111.2, "completions/min_terminated_length": 111.2, "entropy": 1.1031554158777, "epoch": 1.4158315712052956, "frac_reward_zero_std": 0.6, "grad_norm": 5.793660640716553, "kl": 0.00957282172748819, "learning_rate": 2.936698509487029e-06, "loss": 0.0006, "num_tokens": 67145080.0, "reward": -0.35, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2489987015724182, "sampling/importance_sampling_ratio/mean": 0.999462240934372, "sampling/importance_sampling_ratio/min": 0.7602699041366577, "sampling/sampling_logp_difference/max": 0.28111555576324465, "sampling/sampling_logp_difference/mean": 0.022299553640186785, "step": 15400 }, { "epoch": 1.4158315712052956, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 301.16579406631763, "eval_completions/max_terminated_length": 298.15706806282725, "eval_completions/mean_length": 207.92386561954623, "eval_completions/mean_terminated_length": 207.25147533583183, "eval_completions/min_length": 133.6910994764398, "eval_completions/min_terminated_length": 133.6910994764398, "eval_entropy": 1.079056989148025, "eval_frac_reward_zero_std": 0.47643979057591623, "eval_kl": 0.010669574132562984, "eval_loss": 0.0005518257385119796, "eval_num_tokens": 67145080.0, "eval_reward": -0.1099476439790576, "eval_reward_std": 0.4567038396058074, "eval_rewards/reward_fn/mean": -0.1099476439790576, "eval_rewards/reward_fn/std": 0.45670385240052175, "eval_runtime": 1750.694, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2761801997196405, "eval_sampling/importance_sampling_ratio/mean": 1.0000047034618118, "eval_sampling/importance_sampling_ratio/min": 0.7681578847646298, "eval_sampling/sampling_logp_difference/max": 0.297415319277562, "eval_sampling/sampling_logp_difference/mean": 0.020998836559996036, "eval_steps_per_second": 0.041, "step": 15400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.9, "completions/max_terminated_length": 293.4, "completions/mean_length": 210.95, "completions/mean_terminated_length": 207.05892944335938, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.079299521446228, "epoch": 1.4167509423554288, "frac_reward_zero_std": 0.8, "grad_norm": 6.779507160186768, "kl": 0.012451539764879272, "learning_rate": 2.9342036150111972e-06, "loss": 0.0006, "num_tokens": 67191220.0, "reward": 0.19375, "reward_std": 0.15909902453422547, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.15909903049468993, "sampling/importance_sampling_ratio/max": 1.2824450492858888, "sampling/importance_sampling_ratio/mean": 0.9997931957244873, "sampling/importance_sampling_ratio/min": 0.7552426606416702, "sampling/sampling_logp_difference/max": 0.3200062930583954, "sampling/sampling_logp_difference/mean": 0.022183324955403805, "step": 15410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 210.475, "completions/mean_terminated_length": 210.475, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.0985406026244164, "epoch": 1.4176703135055622, "frac_reward_zero_std": 0.6, "grad_norm": 0.12224598973989487, "kl": 0.010706716880667955, "learning_rate": 2.9317082745734476e-06, "loss": 0.0006, "num_tokens": 67234002.0, "reward": -0.325, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.2687348246574401, "sampling/importance_sampling_ratio/mean": 0.9997639179229736, "sampling/importance_sampling_ratio/min": 0.7564866602420807, "sampling/sampling_logp_difference/max": 0.295597505569458, "sampling/sampling_logp_difference/mean": 0.022438018396496773, "step": 15420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 205.6, "completions/mean_terminated_length": 205.6, "completions/min_length": 134.9, "completions/min_terminated_length": 134.9, "entropy": 1.1458257913589478, "epoch": 1.4185896846556956, "frac_reward_zero_std": 0.4, "grad_norm": 5.864645957946777, "kl": 0.011599271901650355, "learning_rate": 2.929212490736693e-06, "loss": 0.0005, "num_tokens": 67278618.0, "reward": 0.3, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.330785298347473, "sampling/importance_sampling_ratio/mean": 1.000210464000702, "sampling/importance_sampling_ratio/min": 0.7883044064044953, "sampling/sampling_logp_difference/max": 0.3024260580539703, "sampling/sampling_logp_difference/mean": 0.023162611573934556, "step": 15430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 184.05, "completions/mean_terminated_length": 184.05, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 1.0489317715167998, "epoch": 1.4195090558058288, "frac_reward_zero_std": 0.5, "grad_norm": 0.08111564069986343, "kl": 0.01019047323788982, "learning_rate": 2.926716266064306e-06, "loss": 0.0004, "num_tokens": 67315998.0, "reward": 0.25, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.244030499458313, "sampling/importance_sampling_ratio/mean": 1.0001695096492766, "sampling/importance_sampling_ratio/min": 0.7939502596855164, "sampling/sampling_logp_difference/max": 0.246437668800354, "sampling/sampling_logp_difference/mean": 0.02199369706213474, "step": 15440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 342.6, "completions/max_terminated_length": 341.5, "completions/mean_length": 243.4125, "completions/mean_terminated_length": 241.09107360839843, "completions/min_length": 161.1, "completions/min_terminated_length": 161.1, "entropy": 1.0988410905003547, "epoch": 1.4204284269559622, "frac_reward_zero_std": 0.3, "grad_norm": 5.2489542961120605, "kl": 0.011183875374263153, "learning_rate": 2.924219603120109e-06, "loss": 0.0008, "num_tokens": 67362479.0, "reward": 0.36875, "reward_std": 0.5218782171607017, "rewards/reward_fn/mean": 0.36875, "rewards/reward_fn/std": 0.5218782171607017, "sampling/importance_sampling_ratio/max": 1.2562772750854492, "sampling/importance_sampling_ratio/mean": 0.9995624125003815, "sampling/importance_sampling_ratio/min": 0.7972452700138092, "sampling/sampling_logp_difference/max": 0.24303098917007446, "sampling/sampling_logp_difference/mean": 0.021373021602630615, "step": 15450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.5, "completions/max_terminated_length": 254.5, "completions/mean_length": 186.2875, "completions/mean_terminated_length": 186.2875, "completions/min_length": 116.9, "completions/min_terminated_length": 116.9, "entropy": 1.0184183813631535, "epoch": 1.4213477981060954, "frac_reward_zero_std": 0.3, "grad_norm": 7.176331520080566, "kl": 0.011188534612301737, "learning_rate": 2.921722504468372e-06, "loss": 0.0001, "num_tokens": 67403894.0, "reward": -0.15, "reward_std": 0.6329763770103455, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6329763770103455, "sampling/importance_sampling_ratio/max": 1.338148331642151, "sampling/importance_sampling_ratio/mean": 0.9999184489250184, "sampling/importance_sampling_ratio/min": 0.8034662008285522, "sampling/sampling_logp_difference/max": 0.29759141206741335, "sampling/sampling_logp_difference/mean": 0.020330111496150494, "step": 15460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.8, "completions/max_terminated_length": 331.8, "completions/mean_length": 230.7, "completions/mean_terminated_length": 230.7, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.1521517045795917, "epoch": 1.4222671692562288, "frac_reward_zero_std": 0.4, "grad_norm": 5.392096519470215, "kl": 0.012591763061936944, "learning_rate": 2.9192249726738193e-06, "loss": 0.0003, "num_tokens": 67449142.0, "reward": -0.43125, "reward_std": 0.49446441531181334, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.49446442127227785, "sampling/importance_sampling_ratio/max": 1.3302367210388184, "sampling/importance_sampling_ratio/mean": 0.9998284101486206, "sampling/importance_sampling_ratio/min": 0.7364111214876174, "sampling/sampling_logp_difference/max": 0.3453613042831421, "sampling/sampling_logp_difference/mean": 0.02259857077151537, "step": 15470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.3, "completions/max_terminated_length": 280.3, "completions/mean_length": 193.1375, "completions/mean_terminated_length": 193.1375, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0919621676206588, "epoch": 1.4231865404063622, "frac_reward_zero_std": 0.7, "grad_norm": 5.3379669189453125, "kl": 0.013846669666236266, "learning_rate": 2.9167270103016143e-06, "loss": 0.0006, "num_tokens": 67489681.0, "reward": 0.0, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2310918331146241, "sampling/importance_sampling_ratio/mean": 1.0001094102859498, "sampling/importance_sampling_ratio/min": 0.7592034667730332, "sampling/sampling_logp_difference/max": 0.29655638337135315, "sampling/sampling_logp_difference/mean": 0.02175151314586401, "step": 15480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 212.05, "completions/mean_terminated_length": 212.05, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.0716444052755834, "epoch": 1.4241059115564954, "frac_reward_zero_std": 0.3, "grad_norm": 0.09140930324792862, "kl": 0.011279014206957072, "learning_rate": 2.914228619917365e-06, "loss": 0.0002, "num_tokens": 67532149.0, "reward": 0.175, "reward_std": 0.5824600756168365, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5824600756168365, "sampling/importance_sampling_ratio/max": 1.3346373319625855, "sampling/importance_sampling_ratio/mean": 0.999277538061142, "sampling/importance_sampling_ratio/min": 0.7762629628181458, "sampling/sampling_logp_difference/max": 0.38244717121124266, "sampling/sampling_logp_difference/mean": 0.02121913954615593, "step": 15490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 291.5, "completions/max_terminated_length": 267.2, "completions/mean_length": 196.9875, "completions/mean_terminated_length": 192.86785736083985, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.0051209434866906, "epoch": 1.4250252827066285, "frac_reward_zero_std": 0.3, "grad_norm": 0.07948829978704453, "kl": 0.010070866352180019, "learning_rate": 2.9117298040871183e-06, "loss": 0.0006, "num_tokens": 67571572.0, "reward": -0.00625, "reward_std": 0.7301128149032593, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.7301128506660461, "sampling/importance_sampling_ratio/max": 1.2380555391311645, "sampling/importance_sampling_ratio/mean": 1.0001094043254852, "sampling/importance_sampling_ratio/min": 0.7750185847282409, "sampling/sampling_logp_difference/max": 0.25968157052993773, "sampling/sampling_logp_difference/mean": 0.02054855078458786, "step": 15500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.2, "completions/max_terminated_length": 295.2, "completions/mean_length": 219.475, "completions/mean_terminated_length": 219.475, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.0993786953389644, "epoch": 1.425944653856762, "frac_reward_zero_std": 0.8, "grad_norm": 0.10531998425722122, "kl": 0.011013527325121685, "learning_rate": 2.909230565377359e-06, "loss": 0.0005, "num_tokens": 67616010.0, "reward": -0.1, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2691298484802247, "sampling/importance_sampling_ratio/mean": 0.9995814383029937, "sampling/importance_sampling_ratio/min": 0.7398152351379395, "sampling/sampling_logp_difference/max": 0.3216733574867249, "sampling/sampling_logp_difference/mean": 0.02210949007421732, "step": 15510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.7, "completions/max_terminated_length": 292.7, "completions/mean_length": 212.8, "completions/mean_terminated_length": 212.8, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.0876597829163075, "epoch": 1.4268640250068954, "frac_reward_zero_std": 0.3, "grad_norm": 5.837980270385742, "kl": 0.012738502759020776, "learning_rate": 2.9067309063550048e-06, "loss": 0.0005, "num_tokens": 67659770.0, "reward": -0.55, "reward_std": 0.5824443817138671, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.5824444055557251, "sampling/importance_sampling_ratio/max": 1.3113064408302306, "sampling/importance_sampling_ratio/mean": 0.9995974242687226, "sampling/importance_sampling_ratio/min": 0.742971533536911, "sampling/sampling_logp_difference/max": 0.3276235342025757, "sampling/sampling_logp_difference/mean": 0.021944118291139604, "step": 15520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.4, "completions/max_terminated_length": 301.4, "completions/mean_length": 213.2875, "completions/mean_terminated_length": 213.2875, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.016070468723774, "epoch": 1.4277833961570285, "frac_reward_zero_std": 0.4, "grad_norm": 6.883810997009277, "kl": 0.010841047667781822, "learning_rate": 2.904230829587407e-06, "loss": 0.0005, "num_tokens": 67701657.0, "reward": 0.425, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2577864050865173, "sampling/importance_sampling_ratio/mean": 1.0002722144126892, "sampling/importance_sampling_ratio/min": 0.7183625072240829, "sampling/sampling_logp_difference/max": 0.3466854989528656, "sampling/sampling_logp_difference/mean": 0.0207559397444129, "step": 15530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 354.1, "completions/max_terminated_length": 327.4, "completions/mean_length": 235.1375, "completions/mean_terminated_length": 229.51428833007813, "completions/min_length": 153.8, "completions/min_terminated_length": 153.8, "entropy": 1.1313384287059307, "epoch": 1.428702767307162, "frac_reward_zero_std": 0.2, "grad_norm": 4.723174571990967, "kl": 0.010688439389923587, "learning_rate": 2.9017303376423436e-06, "loss": 0.001, "num_tokens": 67752372.0, "reward": -0.3875, "reward_std": 0.7096456348896026, "rewards/reward_fn/mean": -0.3875, "rewards/reward_fn/std": 0.7096456348896026, "sampling/importance_sampling_ratio/max": 1.343810772895813, "sampling/importance_sampling_ratio/mean": 1.0001580655574798, "sampling/importance_sampling_ratio/min": 0.7595255613327027, "sampling/sampling_logp_difference/max": 0.35582557022571565, "sampling/sampling_logp_difference/mean": 0.02216667104512453, "step": 15540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.9, "completions/max_terminated_length": 302.9, "completions/mean_length": 218.1125, "completions/mean_terminated_length": 218.1125, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.0867278691381217, "epoch": 1.4296221384572951, "frac_reward_zero_std": 0.2, "grad_norm": 5.350491046905518, "kl": 0.010291214467724786, "learning_rate": 2.8992294330880203e-06, "loss": 0.0002, "num_tokens": 67796245.0, "reward": -0.25625, "reward_std": 0.5673228964209557, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.5673229083418846, "sampling/importance_sampling_ratio/max": 1.329602873325348, "sampling/importance_sampling_ratio/mean": 0.999646258354187, "sampling/importance_sampling_ratio/min": 0.7944300889968872, "sampling/sampling_logp_difference/max": 0.2988171219825745, "sampling/sampling_logp_difference/mean": 0.021632223948836328, "step": 15550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.1, "completions/max_terminated_length": 263.1, "completions/mean_length": 197.025, "completions/mean_terminated_length": 197.025, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.0279989577829838, "epoch": 1.4305415096074285, "frac_reward_zero_std": 0.6, "grad_norm": 0.10359691083431244, "kl": 0.010864170745480805, "learning_rate": 2.8967281184930683e-06, "loss": 0.001, "num_tokens": 67835663.0, "reward": -0.2, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.3874301195144654, "sampling/importance_sampling_ratio/mean": 1.0004807829856872, "sampling/importance_sampling_ratio/min": 0.7785882294178009, "sampling/sampling_logp_difference/max": 0.324761289358139, "sampling/sampling_logp_difference/mean": 0.0209744181483984, "step": 15560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.8, "completions/max_terminated_length": 278.8, "completions/mean_length": 194.7625, "completions/mean_terminated_length": 194.7625, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 0.9858302604407072, "epoch": 1.431460880757562, "frac_reward_zero_std": 0.2, "grad_norm": 0.08843714743852615, "kl": 0.01147694019600749, "learning_rate": 2.894226396426536e-06, "loss": 0.0006, "num_tokens": 67873788.0, "reward": -0.125, "reward_std": 0.7515525221824646, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.7515525817871094, "sampling/importance_sampling_ratio/max": 1.2397111415863038, "sampling/importance_sampling_ratio/mean": 0.9996126770973206, "sampling/importance_sampling_ratio/min": 0.7703938841819763, "sampling/sampling_logp_difference/max": 0.26672973334789274, "sampling/sampling_logp_difference/mean": 0.02049398496747017, "step": 15570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 204.5875, "completions/mean_terminated_length": 204.5875, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 0.9701444439589977, "epoch": 1.4323802519076951, "frac_reward_zero_std": 0.5, "grad_norm": 0.0760716125369072, "kl": 0.009334971295902505, "learning_rate": 2.8917242694578938e-06, "loss": 0.0002, "num_tokens": 67917595.0, "reward": 0.075, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2635969877243043, "sampling/importance_sampling_ratio/mean": 1.0004907369613647, "sampling/importance_sampling_ratio/min": 0.7727900862693786, "sampling/sampling_logp_difference/max": 0.2664857983589172, "sampling/sampling_logp_difference/mean": 0.019756142608821393, "step": 15580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.7, "completions/max_terminated_length": 311.7, "completions/mean_length": 204.8375, "completions/mean_terminated_length": 204.8375, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.0504800140857697, "epoch": 1.4332996230578283, "frac_reward_zero_std": 0.7, "grad_norm": 0.09173505008220673, "kl": 0.01020026192709338, "learning_rate": 2.8892217401570267e-06, "loss": 0.0005, "num_tokens": 67965070.0, "reward": 0.05, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.3059334635734559, "sampling/importance_sampling_ratio/mean": 1.0000239193439484, "sampling/importance_sampling_ratio/min": 0.7885531544685364, "sampling/sampling_logp_difference/max": 0.27959944009780885, "sampling/sampling_logp_difference/mean": 0.021532551012933256, "step": 15590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 353.1, "completions/max_terminated_length": 336.3, "completions/mean_length": 231.075, "completions/mean_terminated_length": 221.72250061035157, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.0426148276776075, "epoch": 1.4342189942079617, "frac_reward_zero_std": 0.4, "grad_norm": 5.362598896026611, "kl": 0.009781624400056898, "learning_rate": 2.886718811094232e-06, "loss": 0.0005, "num_tokens": 68010996.0, "reward": -0.09375, "reward_std": 0.6195390522480011, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.619539076089859, "sampling/importance_sampling_ratio/max": 1.3341090321540832, "sampling/importance_sampling_ratio/mean": 0.9996680378913879, "sampling/importance_sampling_ratio/min": 0.719517320394516, "sampling/sampling_logp_difference/max": 0.3870935797691345, "sampling/sampling_logp_difference/mean": 0.021231275610625745, "step": 15600 }, { "epoch": 1.4342189942079617, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 301.0698080279232, "eval_completions/max_terminated_length": 299.39441535776615, "eval_completions/mean_length": 209.88830715532285, "eval_completions/mean_terminated_length": 209.4426474046957, "eval_completions/min_length": 135.39092495637, "eval_completions/min_terminated_length": 135.39092495637, "eval_entropy": 1.085095188380536, "eval_frac_reward_zero_std": 0.4781849912739965, "eval_kl": 0.011111110632501243, "eval_loss": 0.0005451838369481266, "eval_num_tokens": 68010996.0, "eval_reward": -0.11136561954624782, "eval_reward_std": 0.46429109799612256, "eval_rewards/reward_fn/mean": -0.11136561954624782, "eval_rewards/reward_fn/std": 0.4642911135994327, "eval_runtime": 1748.8013, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.269124984949255, "eval_sampling/importance_sampling_ratio/mean": 1.0000402208606194, "eval_sampling/importance_sampling_ratio/min": 0.773673859410261, "eval_sampling/sampling_logp_difference/max": 0.283211799296201, "eval_sampling/sampling_logp_difference/mean": 0.02118357346534573, "eval_steps_per_second": 0.041, "step": 15600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 362.5, "completions/max_terminated_length": 362.5, "completions/mean_length": 240.3875, "completions/mean_terminated_length": 240.3875, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.182005836814642, "epoch": 1.4351383653580951, "frac_reward_zero_std": 0.5, "grad_norm": 4.9327239990234375, "kl": 0.011581230908632278, "learning_rate": 2.8842154848402187e-06, "loss": 0.0005, "num_tokens": 68064195.0, "reward": 0.15, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.316856563091278, "sampling/importance_sampling_ratio/mean": 0.9999304294586182, "sampling/importance_sampling_ratio/min": 0.731524720788002, "sampling/sampling_logp_difference/max": 0.3663831830024719, "sampling/sampling_logp_difference/mean": 0.023176556825637816, "step": 15610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 218.9375, "completions/mean_terminated_length": 218.9375, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0473126642405988, "epoch": 1.4360577365082283, "frac_reward_zero_std": 0.5, "grad_norm": 0.08700307458639145, "kl": 0.01346702542505227, "learning_rate": 2.8817117639661045e-06, "loss": 0.0004, "num_tokens": 68110342.0, "reward": -0.35, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2953277111053467, "sampling/importance_sampling_ratio/mean": 1.0007056176662446, "sampling/importance_sampling_ratio/min": 0.7974445343017578, "sampling/sampling_logp_difference/max": 0.2746346235275269, "sampling/sampling_logp_difference/mean": 0.020960763655602933, "step": 15620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.9, "completions/max_terminated_length": 310.9, "completions/mean_length": 203.5125, "completions/mean_terminated_length": 203.5125, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1179576940834521, "epoch": 1.4369771076583617, "frac_reward_zero_std": 0.6, "grad_norm": 0.09543260931968689, "kl": 0.011131227243458853, "learning_rate": 2.8792076510434102e-06, "loss": 0.0005, "num_tokens": 68153095.0, "reward": -0.725, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.725, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2507679104804992, "sampling/importance_sampling_ratio/mean": 0.9999574840068817, "sampling/importance_sampling_ratio/min": 0.7411674946546555, "sampling/sampling_logp_difference/max": 0.31779180765151976, "sampling/sampling_logp_difference/mean": 0.022001939453184606, "step": 15630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.6, "completions/max_terminated_length": 285.6, "completions/mean_length": 208.4125, "completions/mean_terminated_length": 208.4125, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.1216760493814946, "epoch": 1.437896478808495, "frac_reward_zero_std": 0.4, "grad_norm": 6.673154354095459, "kl": 0.010465775209013373, "learning_rate": 2.876703148644062e-06, "loss": 0.0002, "num_tokens": 68195464.0, "reward": 0.0, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2468612432479858, "sampling/importance_sampling_ratio/mean": 0.9996781170368194, "sampling/importance_sampling_ratio/min": 0.6778937608003617, "sampling/sampling_logp_difference/max": 0.40610122978687285, "sampling/sampling_logp_difference/mean": 0.022859008237719535, "step": 15640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.2, "completions/max_terminated_length": 260.2, "completions/mean_length": 194.75, "completions/mean_terminated_length": 194.75, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0002588637173175, "epoch": 1.4388158499586283, "frac_reward_zero_std": 0.5, "grad_norm": 5.628326892852783, "kl": 0.011068619857542216, "learning_rate": 2.874198259340383e-06, "loss": 0.0006, "num_tokens": 68234892.0, "reward": -0.125, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.266226863861084, "sampling/importance_sampling_ratio/mean": 0.9996753931045532, "sampling/importance_sampling_ratio/min": 0.7807260990142822, "sampling/sampling_logp_difference/max": 0.25951979160308836, "sampling/sampling_logp_difference/mean": 0.020838422980159522, "step": 15650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 207.2875, "completions/mean_terminated_length": 207.2875, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0318090885877609, "epoch": 1.4397352211087617, "frac_reward_zero_std": 0.5, "grad_norm": 4.592462539672852, "kl": 0.010729833898949437, "learning_rate": 2.871692985705096e-06, "loss": 0.0008, "num_tokens": 68277187.0, "reward": -0.13125, "reward_std": 0.35519086569547653, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.3551908776164055, "sampling/importance_sampling_ratio/max": 1.304776167869568, "sampling/importance_sampling_ratio/mean": 1.0002571225166321, "sampling/importance_sampling_ratio/min": 0.7795508921146392, "sampling/sampling_logp_difference/max": 0.28593101501464846, "sampling/sampling_logp_difference/mean": 0.021080773882567884, "step": 15660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 199.9, "completions/mean_terminated_length": 199.9, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.1121506705880164, "epoch": 1.440654592258895, "frac_reward_zero_std": 0.2, "grad_norm": 5.458993911743164, "kl": 0.011293947568628938, "learning_rate": 2.869187330311318e-06, "loss": 0.0004, "num_tokens": 68318203.0, "reward": -0.1, "reward_std": 0.6674932360649108, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6674932360649108, "sampling/importance_sampling_ratio/max": 1.258486795425415, "sampling/importance_sampling_ratio/mean": 1.0004348695278167, "sampling/importance_sampling_ratio/min": 0.8011007964611053, "sampling/sampling_logp_difference/max": 0.25389414429664614, "sampling/sampling_logp_difference/mean": 0.021519308723509312, "step": 15670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.4, "completions/max_terminated_length": 299.4, "completions/mean_length": 209.525, "completions/mean_terminated_length": 209.525, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.071841062605381, "epoch": 1.441573963409028, "frac_reward_zero_std": 0.5, "grad_norm": 5.088906764984131, "kl": 0.011663743306417017, "learning_rate": 2.866681295732557e-06, "loss": 0.0006, "num_tokens": 68363709.0, "reward": 0.275, "reward_std": 0.39729605317115785, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.39729605317115785, "sampling/importance_sampling_ratio/max": 1.3261524558067321, "sampling/importance_sampling_ratio/mean": 1.0003995835781097, "sampling/importance_sampling_ratio/min": 0.7887206077575684, "sampling/sampling_logp_difference/max": 0.3145547270774841, "sampling/sampling_logp_difference/mean": 0.0216961145401001, "step": 15680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.6, "completions/max_terminated_length": 247.6, "completions/mean_length": 177.0875, "completions/mean_terminated_length": 177.0875, "completions/min_length": 114.1, "completions/min_terminated_length": 114.1, "entropy": 1.1304203905165195, "epoch": 1.4424933345591615, "frac_reward_zero_std": 0.4, "grad_norm": 0.12688317894935608, "kl": 0.011885551788145677, "learning_rate": 2.8641748845427135e-06, "loss": 0.0001, "num_tokens": 68403356.0, "reward": -0.325, "reward_std": 0.5656446695327759, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5656446814537048, "sampling/importance_sampling_ratio/max": 1.2594794750213623, "sampling/importance_sampling_ratio/mean": 0.9999981880187988, "sampling/importance_sampling_ratio/min": 0.782099187374115, "sampling/sampling_logp_difference/max": 0.2634660005569458, "sampling/sampling_logp_difference/mean": 0.02254487741738558, "step": 15690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.1, "completions/max_terminated_length": 308.1, "completions/mean_length": 217.45, "completions/mean_terminated_length": 217.45, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0747482739388943, "epoch": 1.443412705709295, "frac_reward_zero_std": 0.7, "grad_norm": 5.451861381530762, "kl": 0.012478865904267878, "learning_rate": 2.8616680993160704e-06, "loss": 0.0006, "num_tokens": 68448808.0, "reward": -0.1, "reward_std": 0.29206851720809934, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.29206851720809934, "sampling/importance_sampling_ratio/max": 1.2506972074508667, "sampling/importance_sampling_ratio/mean": 0.9995209097862243, "sampling/importance_sampling_ratio/min": 0.7087956011295319, "sampling/sampling_logp_difference/max": 0.35738368034362794, "sampling/sampling_logp_difference/mean": 0.02171484362334013, "step": 15700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 208.2, "completions/mean_terminated_length": 208.2, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.078805061429739, "epoch": 1.444332076859428, "frac_reward_zero_std": 0.2, "grad_norm": 6.789592742919922, "kl": 0.010586558206705376, "learning_rate": 2.8591609426272987e-06, "loss": 0.0005, "num_tokens": 68494064.0, "reward": -0.05, "reward_std": 0.6237505674362183, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6237505674362183, "sampling/importance_sampling_ratio/max": 1.3614868521690369, "sampling/importance_sampling_ratio/mean": 1.000104546546936, "sampling/importance_sampling_ratio/min": 0.760021835565567, "sampling/sampling_logp_difference/max": 0.371289324760437, "sampling/sampling_logp_difference/mean": 0.021814503706991673, "step": 15710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 207.15, "completions/mean_terminated_length": 207.15, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0769534848630429, "epoch": 1.4452514480095615, "frac_reward_zero_std": 0.4, "grad_norm": 5.583586692810059, "kl": 0.012458079928182996, "learning_rate": 2.8566534170514484e-06, "loss": 0.0008, "num_tokens": 68539092.0, "reward": -0.1, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.2475173115730285, "sampling/importance_sampling_ratio/mean": 1.000686025619507, "sampling/importance_sampling_ratio/min": 0.7582113802433014, "sampling/sampling_logp_difference/max": 0.29674196243286133, "sampling/sampling_logp_difference/mean": 0.02170926295220852, "step": 15720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.1, "completions/max_terminated_length": 294.1, "completions/mean_length": 211.925, "completions/mean_terminated_length": 211.925, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.1709227181971074, "epoch": 1.4461708191596947, "frac_reward_zero_std": 0.7, "grad_norm": 0.10603852570056915, "kl": 0.012054099660599605, "learning_rate": 2.8541455251639494e-06, "loss": 0.0006, "num_tokens": 68581894.0, "reward": -0.025, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.2475736141204834, "sampling/importance_sampling_ratio/mean": 1.0004579603672028, "sampling/importance_sampling_ratio/min": 0.772126841545105, "sampling/sampling_logp_difference/max": 0.2632270693778992, "sampling/sampling_logp_difference/mean": 0.02284443937242031, "step": 15730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 198.25, "completions/mean_terminated_length": 198.25, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0682642094790935, "epoch": 1.447090190309828, "frac_reward_zero_std": 0.5, "grad_norm": 0.15300928056240082, "kl": 0.01142643985513132, "learning_rate": 2.8516372695406077e-06, "loss": 0.0008, "num_tokens": 68632146.0, "reward": -0.25, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.2640965223312377, "sampling/importance_sampling_ratio/mean": 0.9997036755084991, "sampling/importance_sampling_ratio/min": 0.7749470353126526, "sampling/sampling_logp_difference/max": 0.2828665167093277, "sampling/sampling_logp_difference/mean": 0.0216761352494359, "step": 15740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 194.125, "completions/mean_terminated_length": 194.125, "completions/min_length": 113.9, "completions/min_terminated_length": 113.9, "entropy": 1.0563524663448334, "epoch": 1.4480095614599615, "frac_reward_zero_std": 0.4, "grad_norm": 0.10483602434396744, "kl": 0.011891178961377591, "learning_rate": 2.8491286527576033e-06, "loss": 0.0005, "num_tokens": 68674748.0, "reward": -0.025, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.2911832451820373, "sampling/importance_sampling_ratio/mean": 0.9997623026371002, "sampling/importance_sampling_ratio/min": 0.7811301827430726, "sampling/sampling_logp_difference/max": 0.2905948042869568, "sampling/sampling_logp_difference/mean": 0.021330969594419002, "step": 15750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.3, "completions/max_terminated_length": 272.3, "completions/mean_length": 192.5875, "completions/mean_terminated_length": 192.5875, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.0030665853992105, "epoch": 1.4489289326100947, "frac_reward_zero_std": 0.4, "grad_norm": 5.836564540863037, "kl": 0.010106572727090679, "learning_rate": 2.846619677391486e-06, "loss": 0.0005, "num_tokens": 68718923.0, "reward": -0.375, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2663194417953492, "sampling/importance_sampling_ratio/mean": 0.999169933795929, "sampling/importance_sampling_ratio/min": 0.7763314068317413, "sampling/sampling_logp_difference/max": 0.25971164703369143, "sampling/sampling_logp_difference/mean": 0.020366671774536372, "step": 15760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 200.8375, "completions/mean_terminated_length": 200.8375, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0780627310276032, "epoch": 1.4498483037602279, "frac_reward_zero_std": 0.3, "grad_norm": 0.10444863885641098, "kl": 0.010051544359885155, "learning_rate": 2.8441103460191756e-06, "loss": 0.0006, "num_tokens": 68760086.0, "reward": -0.275, "reward_std": 0.538717406988144, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.538717406988144, "sampling/importance_sampling_ratio/max": 1.2454630613327027, "sampling/importance_sampling_ratio/mean": 1.0001631200313568, "sampling/importance_sampling_ratio/min": 0.7892050445079803, "sampling/sampling_logp_difference/max": 0.251979660987854, "sampling/sampling_logp_difference/mean": 0.02158531378954649, "step": 15770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.1, "completions/max_terminated_length": 304.1, "completions/mean_length": 207.3625, "completions/mean_terminated_length": 207.3625, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 1.0428454719483853, "epoch": 1.4507676749103613, "frac_reward_zero_std": 0.7, "grad_norm": 0.15705519914627075, "kl": 0.010947640304220841, "learning_rate": 2.8416006612179558e-06, "loss": 0.0004, "num_tokens": 68802427.0, "reward": 0.35, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2520138263702392, "sampling/importance_sampling_ratio/mean": 0.9993994176387787, "sampling/importance_sampling_ratio/min": 0.7457746803760529, "sampling/sampling_logp_difference/max": 0.3073705554008484, "sampling/sampling_logp_difference/mean": 0.02056888025254011, "step": 15780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.0, "completions/max_terminated_length": 309.0, "completions/mean_length": 211.725, "completions/mean_terminated_length": 211.725, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.1326336942613124, "epoch": 1.4516870460604947, "frac_reward_zero_std": 0.1, "grad_norm": 6.45364236831665, "kl": 0.011583275132579729, "learning_rate": 2.839090625565474e-06, "loss": 0.0002, "num_tokens": 68848717.0, "reward": 0.14375, "reward_std": 0.7873193502426148, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.7873193800449372, "sampling/importance_sampling_ratio/max": 1.274741291999817, "sampling/importance_sampling_ratio/mean": 1.0000174164772033, "sampling/importance_sampling_ratio/min": 0.7606618463993072, "sampling/sampling_logp_difference/max": 0.2903634011745453, "sampling/sampling_logp_difference/mean": 0.022394235245883465, "step": 15790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.7, "completions/max_terminated_length": 272.7, "completions/mean_length": 196.775, "completions/mean_terminated_length": 196.775, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 1.0901940241456032, "epoch": 1.4526064172106279, "frac_reward_zero_std": 0.4, "grad_norm": 0.1145005077123642, "kl": 0.012594662344781682, "learning_rate": 2.8365802416397393e-06, "loss": 0.0006, "num_tokens": 68891163.0, "reward": 0.1, "reward_std": 0.4898623704910278, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.48986239433288575, "sampling/importance_sampling_ratio/max": 1.301582932472229, "sampling/importance_sampling_ratio/mean": 0.9995537519454956, "sampling/importance_sampling_ratio/min": 0.7696180760860443, "sampling/sampling_logp_difference/max": 0.2995130240917206, "sampling/sampling_logp_difference/mean": 0.02206204440444708, "step": 15800 }, { "epoch": 1.4526064172106279, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 299.65794066317625, "eval_completions/max_terminated_length": 296.6910994764398, "eval_completions/mean_length": 205.50610820244327, "eval_completions/mean_terminated_length": 204.79495995539853, "eval_completions/min_length": 128.54101221640488, "eval_completions/min_terminated_length": 128.54101221640488, "eval_entropy": 1.0912503447208104, "eval_frac_reward_zero_std": 0.44153577661431065, "eval_kl": 0.011941948311523669, "eval_loss": 0.0006015050457790494, "eval_num_tokens": 68891163.0, "eval_reward": -0.08453315881326352, "eval_reward_std": 0.4805022332212688, "eval_rewards/reward_fn/mean": -0.08453315881326352, "eval_rewards/reward_fn/std": 0.48050224747229203, "eval_runtime": 1741.0149, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.2786098627103768, "eval_sampling/importance_sampling_ratio/mean": 1.0000600762808303, "eval_sampling/importance_sampling_ratio/min": 0.7634129571560998, "eval_sampling/sampling_logp_difference/max": 0.3001542441923997, "eval_sampling/sampling_logp_difference/mean": 0.02172371036134153, "eval_steps_per_second": 0.041, "step": 15800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.9, "completions/max_terminated_length": 319.9, "completions/mean_length": 216.325, "completions/mean_terminated_length": 216.325, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.069610559940338, "epoch": 1.4535257883607613, "frac_reward_zero_std": 0.3, "grad_norm": 6.974418640136719, "kl": 0.012363980372902006, "learning_rate": 2.8340695120191155e-06, "loss": 0.0003, "num_tokens": 68934645.0, "reward": -0.2, "reward_std": 0.6876311719417572, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6876312077045441, "sampling/importance_sampling_ratio/max": 1.250335431098938, "sampling/importance_sampling_ratio/mean": 0.9999209880828858, "sampling/importance_sampling_ratio/min": 0.7219511181116104, "sampling/sampling_logp_difference/max": 0.3420119762420654, "sampling/sampling_logp_difference/mean": 0.021786784194409846, "step": 15810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.2, "completions/max_terminated_length": 287.2, "completions/mean_length": 202.7625, "completions/mean_terminated_length": 202.7625, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0762132056057454, "epoch": 1.4544451595108945, "frac_reward_zero_std": 0.7, "grad_norm": 0.12667052447795868, "kl": 0.009874169260729104, "learning_rate": 2.8315584392823235e-06, "loss": 0.0003, "num_tokens": 68976826.0, "reward": -0.25, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2443912744522094, "sampling/importance_sampling_ratio/mean": 1.0003045737743377, "sampling/importance_sampling_ratio/min": 0.7689724802970886, "sampling/sampling_logp_difference/max": 0.28254628777503965, "sampling/sampling_logp_difference/mean": 0.021110586635768414, "step": 15820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 342.7, "completions/max_terminated_length": 333.3, "completions/mean_length": 240.225, "completions/mean_terminated_length": 237.5017883300781, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.059398716315627, "epoch": 1.4553645306610279, "frac_reward_zero_std": 0.2, "grad_norm": 0.10822849720716476, "kl": 0.011369555920828134, "learning_rate": 2.8290470260084374e-06, "loss": 0.0005, "num_tokens": 69023596.0, "reward": -0.48125, "reward_std": 0.6628375470638275, "rewards/reward_fn/mean": -0.48125, "rewards/reward_fn/std": 0.6628375589847565, "sampling/importance_sampling_ratio/max": 1.2806041717529297, "sampling/importance_sampling_ratio/mean": 0.9996991991996765, "sampling/importance_sampling_ratio/min": 0.7584553301334381, "sampling/sampling_logp_difference/max": 0.2859283804893494, "sampling/sampling_logp_difference/mean": 0.02099619610235095, "step": 15830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 199.1625, "completions/mean_terminated_length": 199.1625, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.121100052446127, "epoch": 1.4562839018111613, "frac_reward_zero_std": 0.7, "grad_norm": 4.907668113708496, "kl": 0.011057867648196407, "learning_rate": 2.8265352747768785e-06, "loss": 0.0006, "num_tokens": 69067129.0, "reward": -0.15, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2451226472854615, "sampling/importance_sampling_ratio/mean": 0.9994716167449951, "sampling/importance_sampling_ratio/min": 0.7755327343940734, "sampling/sampling_logp_difference/max": 0.26503958702087405, "sampling/sampling_logp_difference/mean": 0.02150552477687597, "step": 15840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.9, "completions/max_terminated_length": 321.9, "completions/mean_length": 237.6, "completions/mean_terminated_length": 237.6, "completions/min_length": 159.5, "completions/min_terminated_length": 159.5, "entropy": 1.1144039772450924, "epoch": 1.4572032729612945, "frac_reward_zero_std": 0.5, "grad_norm": 0.1555003523826599, "kl": 0.011142435914371163, "learning_rate": 2.824023188167418e-06, "loss": 0.0006, "num_tokens": 69112249.0, "reward": -0.3, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2704443335533142, "sampling/importance_sampling_ratio/mean": 1.000181359052658, "sampling/importance_sampling_ratio/min": 0.7822971045970917, "sampling/sampling_logp_difference/max": 0.2639064729213715, "sampling/sampling_logp_difference/mean": 0.02121233306825161, "step": 15850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 211.875, "completions/mean_terminated_length": 211.875, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.107230795174837, "epoch": 1.4581226441114277, "frac_reward_zero_std": 0.7, "grad_norm": 8.057519912719727, "kl": 0.010694772971328348, "learning_rate": 2.8215107687601694e-06, "loss": 0.0007, "num_tokens": 69156671.0, "reward": -0.05, "reward_std": 0.31392415761947634, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3139241814613342, "sampling/importance_sampling_ratio/max": 1.2981598019599914, "sampling/importance_sampling_ratio/mean": 0.9998953580856323, "sampling/importance_sampling_ratio/min": 0.7751563608646392, "sampling/sampling_logp_difference/max": 0.302254730463028, "sampling/sampling_logp_difference/mean": 0.02194340955466032, "step": 15860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 210.925, "completions/mean_terminated_length": 210.925, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0628220247104765, "epoch": 1.459042015261561, "frac_reward_zero_std": 0.4, "grad_norm": 6.356097221374512, "kl": 0.010931001885910518, "learning_rate": 2.8189980191355894e-06, "loss": 0.0003, "num_tokens": 69201601.0, "reward": 0.325, "reward_std": 0.5513221859931946, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.5513221979141235, "sampling/importance_sampling_ratio/max": 1.2385257005691528, "sampling/importance_sampling_ratio/mean": 1.000063705444336, "sampling/importance_sampling_ratio/min": 0.7439220368862152, "sampling/sampling_logp_difference/max": 0.31403729915618894, "sampling/sampling_logp_difference/mean": 0.020853255037218334, "step": 15870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 325.0, "completions/max_terminated_length": 305.4, "completions/mean_length": 218.4125, "completions/mean_terminated_length": 215.0464294433594, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.1048476420342923, "epoch": 1.4599613864116945, "frac_reward_zero_std": 0.5, "grad_norm": 0.13014543056488037, "kl": 0.01007257039891556, "learning_rate": 2.816484941874472e-06, "loss": 0.0004, "num_tokens": 69245946.0, "reward": 0.19375, "reward_std": 0.4573550045490265, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.4573550224304199, "sampling/importance_sampling_ratio/max": 1.2810516715049745, "sampling/importance_sampling_ratio/mean": 1.00033957362175, "sampling/importance_sampling_ratio/min": 0.7962487459182739, "sampling/sampling_logp_difference/max": 0.25525341033935545, "sampling/sampling_logp_difference/mean": 0.021803479827940463, "step": 15880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.3, "completions/max_terminated_length": 297.3, "completions/mean_length": 213.6875, "completions/mean_terminated_length": 213.6875, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.1172901652753353, "epoch": 1.4608807575618277, "frac_reward_zero_std": 0.5, "grad_norm": 6.363350868225098, "kl": 0.010594460769789293, "learning_rate": 2.8139715395579515e-06, "loss": 0.0004, "num_tokens": 69290433.0, "reward": 0.1, "reward_std": 0.5134106636047363, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5134106874465942, "sampling/importance_sampling_ratio/max": 1.2507949948310852, "sampling/importance_sampling_ratio/mean": 0.9999550700187683, "sampling/importance_sampling_ratio/min": 0.7971917510032653, "sampling/sampling_logp_difference/max": 0.2504700720310211, "sampling/sampling_logp_difference/mean": 0.02171371504664421, "step": 15890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.2, "completions/max_terminated_length": 298.2, "completions/mean_length": 206.2125, "completions/mean_terminated_length": 206.2125, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 0.957826292514801, "epoch": 1.461800128711961, "frac_reward_zero_std": 0.5, "grad_norm": 6.2479376792907715, "kl": 0.010145365618518554, "learning_rate": 2.8114578147674904e-06, "loss": 0.0005, "num_tokens": 69332810.0, "reward": 0.175, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2260746836662293, "sampling/importance_sampling_ratio/mean": 0.999755859375, "sampling/importance_sampling_ratio/min": 0.782980614900589, "sampling/sampling_logp_difference/max": 0.2498856544494629, "sampling/sampling_logp_difference/mean": 0.019686573557555674, "step": 15900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.5, "completions/max_terminated_length": 312.5, "completions/mean_length": 224.4375, "completions/mean_terminated_length": 224.4375, "completions/min_length": 148.6, "completions/min_terminated_length": 148.6, "entropy": 1.0819176726043225, "epoch": 1.4627194998620943, "frac_reward_zero_std": 0.5, "grad_norm": 5.783807754516602, "kl": 0.010686644306406378, "learning_rate": 2.808943770084888e-06, "loss": 0.001, "num_tokens": 69383749.0, "reward": -0.025, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2699370980262756, "sampling/importance_sampling_ratio/mean": 0.9997581958770752, "sampling/importance_sampling_ratio/min": 0.8005149900913239, "sampling/sampling_logp_difference/max": 0.24851916432380677, "sampling/sampling_logp_difference/mean": 0.02085205027833581, "step": 15910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 208.0125, "completions/mean_terminated_length": 208.0125, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0174452368170024, "epoch": 1.4636388710122277, "frac_reward_zero_std": 0.4, "grad_norm": 4.982801914215088, "kl": 0.010841008310671895, "learning_rate": 2.8064294080922685e-06, "loss": 0.0005, "num_tokens": 69425550.0, "reward": -0.1, "reward_std": 0.5554606795310975, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5554607272148132, "sampling/importance_sampling_ratio/max": 1.2421096324920655, "sampling/importance_sampling_ratio/mean": 0.9999293088912964, "sampling/importance_sampling_ratio/min": 0.7804954349994659, "sampling/sampling_logp_difference/max": 0.26187588572502135, "sampling/sampling_logp_difference/mean": 0.020360058918595315, "step": 15920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.2, "completions/max_terminated_length": 286.0, "completions/mean_length": 214.475, "completions/mean_terminated_length": 212.28928833007814, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0619285121560096, "epoch": 1.464558242162361, "frac_reward_zero_std": 0.4, "grad_norm": 6.93414831161499, "kl": 0.011420263961190358, "learning_rate": 2.803914731372084e-06, "loss": 0.0004, "num_tokens": 69472100.0, "reward": -0.05625, "reward_std": 0.5642594993114471, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.5642595171928406, "sampling/importance_sampling_ratio/max": 1.2411105394363404, "sampling/importance_sampling_ratio/mean": 1.0002029597759248, "sampling/importance_sampling_ratio/min": 0.7764306604862213, "sampling/sampling_logp_difference/max": 0.2711660146713257, "sampling/sampling_logp_difference/mean": 0.02100929133594036, "step": 15930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 189.025, "completions/mean_terminated_length": 189.025, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.082667787373066, "epoch": 1.4654776133124943, "frac_reward_zero_std": 0.6, "grad_norm": 0.12196296453475952, "kl": 0.01217591076856479, "learning_rate": 2.801399742507108e-06, "loss": 0.0004, "num_tokens": 69513742.0, "reward": 0.225, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.3244661450386048, "sampling/importance_sampling_ratio/mean": 1.0001570761203766, "sampling/importance_sampling_ratio/min": 0.7557935297489167, "sampling/sampling_logp_difference/max": 0.338392448425293, "sampling/sampling_logp_difference/mean": 0.021790358424186706, "step": 15940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 199.8375, "completions/mean_terminated_length": 199.8375, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "entropy": 1.0194630384445191, "epoch": 1.4663969844626275, "frac_reward_zero_std": 0.5, "grad_norm": 6.649239540100098, "kl": 0.010348373203305528, "learning_rate": 2.798884444080437e-06, "loss": 0.0007, "num_tokens": 69555145.0, "reward": -0.125, "reward_std": 0.3535533845424652, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2537140727043152, "sampling/importance_sampling_ratio/mean": 0.9999072849750519, "sampling/importance_sampling_ratio/min": 0.7823604702949524, "sampling/sampling_logp_difference/max": 0.2591871201992035, "sampling/sampling_logp_difference/mean": 0.02048201095312834, "step": 15950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.4, "completions/max_terminated_length": 270.4, "completions/mean_length": 191.0875, "completions/mean_terminated_length": 191.0875, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.1218218505382538, "epoch": 1.4673163556127609, "frac_reward_zero_std": 0.6, "grad_norm": 0.10319480299949646, "kl": 0.011607227858621626, "learning_rate": 2.796368838675483e-06, "loss": 0.0011, "num_tokens": 69598048.0, "reward": 0.25, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.259937572479248, "sampling/importance_sampling_ratio/mean": 1.0002452909946442, "sampling/importance_sampling_ratio/min": 0.7917516946792602, "sampling/sampling_logp_difference/max": 0.25050928592681887, "sampling/sampling_logp_difference/mean": 0.022210326604545118, "step": 15960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.2, "completions/max_terminated_length": 251.2, "completions/mean_length": 174.1625, "completions/mean_terminated_length": 174.1625, "completions/min_length": 107.1, "completions/min_terminated_length": 107.1, "entropy": 0.9591636836528779, "epoch": 1.4682357267628943, "frac_reward_zero_std": 0.1, "grad_norm": 0.08852626383304596, "kl": 0.011604238289874048, "learning_rate": 2.7938529288759748e-06, "loss": -0.0003, "num_tokens": 69635877.0, "reward": 0.15, "reward_std": 0.8475291967391968, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.8475292444229126, "sampling/importance_sampling_ratio/max": 1.2549394726753236, "sampling/importance_sampling_ratio/mean": 0.9994439661502839, "sampling/importance_sampling_ratio/min": 0.7542195379734039, "sampling/sampling_logp_difference/max": 0.31176605224609377, "sampling/sampling_logp_difference/mean": 0.020705741457641126, "step": 15970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.8, "completions/max_terminated_length": 284.8, "completions/mean_length": 196.575, "completions/mean_terminated_length": 196.575, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.040791779011488, "epoch": 1.4691550979130275, "frac_reward_zero_std": 0.1, "grad_norm": 6.318051815032959, "kl": 0.01310765098896809, "learning_rate": 2.7913367172659535e-06, "loss": 0.0001, "num_tokens": 69677291.0, "reward": 0.125, "reward_std": 0.7785362243652344, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.7785362601280212, "sampling/importance_sampling_ratio/max": 1.264781391620636, "sampling/importance_sampling_ratio/mean": 0.9998329699039459, "sampling/importance_sampling_ratio/min": 0.7777739822864532, "sampling/sampling_logp_difference/max": 0.2750908136367798, "sampling/sampling_logp_difference/mean": 0.021673418395221234, "step": 15980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.1, "completions/max_terminated_length": 310.1, "completions/mean_length": 209.6875, "completions/mean_terminated_length": 209.6875, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.0621080346405507, "epoch": 1.4700744690631609, "frac_reward_zero_std": 0.4, "grad_norm": 5.189009666442871, "kl": 0.016128375241532922, "learning_rate": 2.78882020642977e-06, "loss": 0.001, "num_tokens": 69723954.0, "reward": 0.225, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.2825523972511292, "sampling/importance_sampling_ratio/mean": 0.9998611509799957, "sampling/importance_sampling_ratio/min": 0.7488759845495224, "sampling/sampling_logp_difference/max": 0.3332488238811493, "sampling/sampling_logp_difference/mean": 0.021986459754407404, "step": 15990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 285.5, "completions/max_terminated_length": 263.5, "completions/mean_length": 197.3875, "completions/mean_terminated_length": 193.91607208251952, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 1.0905563704669476, "epoch": 1.470993840213294, "frac_reward_zero_std": 0.7, "grad_norm": 5.569067478179932, "kl": 0.011118243483360856, "learning_rate": 2.7863033989520826e-06, "loss": 0.0007, "num_tokens": 69766753.0, "reward": -0.40625, "reward_std": 0.15909902304410933, "rewards/reward_fn/mean": -0.40625, "rewards/reward_fn/std": 0.15909902304410933, "sampling/importance_sampling_ratio/max": 1.2648885011672975, "sampling/importance_sampling_ratio/mean": 1.000233656167984, "sampling/importance_sampling_ratio/min": 0.7295570790767669, "sampling/sampling_logp_difference/max": 0.3340642273426056, "sampling/sampling_logp_difference/mean": 0.02143674176186323, "step": 16000 }, { "epoch": 1.470993840213294, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 289.1151832460733, "eval_completions/max_terminated_length": 287.9197207678883, "eval_completions/mean_length": 201.96575043630017, "eval_completions/mean_terminated_length": 201.6029607163674, "eval_completions/min_length": 131.61431064572426, "eval_completions/min_terminated_length": 131.61431064572426, "eval_entropy": 1.0707996139663676, "eval_frac_reward_zero_std": 0.5026178010471204, "eval_kl": 0.010951509883702061, "eval_loss": 0.000540945038665086, "eval_num_tokens": 69766753.0, "eval_reward": -0.10929319371727748, "eval_reward_std": 0.437206458332859, "eval_rewards/reward_fn/mean": -0.10929319371727748, "eval_rewards/reward_fn/std": 0.4372064718557278, "eval_runtime": 1689.7134, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2844911000283392, "eval_sampling/importance_sampling_ratio/mean": 0.9999828625723953, "eval_sampling/importance_sampling_ratio/min": 0.7720420904600599, "eval_sampling/sampling_logp_difference/max": 0.2963079298093889, "eval_sampling/sampling_logp_difference/mean": 0.021188579832917213, "eval_steps_per_second": 0.043, "step": 16000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.5, "completions/max_terminated_length": 316.5, "completions/mean_length": 222.1375, "completions/mean_terminated_length": 222.1375, "completions/min_length": 153.3, "completions/min_terminated_length": 153.3, "entropy": 1.0938123688101768, "epoch": 1.4719132113634275, "frac_reward_zero_std": 0.3, "grad_norm": 0.11407694965600967, "kl": 0.013046811841195449, "learning_rate": 2.783786297417854e-06, "loss": 0.0008, "num_tokens": 69812420.0, "reward": 0.24375, "reward_std": 0.5979742467403412, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.5979742646217346, "sampling/importance_sampling_ratio/max": 1.274784791469574, "sampling/importance_sampling_ratio/mean": 1.0001179337501527, "sampling/importance_sampling_ratio/min": 0.7648359596729278, "sampling/sampling_logp_difference/max": 0.27716822624206544, "sampling/sampling_logp_difference/mean": 0.022547826543450354, "step": 16010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.7, "completions/max_terminated_length": 332.7, "completions/mean_length": 235.6, "completions/mean_terminated_length": 235.6, "completions/min_length": 161.9, "completions/min_terminated_length": 161.9, "entropy": 1.16179903075099, "epoch": 1.4728325825135609, "frac_reward_zero_std": 0.6, "grad_norm": 0.08248589187860489, "kl": 0.011589195442502387, "learning_rate": 2.7812689044123515e-06, "loss": 0.0004, "num_tokens": 69860892.0, "reward": 0.2, "reward_std": 0.3552303433418274, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3552303433418274, "sampling/importance_sampling_ratio/max": 1.375730276107788, "sampling/importance_sampling_ratio/mean": 0.9999300181865692, "sampling/importance_sampling_ratio/min": 0.7866957426071167, "sampling/sampling_logp_difference/max": 0.32044545412063596, "sampling/sampling_logp_difference/mean": 0.02272218968719244, "step": 16020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.3, "completions/max_terminated_length": 316.3, "completions/mean_length": 217.2875, "completions/mean_terminated_length": 217.2875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.048665764182806, "epoch": 1.473751953663694, "frac_reward_zero_std": 0.4, "grad_norm": 6.8184380531311035, "kl": 0.011181362811475993, "learning_rate": 2.7787512225211356e-06, "loss": 0.0004, "num_tokens": 69904731.0, "reward": -0.2, "reward_std": 0.515128368139267, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5151283800601959, "sampling/importance_sampling_ratio/max": 1.3179278016090392, "sampling/importance_sampling_ratio/mean": 0.9999259173870086, "sampling/importance_sampling_ratio/min": 0.777493292093277, "sampling/sampling_logp_difference/max": 0.2875626921653748, "sampling/sampling_logp_difference/mean": 0.021112887002527714, "step": 16030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 197.825, "completions/mean_terminated_length": 197.825, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.024294525384903, "epoch": 1.4746713248138272, "frac_reward_zero_std": 0.3, "grad_norm": 4.514741897583008, "kl": 0.01474850078811869, "learning_rate": 2.776233254330071e-06, "loss": 0.0001, "num_tokens": 69943277.0, "reward": -0.425, "reward_std": 0.6767033517360688, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.6767033755779266, "sampling/importance_sampling_ratio/max": 1.2461853623390198, "sampling/importance_sampling_ratio/mean": 0.9999113023281098, "sampling/importance_sampling_ratio/min": 0.7261159121990204, "sampling/sampling_logp_difference/max": 0.34534232020378114, "sampling/sampling_logp_difference/mean": 0.021169849298894407, "step": 16040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 213.25, "completions/mean_terminated_length": 213.25, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0876991264522076, "epoch": 1.4755906959639606, "frac_reward_zero_std": 0.8, "grad_norm": 0.12288016825914383, "kl": 0.010751966619864106, "learning_rate": 2.7737150024253107e-06, "loss": 0.0006, "num_tokens": 69988097.0, "reward": 0.175, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2292598366737366, "sampling/importance_sampling_ratio/mean": 1.0000931560993194, "sampling/importance_sampling_ratio/min": 0.7770973980426789, "sampling/sampling_logp_difference/max": 0.2629625082015991, "sampling/sampling_logp_difference/mean": 0.020700944401323796, "step": 16050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.3, "completions/max_terminated_length": 262.3, "completions/mean_length": 197.7375, "completions/mean_terminated_length": 197.7375, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0398176692426204, "epoch": 1.476510067114094, "frac_reward_zero_std": 0.6, "grad_norm": 5.6576080322265625, "kl": 0.01580139233265072, "learning_rate": 2.7711964693933024e-06, "loss": 0.001, "num_tokens": 70031252.0, "reward": -0.05, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.2476849198341369, "sampling/importance_sampling_ratio/mean": 1.000134426355362, "sampling/importance_sampling_ratio/min": 0.7818365156650543, "sampling/sampling_logp_difference/max": 0.2555751860141754, "sampling/sampling_logp_difference/mean": 0.020858503133058547, "step": 16060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 288.3, "completions/max_terminated_length": 272.0, "completions/mean_length": 206.2125, "completions/mean_terminated_length": 203.40535888671874, "completions/min_length": 142.8, "completions/min_terminated_length": 142.8, "entropy": 1.062238298356533, "epoch": 1.4774294382642272, "frac_reward_zero_std": 0.5, "grad_norm": 5.785917282104492, "kl": 0.011291472002631054, "learning_rate": 2.7686776578207804e-06, "loss": 0.0009, "num_tokens": 70073021.0, "reward": 0.06875, "reward_std": 0.447772878408432, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.4477728962898254, "sampling/importance_sampling_ratio/max": 1.3564360737800598, "sampling/importance_sampling_ratio/mean": 0.9994925081729888, "sampling/importance_sampling_ratio/min": 0.7933215439319611, "sampling/sampling_logp_difference/max": 0.32502213716506956, "sampling/sampling_logp_difference/mean": 0.02137624826282263, "step": 16070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 203.65, "completions/mean_terminated_length": 203.65, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.0390592347830534, "epoch": 1.4783488094143606, "frac_reward_zero_std": 0.3, "grad_norm": 4.497909069061279, "kl": 0.011604944244027138, "learning_rate": 2.7661585702947668e-06, "loss": 0.0005, "num_tokens": 70115761.0, "reward": -0.1, "reward_std": 0.6480426907539367, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6480427384376526, "sampling/importance_sampling_ratio/max": 1.2559283256530762, "sampling/importance_sampling_ratio/mean": 0.9996781408786773, "sampling/importance_sampling_ratio/min": 0.7718812048435211, "sampling/sampling_logp_difference/max": 0.27634879350662234, "sampling/sampling_logp_difference/mean": 0.02118032965809107, "step": 16080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.2, "completions/max_terminated_length": 295.2, "completions/mean_length": 209.175, "completions/mean_terminated_length": 209.175, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 0.913833899050951, "epoch": 1.4792681805644938, "frac_reward_zero_std": 0.6, "grad_norm": 4.247652530670166, "kl": 0.010876337063382379, "learning_rate": 2.7636392094025667e-06, "loss": 0.0004, "num_tokens": 70157999.0, "reward": -0.025, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2481418013572694, "sampling/importance_sampling_ratio/mean": 0.9999559044837951, "sampling/importance_sampling_ratio/min": 0.7180305659770966, "sampling/sampling_logp_difference/max": 0.3644193857908249, "sampling/sampling_logp_difference/mean": 0.018908297084271906, "step": 16090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 364.3, "completions/max_terminated_length": 349.5, "completions/mean_length": 229.325, "completions/mean_terminated_length": 226.5642883300781, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.1730732873082161, "epoch": 1.4801875517146272, "frac_reward_zero_std": 0.4, "grad_norm": 7.107844829559326, "kl": 0.01356487869634293, "learning_rate": 2.761119577731765e-06, "loss": 0.0009, "num_tokens": 70205937.0, "reward": -0.36875, "reward_std": 0.5105068549513817, "rewards/reward_fn/mean": -0.36875, "rewards/reward_fn/std": 0.5105068549513817, "sampling/importance_sampling_ratio/max": 1.3057173848152162, "sampling/importance_sampling_ratio/mean": 1.0004940271377563, "sampling/importance_sampling_ratio/min": 0.7604783087968826, "sampling/sampling_logp_difference/max": 0.3337121486663818, "sampling/sampling_logp_difference/mean": 0.022207452356815337, "step": 16100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.5, "completions/max_terminated_length": 314.5, "completions/mean_length": 208.8875, "completions/mean_terminated_length": 208.8875, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.1030845869332553, "epoch": 1.4811069228647604, "frac_reward_zero_std": 0.6, "grad_norm": 7.462610721588135, "kl": 0.011513891993672587, "learning_rate": 2.7585996778702257e-06, "loss": 0.0005, "num_tokens": 70250752.0, "reward": -0.325, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.3373195290565492, "sampling/importance_sampling_ratio/mean": 0.9998654782772064, "sampling/importance_sampling_ratio/min": 0.7572539806365967, "sampling/sampling_logp_difference/max": 0.3773972660303116, "sampling/sampling_logp_difference/mean": 0.021470255218446255, "step": 16110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 343.7, "completions/max_terminated_length": 343.7, "completions/mean_length": 235.6, "completions/mean_terminated_length": 235.6, "completions/min_length": 169.2, "completions/min_terminated_length": 169.2, "entropy": 1.1653615035116673, "epoch": 1.4820262940148938, "frac_reward_zero_std": 0.7, "grad_norm": 0.090870201587677, "kl": 0.010437624997575768, "learning_rate": 2.756079512406088e-06, "loss": 0.0005, "num_tokens": 70302512.0, "reward": -0.53125, "reward_std": 0.2612631618976593, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.26126316785812376, "sampling/importance_sampling_ratio/max": 1.3385656237602235, "sampling/importance_sampling_ratio/mean": 0.9998770356178284, "sampling/importance_sampling_ratio/min": 0.7791520059108734, "sampling/sampling_logp_difference/max": 0.3140501916408539, "sampling/sampling_logp_difference/mean": 0.02177283149212599, "step": 16120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.5, "completions/max_terminated_length": 313.5, "completions/mean_length": 219.325, "completions/mean_terminated_length": 219.325, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.112204209715128, "epoch": 1.482945665165027, "frac_reward_zero_std": 0.2, "grad_norm": 7.705333709716797, "kl": 0.014053219958441332, "learning_rate": 2.7535590839277638e-06, "loss": 0.0011, "num_tokens": 70348906.0, "reward": -0.5, "reward_std": 0.7296811878681183, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.7296812474727631, "sampling/importance_sampling_ratio/max": 1.3074647307395935, "sampling/importance_sampling_ratio/mean": 1.0002061426639557, "sampling/importance_sampling_ratio/min": 0.7751895129680634, "sampling/sampling_logp_difference/max": 0.28151928782463076, "sampling/sampling_logp_difference/mean": 0.021770511381328107, "step": 16130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 344.1, "completions/max_terminated_length": 344.1, "completions/mean_length": 223.2125, "completions/mean_terminated_length": 223.2125, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.0787090487778186, "epoch": 1.4838650363151604, "frac_reward_zero_std": 0.3, "grad_norm": 5.648470401763916, "kl": 0.010128090844955296, "learning_rate": 2.7510383950239342e-06, "loss": 0.0008, "num_tokens": 70395851.0, "reward": 0.05, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.2871819138526917, "sampling/importance_sampling_ratio/mean": 0.9998345792293548, "sampling/importance_sampling_ratio/min": 0.7612739861011505, "sampling/sampling_logp_difference/max": 0.2912806898355484, "sampling/sampling_logp_difference/mean": 0.021011025831103324, "step": 16140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 343.8, "completions/max_terminated_length": 328.1, "completions/mean_length": 245.4, "completions/mean_terminated_length": 242.66607360839845, "completions/min_length": 174.8, "completions/min_terminated_length": 174.8, "entropy": 1.1114398427307606, "epoch": 1.4847844074652938, "frac_reward_zero_std": 0.3, "grad_norm": 0.21279549598693848, "kl": 0.013249842345248907, "learning_rate": 2.7485174482835513e-06, "loss": 0.0007, "num_tokens": 70444195.0, "reward": -0.23125, "reward_std": 0.6435969233512878, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.6435969471931458, "sampling/importance_sampling_ratio/max": 1.4812996983528137, "sampling/importance_sampling_ratio/mean": 0.9996077001094819, "sampling/importance_sampling_ratio/min": 0.7345545262098312, "sampling/sampling_logp_difference/max": 0.44191122651100156, "sampling/sampling_logp_difference/mean": 0.021410958655178548, "step": 16150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 193.55, "completions/mean_terminated_length": 193.55, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.0694488056004048, "epoch": 1.485703778615427, "frac_reward_zero_std": 0.4, "grad_norm": 0.10802203416824341, "kl": 0.017555760638788342, "learning_rate": 2.7459962462958273e-06, "loss": 0.0006, "num_tokens": 70482975.0, "reward": -0.125, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.2916401386260987, "sampling/importance_sampling_ratio/mean": 1.0004652559757232, "sampling/importance_sampling_ratio/min": 0.7718721270561218, "sampling/sampling_logp_difference/max": 0.2997742295265198, "sampling/sampling_logp_difference/mean": 0.021838624961674213, "step": 16160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.7, "completions/max_terminated_length": 308.7, "completions/mean_length": 237.375, "completions/mean_terminated_length": 237.375, "completions/min_length": 162.9, "completions/min_terminated_length": 162.9, "entropy": 1.088700482994318, "epoch": 1.4866231497655604, "frac_reward_zero_std": 0.3, "grad_norm": 0.10548653453588486, "kl": 0.0133067065034993, "learning_rate": 2.7434747916502386e-06, "loss": 0.0005, "num_tokens": 70531189.0, "reward": -0.075, "reward_std": 0.6514373540878295, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6514373898506165, "sampling/importance_sampling_ratio/max": 1.2543985843658447, "sampling/importance_sampling_ratio/mean": 0.9998445153236389, "sampling/importance_sampling_ratio/min": 0.7894846498966217, "sampling/sampling_logp_difference/max": 0.2592636406421661, "sampling/sampling_logp_difference/mean": 0.020838474109768867, "step": 16170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 217.4375, "completions/mean_terminated_length": 217.4375, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.0935436330735684, "epoch": 1.4875425209156936, "frac_reward_zero_std": 0.7, "grad_norm": 0.08466136455535889, "kl": 0.01134665617137216, "learning_rate": 2.740953086936523e-06, "loss": 0.0006, "num_tokens": 70575024.0, "reward": -0.3, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.372118628025055, "sampling/importance_sampling_ratio/mean": 1.0004047870635986, "sampling/importance_sampling_ratio/min": 0.7626102149486542, "sampling/sampling_logp_difference/max": 0.3553152322769165, "sampling/sampling_logp_difference/mean": 0.021450469642877577, "step": 16180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.7, "completions/max_terminated_length": 307.7, "completions/mean_length": 220.825, "completions/mean_terminated_length": 220.825, "completions/min_length": 155.8, "completions/min_terminated_length": 155.8, "entropy": 1.0456454031169415, "epoch": 1.488461892065827, "frac_reward_zero_std": 0.6, "grad_norm": 0.121537946164608, "kl": 0.01154740778147243, "learning_rate": 2.738431134744672e-06, "loss": 0.0008, "num_tokens": 70619298.0, "reward": 0.0, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2435135960578918, "sampling/importance_sampling_ratio/mean": 0.9999648571014405, "sampling/importance_sampling_ratio/min": 0.7698234617710114, "sampling/sampling_logp_difference/max": 0.2732710838317871, "sampling/sampling_logp_difference/mean": 0.0205126253888011, "step": 16190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 219.6125, "completions/mean_terminated_length": 219.6125, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.1587072677910328, "epoch": 1.4893812632159602, "frac_reward_zero_std": 0.5, "grad_norm": 4.520900726318359, "kl": 0.011118094704579562, "learning_rate": 2.7359089376649328e-06, "loss": 0.0006, "num_tokens": 70664707.0, "reward": 0.3, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.237620770931244, "sampling/importance_sampling_ratio/mean": 1.0000795602798462, "sampling/importance_sampling_ratio/min": 0.7774447679519654, "sampling/sampling_logp_difference/max": 0.266456139087677, "sampling/sampling_logp_difference/mean": 0.022148588858544826, "step": 16200 }, { "epoch": 1.4893812632159602, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 301.4083769633508, "eval_completions/max_terminated_length": 298.7190226876091, "eval_completions/mean_length": 211.5235602094241, "eval_completions/mean_terminated_length": 210.83981570897927, "eval_completions/min_length": 138.6195462478185, "eval_completions/min_terminated_length": 138.6195462478185, "eval_entropy": 1.1013988604408285, "eval_frac_reward_zero_std": 0.4677137870855148, "eval_kl": 0.010997087276178244, "eval_loss": 0.0005486948066391051, "eval_num_tokens": 70664707.0, "eval_reward": -0.10951134380453752, "eval_reward_std": 0.4585722701243705, "eval_rewards/reward_fn/mean": -0.10951134380453752, "eval_rewards/reward_fn/std": 0.4585722838552834, "eval_runtime": 1750.8256, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2820920045463202, "eval_sampling/importance_sampling_ratio/mean": 1.0000462776495316, "eval_sampling/importance_sampling_ratio/min": 0.773680169544919, "eval_sampling/sampling_logp_difference/max": 0.2933303446461809, "eval_sampling/sampling_logp_difference/mean": 0.021248183040887793, "eval_steps_per_second": 0.041, "step": 16200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.2, "completions/max_terminated_length": 288.2, "completions/mean_length": 216.75, "completions/mean_terminated_length": 216.75, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.1374549463391304, "epoch": 1.4903006343660936, "frac_reward_zero_std": 0.4, "grad_norm": 0.11165888607501984, "kl": 0.010854947671759874, "learning_rate": 2.7333864982878043e-06, "loss": 0.0004, "num_tokens": 70709695.0, "reward": -0.575, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2643120646476746, "sampling/importance_sampling_ratio/mean": 1.0005427062511445, "sampling/importance_sampling_ratio/min": 0.7182447493076325, "sampling/sampling_logp_difference/max": 0.3649534434080124, "sampling/sampling_logp_difference/mean": 0.022594679333269595, "step": 16210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 226.3375, "completions/mean_terminated_length": 226.3375, "completions/min_length": 158.9, "completions/min_terminated_length": 158.9, "entropy": 1.147709919512272, "epoch": 1.4912200055162268, "frac_reward_zero_std": 0.6, "grad_norm": 0.09870725870132446, "kl": 0.01149239763035439, "learning_rate": 2.7308638192040343e-06, "loss": 0.0007, "num_tokens": 70755770.0, "reward": -0.375, "reward_std": 0.41743398904800416, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.417434024810791, "sampling/importance_sampling_ratio/max": 1.272901749610901, "sampling/importance_sampling_ratio/mean": 1.0001004040241241, "sampling/importance_sampling_ratio/min": 0.7740045487880707, "sampling/sampling_logp_difference/max": 0.26504482626914977, "sampling/sampling_logp_difference/mean": 0.022577319666743277, "step": 16220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.2, "completions/max_terminated_length": 296.2, "completions/mean_length": 210.3375, "completions/mean_terminated_length": 210.3375, "completions/min_length": 153.2, "completions/min_terminated_length": 153.2, "entropy": 1.1104622542858125, "epoch": 1.4921393766663602, "frac_reward_zero_std": 0.4, "grad_norm": 0.12413740158081055, "kl": 0.012519129569409416, "learning_rate": 2.7283409030046155e-06, "loss": 0.0006, "num_tokens": 70798597.0, "reward": 0.025, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.261998748779297, "sampling/importance_sampling_ratio/mean": 1.0000887274742127, "sampling/importance_sampling_ratio/min": 0.7710164248943329, "sampling/sampling_logp_difference/max": 0.2700685739517212, "sampling/sampling_logp_difference/mean": 0.022184784524142742, "step": 16230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.5, "completions/max_terminated_length": 321.5, "completions/mean_length": 228.6125, "completions/mean_terminated_length": 228.6125, "completions/min_length": 151.2, "completions/min_terminated_length": 151.2, "entropy": 1.05801038518548, "epoch": 1.4930587478164936, "frac_reward_zero_std": 0.5, "grad_norm": 0.08380209654569626, "kl": 0.010299819707870483, "learning_rate": 2.725817752280786e-06, "loss": 0.0009, "num_tokens": 70843278.0, "reward": -0.125, "reward_std": 0.3535533845424652, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2460516095161438, "sampling/importance_sampling_ratio/mean": 1.0001856505870819, "sampling/importance_sampling_ratio/min": 0.7603993654251099, "sampling/sampling_logp_difference/max": 0.28024069368839266, "sampling/sampling_logp_difference/mean": 0.020912396349012853, "step": 16240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.0, "completions/max_terminated_length": 330.0, "completions/mean_length": 241.925, "completions/mean_terminated_length": 241.925, "completions/min_length": 168.6, "completions/min_terminated_length": 168.6, "entropy": 1.08641335144639, "epoch": 1.4939781189666268, "frac_reward_zero_std": 0.4, "grad_norm": 5.120948314666748, "kl": 0.009901526238536462, "learning_rate": 2.7232943696240228e-06, "loss": 0.0004, "num_tokens": 70890752.0, "reward": 0.275, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2600421190261841, "sampling/importance_sampling_ratio/mean": 0.9995533108711243, "sampling/importance_sampling_ratio/min": 0.7822311758995056, "sampling/sampling_logp_difference/max": 0.25334627032279966, "sampling/sampling_logp_difference/mean": 0.02133672758936882, "step": 16250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 210.55, "completions/mean_terminated_length": 210.55, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.10544528439641, "epoch": 1.4948974901167602, "frac_reward_zero_std": 0.4, "grad_norm": 5.61191463470459, "kl": 0.011047621991019696, "learning_rate": 2.7207707576260433e-06, "loss": 0.0009, "num_tokens": 70933388.0, "reward": -0.00625, "reward_std": 0.45116754025220873, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.45116754025220873, "sampling/importance_sampling_ratio/max": 1.3222073793411255, "sampling/importance_sampling_ratio/mean": 1.0002821147441865, "sampling/importance_sampling_ratio/min": 0.7715927600860596, "sampling/sampling_logp_difference/max": 0.2986250162124634, "sampling/sampling_logp_difference/mean": 0.021614334546029568, "step": 16260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.7, "completions/max_terminated_length": 286.7, "completions/mean_length": 194.925, "completions/mean_terminated_length": 194.925, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 0.9990811817348003, "epoch": 1.4958168612668934, "frac_reward_zero_std": 0.6, "grad_norm": 6.501304626464844, "kl": 0.012317990884184837, "learning_rate": 2.7182469188787986e-06, "loss": 0.0006, "num_tokens": 70975590.0, "reward": -0.175, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.2397872447967528, "sampling/importance_sampling_ratio/mean": 0.9997337341308594, "sampling/importance_sampling_ratio/min": 0.7470907151699067, "sampling/sampling_logp_difference/max": 0.3455846071243286, "sampling/sampling_logp_difference/mean": 0.020117070339620114, "step": 16270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 292.8, "completions/max_terminated_length": 279.2, "completions/mean_length": 204.3875, "completions/mean_terminated_length": 201.56785888671874, "completions/min_length": 151.3, "completions/min_terminated_length": 151.3, "entropy": 1.0746709942817687, "epoch": 1.4967362324170268, "frac_reward_zero_std": 0.2, "grad_norm": 6.803122520446777, "kl": 0.010420910493121483, "learning_rate": 2.7157228559744725e-06, "loss": 0.0005, "num_tokens": 71017573.0, "reward": -0.08125, "reward_std": 0.6472593829035759, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.6472593948245049, "sampling/importance_sampling_ratio/max": 1.3185902118682862, "sampling/importance_sampling_ratio/mean": 1.0006916403770447, "sampling/importance_sampling_ratio/min": 0.7777548968791962, "sampling/sampling_logp_difference/max": 0.2972171545028687, "sampling/sampling_logp_difference/mean": 0.02104577012360096, "step": 16280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 207.4375, "completions/mean_terminated_length": 207.4375, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.0374103516340256, "epoch": 1.49765560356716, "frac_reward_zero_std": 0.4, "grad_norm": 6.9828572273254395, "kl": 0.012355238484451548, "learning_rate": 2.7131985715054827e-06, "loss": 0.0011, "num_tokens": 71058896.0, "reward": -0.075, "reward_std": 0.6025980114936829, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6025980472564697, "sampling/importance_sampling_ratio/max": 1.2924017906188965, "sampling/importance_sampling_ratio/mean": 1.0001197814941407, "sampling/importance_sampling_ratio/min": 0.7520349323749542, "sampling/sampling_logp_difference/max": 0.292980694770813, "sampling/sampling_logp_difference/mean": 0.02025200705975294, "step": 16290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.6, "completions/max_terminated_length": 251.6, "completions/mean_length": 188.75, "completions/mean_terminated_length": 188.75, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.0308314755558967, "epoch": 1.4985749747172934, "frac_reward_zero_std": 0.4, "grad_norm": 5.518887996673584, "kl": 0.012399762304266915, "learning_rate": 2.7106740680644695e-06, "loss": 0.001, "num_tokens": 71099572.0, "reward": 0.4, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2520723938941956, "sampling/importance_sampling_ratio/mean": 1.0003149867057801, "sampling/importance_sampling_ratio/min": 0.7910041213035583, "sampling/sampling_logp_difference/max": 0.2508274257183075, "sampling/sampling_logp_difference/mean": 0.021070247888565062, "step": 16300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 209.0, "completions/mean_terminated_length": 209.0, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.1334906011819839, "epoch": 1.4994943458674266, "frac_reward_zero_std": 0.5, "grad_norm": 0.10556570440530777, "kl": 0.010608939902158453, "learning_rate": 2.7081493482443006e-06, "loss": 0.0004, "num_tokens": 71144444.0, "reward": 0.275, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.24161856174469, "sampling/importance_sampling_ratio/mean": 1.0004762947559356, "sampling/importance_sampling_ratio/min": 0.754644563794136, "sampling/sampling_logp_difference/max": 0.3072591722011566, "sampling/sampling_logp_difference/mean": 0.022159248031675814, "step": 16310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 207.05, "completions/mean_terminated_length": 207.05, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.06961971744895, "epoch": 1.50041371701756, "frac_reward_zero_std": 0.5, "grad_norm": 8.562644004821777, "kl": 0.010672023223014549, "learning_rate": 2.7056244146380655e-06, "loss": 0.0006, "num_tokens": 71187336.0, "reward": -0.15, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.3462988257408142, "sampling/importance_sampling_ratio/mean": 0.9999511063098907, "sampling/importance_sampling_ratio/min": 0.7777880728244781, "sampling/sampling_logp_difference/max": 0.32256593704223635, "sampling/sampling_logp_difference/mean": 0.02092704325914383, "step": 16320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.2, "completions/max_terminated_length": 286.2, "completions/mean_length": 205.5625, "completions/mean_terminated_length": 205.5625, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.0498481087386609, "epoch": 1.5013330881676934, "frac_reward_zero_std": 0.7, "grad_norm": 5.1214919090271, "kl": 0.010285179445054382, "learning_rate": 2.703099269839074e-06, "loss": 0.0005, "num_tokens": 71231877.0, "reward": -0.25, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.259429943561554, "sampling/importance_sampling_ratio/mean": 0.9999335408210754, "sampling/importance_sampling_ratio/min": 0.78575199842453, "sampling/sampling_logp_difference/max": 0.26236856877803805, "sampling/sampling_logp_difference/mean": 0.020894110202789307, "step": 16330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.1, "completions/max_terminated_length": 333.1, "completions/mean_length": 213.8625, "completions/mean_terminated_length": 213.8625, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.0318271100521088, "epoch": 1.5022524593178266, "frac_reward_zero_std": 0.3, "grad_norm": 6.621706008911133, "kl": 0.01257450266275555, "learning_rate": 2.700573916440852e-06, "loss": 0.0008, "num_tokens": 71277890.0, "reward": 0.25, "reward_std": 0.6480426907539367, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.6480427384376526, "sampling/importance_sampling_ratio/max": 1.3080363273620605, "sampling/importance_sampling_ratio/mean": 0.9998296082019806, "sampling/importance_sampling_ratio/min": 0.7545475244522095, "sampling/sampling_logp_difference/max": 0.34327555894851686, "sampling/sampling_logp_difference/mean": 0.02075657304376364, "step": 16340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 206.2875, "completions/mean_terminated_length": 206.2875, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0043028317391873, "epoch": 1.5031718304679598, "frac_reward_zero_std": 0.6, "grad_norm": 5.912126064300537, "kl": 0.009599857294233516, "learning_rate": 2.698048357037141e-06, "loss": 0.0001, "num_tokens": 71323449.0, "reward": -0.05, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2445905923843383, "sampling/importance_sampling_ratio/mean": 0.999911504983902, "sampling/importance_sampling_ratio/min": 0.772731626033783, "sampling/sampling_logp_difference/max": 0.2687408596277237, "sampling/sampling_logp_difference/mean": 0.020841498859226704, "step": 16350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.7, "completions/max_terminated_length": 332.7, "completions/mean_length": 239.875, "completions/mean_terminated_length": 239.875, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.0473971307277679, "epoch": 1.5040912016180932, "frac_reward_zero_std": 0.7, "grad_norm": 0.10417888313531876, "kl": 0.012361386918928473, "learning_rate": 2.69552259422189e-06, "loss": 0.0005, "num_tokens": 71372359.0, "reward": -0.075, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2617588877677917, "sampling/importance_sampling_ratio/mean": 1.0002805531024932, "sampling/importance_sampling_ratio/min": 0.7754354834556579, "sampling/sampling_logp_difference/max": 0.25879735946655275, "sampling/sampling_logp_difference/mean": 0.020368739031255245, "step": 16360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 352.6, "completions/max_terminated_length": 352.6, "completions/mean_length": 232.3, "completions/mean_terminated_length": 232.3, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 1.083886867761612, "epoch": 1.5050105727682266, "frac_reward_zero_std": 0.6, "grad_norm": 0.1056039109826088, "kl": 0.011143606569385156, "learning_rate": 2.6929966305892623e-06, "loss": 0.0009, "num_tokens": 71417583.0, "reward": 0.05, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.323927628993988, "sampling/importance_sampling_ratio/mean": 1.0004765331745147, "sampling/importance_sampling_ratio/min": 0.76007439494133, "sampling/sampling_logp_difference/max": 0.34064104557037356, "sampling/sampling_logp_difference/mean": 0.020914109610021114, "step": 16370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 201.85, "completions/mean_terminated_length": 201.85, "completions/min_length": 124.7, "completions/min_terminated_length": 124.7, "entropy": 1.000721763446927, "epoch": 1.50592994391836, "frac_reward_zero_std": 0.6, "grad_norm": 0.10513613373041153, "kl": 0.009451196534791961, "learning_rate": 2.690470468733625e-06, "loss": 0.0003, "num_tokens": 71460635.0, "reward": 0.05, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2549748539924621, "sampling/importance_sampling_ratio/mean": 1.0000597357749939, "sampling/importance_sampling_ratio/min": 0.7570525705814362, "sampling/sampling_logp_difference/max": 0.3012875556945801, "sampling/sampling_logp_difference/mean": 0.020395008940249683, "step": 16380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 196.8875, "completions/mean_terminated_length": 196.8875, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 1.016432949155569, "epoch": 1.5068493150684932, "frac_reward_zero_std": 0.6, "grad_norm": 0.10717727988958359, "kl": 0.010007614648202434, "learning_rate": 2.687944111249547e-06, "loss": 0.0009, "num_tokens": 71504138.0, "reward": -0.1, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.2387699246406556, "sampling/importance_sampling_ratio/mean": 1.000164622068405, "sampling/importance_sampling_ratio/min": 0.7833089232444763, "sampling/sampling_logp_difference/max": 0.2525634169578552, "sampling/sampling_logp_difference/mean": 0.019813166372478007, "step": 16390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 202.7625, "completions/mean_terminated_length": 202.7625, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.106041793525219, "epoch": 1.5077686862186264, "frac_reward_zero_std": 0.4, "grad_norm": 6.147555351257324, "kl": 0.010300009266939015, "learning_rate": 2.685417560731802e-06, "loss": 0.0002, "num_tokens": 71546191.0, "reward": -0.125, "reward_std": 0.6025980114936829, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.6025980472564697, "sampling/importance_sampling_ratio/max": 1.2387928724288941, "sampling/importance_sampling_ratio/mean": 0.9997444331645966, "sampling/importance_sampling_ratio/min": 0.7840739190578461, "sampling/sampling_logp_difference/max": 0.255219554901123, "sampling/sampling_logp_difference/mean": 0.021720259636640548, "step": 16400 }, { "epoch": 1.5077686862186264, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 289.55148342059334, "eval_completions/max_terminated_length": 288.3490401396161, "eval_completions/mean_length": 204.40815881326353, "eval_completions/mean_terminated_length": 204.16121307426306, "eval_completions/min_length": 133.1675392670157, "eval_completions/min_terminated_length": 133.1675392670157, "eval_entropy": 1.0727073217873797, "eval_frac_reward_zero_std": 0.4973821989528796, "eval_kl": 0.010648830610946678, "eval_loss": 0.000504202616866678, "eval_num_tokens": 71546191.0, "eval_reward": -0.10514834205933682, "eval_reward_std": 0.4373027646416769, "eval_rewards/reward_fn/mean": -0.10514834205933682, "eval_rewards/reward_fn/std": 0.4373027791007443, "eval_runtime": 1691.6566, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2768120247656138, "eval_sampling/importance_sampling_ratio/mean": 1.0000258484435955, "eval_sampling/importance_sampling_ratio/min": 0.7759121664739612, "eval_sampling/sampling_logp_difference/max": 0.2865212427591986, "eval_sampling/sampling_logp_difference/mean": 0.0212375770532648, "eval_steps_per_second": 0.043, "step": 16400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.5, "completions/max_terminated_length": 249.5, "completions/mean_length": 186.05, "completions/mean_terminated_length": 186.05, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 0.9678924843668938, "epoch": 1.5086880573687598, "frac_reward_zero_std": 0.7, "grad_norm": 0.12630614638328552, "kl": 0.010582885734038427, "learning_rate": 2.6828908197753596e-06, "loss": 0.0007, "num_tokens": 71586779.0, "reward": 0.3, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.253994071483612, "sampling/importance_sampling_ratio/mean": 1.0004788756370544, "sampling/importance_sampling_ratio/min": 0.7946307837963105, "sampling/sampling_logp_difference/max": 0.24983508586883546, "sampling/sampling_logp_difference/mean": 0.02024951819330454, "step": 16410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 192.4, "completions/mean_terminated_length": 192.4, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.1273572079837322, "epoch": 1.5096074285188932, "frac_reward_zero_std": 0.7, "grad_norm": 0.09439144283533096, "kl": 0.010054976763785816, "learning_rate": 2.6803638909753837e-06, "loss": 0.0005, "num_tokens": 71627955.0, "reward": -0.15, "reward_std": 0.2777460336685181, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.2777460336685181, "sampling/importance_sampling_ratio/max": 1.2584311723709107, "sampling/importance_sampling_ratio/mean": 1.0000857055187224, "sampling/importance_sampling_ratio/min": 0.7768506348133087, "sampling/sampling_logp_difference/max": 0.26371135711669924, "sampling/sampling_logp_difference/mean": 0.022436068020761014, "step": 16420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.6, "completions/max_terminated_length": 304.5, "completions/mean_length": 212.9, "completions/mean_terminated_length": 209.44107208251953, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.1400655761361123, "epoch": 1.5105267996690264, "frac_reward_zero_std": 0.6, "grad_norm": 0.09022532403469086, "kl": 0.009889010252663866, "learning_rate": 2.6778367769272347e-06, "loss": 0.0007, "num_tokens": 71669939.0, "reward": -0.45625, "reward_std": 0.40186617374420164, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.4018661856651306, "sampling/importance_sampling_ratio/max": 1.3698946714401246, "sampling/importance_sampling_ratio/mean": 1.0005903482437133, "sampling/importance_sampling_ratio/min": 0.7819397032260895, "sampling/sampling_logp_difference/max": 0.3224765419960022, "sampling/sampling_logp_difference/mean": 0.022019593976438046, "step": 16430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.9, "completions/max_terminated_length": 282.5, "completions/mean_length": 212.075, "completions/mean_terminated_length": 209.53392944335937, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0470775742083789, "epoch": 1.5114461708191596, "frac_reward_zero_std": 0.5, "grad_norm": 0.21747741103172302, "kl": 0.011803712858818472, "learning_rate": 2.67530948022646e-06, "loss": 0.0005, "num_tokens": 71712457.0, "reward": -0.23125, "reward_std": 0.401882404088974, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.4018824100494385, "sampling/importance_sampling_ratio/max": 1.2642396450042725, "sampling/importance_sampling_ratio/mean": 0.9998020529747009, "sampling/importance_sampling_ratio/min": 0.788675993680954, "sampling/sampling_logp_difference/max": 0.25320656299591066, "sampling/sampling_logp_difference/mean": 0.021706141903996466, "step": 16440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.4, "completions/max_terminated_length": 309.2, "completions/mean_length": 210.0375, "completions/mean_terminated_length": 207.74464416503906, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.106161568313837, "epoch": 1.512365541969293, "frac_reward_zero_std": 0.7, "grad_norm": 0.1832747906446457, "kl": 0.010882898862473667, "learning_rate": 2.6727820034687964e-06, "loss": 0.0006, "num_tokens": 71754644.0, "reward": -0.10625, "reward_std": 0.27328457236289977, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.27328457236289977, "sampling/importance_sampling_ratio/max": 1.2904216408729554, "sampling/importance_sampling_ratio/mean": 1.0003708600997925, "sampling/importance_sampling_ratio/min": 0.785501766204834, "sampling/sampling_logp_difference/max": 0.2867657899856567, "sampling/sampling_logp_difference/mean": 0.022099316120147705, "step": 16450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.5, "completions/max_terminated_length": 246.5, "completions/mean_length": 189.325, "completions/mean_terminated_length": 189.325, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0017889987677335, "epoch": 1.5132849131194264, "frac_reward_zero_std": 0.6, "grad_norm": 0.19013842940330505, "kl": 0.009771828821976669, "learning_rate": 2.6702543492501652e-06, "loss": 0.0008, "num_tokens": 71796086.0, "reward": 0.35, "reward_std": 0.3552303433418274, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.3552303433418274, "sampling/importance_sampling_ratio/max": 1.309941577911377, "sampling/importance_sampling_ratio/mean": 1.0005174934864045, "sampling/importance_sampling_ratio/min": 0.7783546686172486, "sampling/sampling_logp_difference/max": 0.3053327023983002, "sampling/sampling_logp_difference/mean": 0.020562821347266436, "step": 16460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.0, "completions/max_terminated_length": 285.5, "completions/mean_length": 212.6625, "completions/mean_terminated_length": 208.92857208251954, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.1134965315461158, "epoch": 1.5142042842695598, "frac_reward_zero_std": 0.3, "grad_norm": 5.79940938949585, "kl": 0.01172060436801985, "learning_rate": 2.667726520166669e-06, "loss": 0.0005, "num_tokens": 71844555.0, "reward": 0.04375, "reward_std": 0.5998699486255645, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.5998699486255645, "sampling/importance_sampling_ratio/max": 1.2910872220993042, "sampling/importance_sampling_ratio/mean": 0.9997046828269959, "sampling/importance_sampling_ratio/min": 0.7675524830818177, "sampling/sampling_logp_difference/max": 0.29042806625366213, "sampling/sampling_logp_difference/mean": 0.021785712614655495, "step": 16470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.3, "completions/max_terminated_length": 299.3, "completions/mean_length": 203.6875, "completions/mean_terminated_length": 203.6875, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0199626885354518, "epoch": 1.515123655419693, "frac_reward_zero_std": 0.4, "grad_norm": 5.12945032119751, "kl": 0.010381739411968738, "learning_rate": 2.665198518814593e-06, "loss": 0.0006, "num_tokens": 71885642.0, "reward": -0.3, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.3403050780296326, "sampling/importance_sampling_ratio/mean": 1.000146347284317, "sampling/importance_sampling_ratio/min": 0.7664498627185822, "sampling/sampling_logp_difference/max": 0.3296523213386536, "sampling/sampling_logp_difference/mean": 0.020120667386800052, "step": 16480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.4, "completions/max_terminated_length": 293.4, "completions/mean_length": 202.425, "completions/mean_terminated_length": 202.425, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.1292654555290937, "epoch": 1.5160430265698261, "frac_reward_zero_std": 0.5, "grad_norm": 7.332207679748535, "kl": 0.010231299774022772, "learning_rate": 2.6626703477903953e-06, "loss": 0.0004, "num_tokens": 71927692.0, "reward": -0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.3018643736839295, "sampling/importance_sampling_ratio/mean": 0.9997356832027435, "sampling/importance_sampling_ratio/min": 0.7793424963951111, "sampling/sampling_logp_difference/max": 0.29351269006729125, "sampling/sampling_logp_difference/mean": 0.022046674601733686, "step": 16490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.6, "completions/max_terminated_length": 267.7, "completions/mean_length": 188.25, "completions/mean_terminated_length": 184.01071472167968, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.004975213855505, "epoch": 1.5169623977199596, "frac_reward_zero_std": 0.5, "grad_norm": 6.307632923126221, "kl": 0.010005134300445206, "learning_rate": 2.66014200969071e-06, "loss": 0.0006, "num_tokens": 71965712.0, "reward": -0.30625, "reward_std": 0.47167748808860777, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.47167750597000124, "sampling/importance_sampling_ratio/max": 1.2441654562950135, "sampling/importance_sampling_ratio/mean": 1.000243252515793, "sampling/importance_sampling_ratio/min": 0.7956024765968323, "sampling/sampling_logp_difference/max": 0.23618719577789307, "sampling/sampling_logp_difference/mean": 0.02013558279722929, "step": 16500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 211.8375, "completions/mean_terminated_length": 211.8375, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0910389587283134, "epoch": 1.517881768870093, "frac_reward_zero_std": 0.6, "grad_norm": 0.10378910601139069, "kl": 0.010462652373826131, "learning_rate": 2.6576135071123447e-06, "loss": 0.0004, "num_tokens": 72008795.0, "reward": -0.4, "reward_std": 0.420828652381897, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.42082867622375486, "sampling/importance_sampling_ratio/max": 1.24943687915802, "sampling/importance_sampling_ratio/mean": 0.9997507631778717, "sampling/importance_sampling_ratio/min": 0.764584994316101, "sampling/sampling_logp_difference/max": 0.2734951972961426, "sampling/sampling_logp_difference/mean": 0.021075851656496523, "step": 16510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.2, "completions/max_terminated_length": 265.2, "completions/mean_length": 188.2375, "completions/mean_terminated_length": 188.2375, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.004058811068535, "epoch": 1.5188011400202261, "frac_reward_zero_std": 0.5, "grad_norm": 0.08776632696390152, "kl": 0.010838454368058591, "learning_rate": 2.6550848426522735e-06, "loss": 0.0005, "num_tokens": 72046190.0, "reward": 0.1, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.294631552696228, "sampling/importance_sampling_ratio/mean": 1.0001485347747803, "sampling/importance_sampling_ratio/min": 0.7893197774887085, "sampling/sampling_logp_difference/max": 0.2818747401237488, "sampling/sampling_logp_difference/mean": 0.020505945198237896, "step": 16520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 270.0, "completions/max_terminated_length": 258.5, "completions/mean_length": 198.2, "completions/mean_terminated_length": 195.10714416503907, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.050988058000803, "epoch": 1.5197205111703593, "frac_reward_zero_std": 0.4, "grad_norm": 0.09623115509748459, "kl": 0.011166518431855366, "learning_rate": 2.6525560189076366e-06, "loss": 0.0008, "num_tokens": 72087718.0, "reward": -0.10625, "reward_std": 0.4947557091712952, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.4947557091712952, "sampling/importance_sampling_ratio/max": 1.3179702043533326, "sampling/importance_sampling_ratio/mean": 1.000511062145233, "sampling/importance_sampling_ratio/min": 0.7758313715457916, "sampling/sampling_logp_difference/max": 0.30929749011993407, "sampling/sampling_logp_difference/mean": 0.02144247479736805, "step": 16530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 206.2375, "completions/mean_terminated_length": 206.2375, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.145120295137167, "epoch": 1.5206398823204927, "frac_reward_zero_std": 0.6, "grad_norm": 0.0897015631198883, "kl": 0.010532986308680848, "learning_rate": 2.65002703847574e-06, "loss": 0.0005, "num_tokens": 72132001.0, "reward": -0.05, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2615511417388916, "sampling/importance_sampling_ratio/mean": 0.9997897386550904, "sampling/importance_sampling_ratio/min": 0.8004835724830628, "sampling/sampling_logp_difference/max": 0.2529992640018463, "sampling/sampling_logp_difference/mean": 0.022175714559853076, "step": 16540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.9, "completions/max_terminated_length": 314.5, "completions/mean_length": 215.0375, "completions/mean_terminated_length": 211.17857208251954, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.0609931737184524, "epoch": 1.5215592534706261, "frac_reward_zero_std": 0.2, "grad_norm": 5.761446475982666, "kl": 0.011340843403013423, "learning_rate": 2.647497903954048e-06, "loss": 0.0004, "num_tokens": 72175356.0, "reward": -0.1375, "reward_std": 0.6747831761837005, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.6747831881046296, "sampling/importance_sampling_ratio/max": 1.276304507255554, "sampling/importance_sampling_ratio/mean": 0.9998039901256561, "sampling/importance_sampling_ratio/min": 0.7759999334812164, "sampling/sampling_logp_difference/max": 0.2913807541131973, "sampling/sampling_logp_difference/mean": 0.021014085039496422, "step": 16550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.2, "completions/max_terminated_length": 303.2, "completions/mean_length": 218.2625, "completions/mean_terminated_length": 218.2625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1114592991769314, "epoch": 1.5224786246207596, "frac_reward_zero_std": 0.3, "grad_norm": 5.202239513397217, "kl": 0.012610984413186088, "learning_rate": 2.6449686179401863e-06, "loss": 0.0008, "num_tokens": 72217753.0, "reward": -0.15, "reward_std": 0.6043314099311828, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6043314099311828, "sampling/importance_sampling_ratio/max": 1.2749709606170654, "sampling/importance_sampling_ratio/mean": 1.000207793712616, "sampling/importance_sampling_ratio/min": 0.7681046307086945, "sampling/sampling_logp_difference/max": 0.29295650124549866, "sampling/sampling_logp_difference/mean": 0.02174436952918768, "step": 16560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.8, "completions/max_terminated_length": 258.8, "completions/mean_length": 185.625, "completions/mean_terminated_length": 185.625, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.0739954866468906, "epoch": 1.5233979957708927, "frac_reward_zero_std": 0.7, "grad_norm": 0.11334652453660965, "kl": 0.01137023233459331, "learning_rate": 2.642439183031933e-06, "loss": 0.0006, "num_tokens": 72257699.0, "reward": -0.25, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.3148919463157653, "sampling/importance_sampling_ratio/mean": 0.999934297800064, "sampling/importance_sampling_ratio/min": 0.7981087625026703, "sampling/sampling_logp_difference/max": 0.28309392333030703, "sampling/sampling_logp_difference/mean": 0.021293073333799838, "step": 16570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.7, "completions/max_terminated_length": 304.7, "completions/mean_length": 218.875, "completions/mean_terminated_length": 218.875, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.085449993610382, "epoch": 1.524317366921026, "frac_reward_zero_std": 0.3, "grad_norm": 0.11521016061306, "kl": 0.010403705120552331, "learning_rate": 2.6399096018272203e-06, "loss": 0.0007, "num_tokens": 72306625.0, "reward": -0.075, "reward_std": 0.6001615285873413, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6001615405082703, "sampling/importance_sampling_ratio/max": 1.258434522151947, "sampling/importance_sampling_ratio/mean": 1.0002072691917419, "sampling/importance_sampling_ratio/min": 0.746683531999588, "sampling/sampling_logp_difference/max": 0.3135216236114502, "sampling/sampling_logp_difference/mean": 0.02138123083859682, "step": 16580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.2, "completions/max_terminated_length": 269.2, "completions/mean_length": 183.7125, "completions/mean_terminated_length": 183.7125, "completions/min_length": 112.0, "completions/min_terminated_length": 112.0, "entropy": 1.0277709074318409, "epoch": 1.5252367380711593, "frac_reward_zero_std": 0.5, "grad_norm": 0.14578185975551605, "kl": 0.010957166465232149, "learning_rate": 2.6373798769241314e-06, "loss": 0.0006, "num_tokens": 72345410.0, "reward": -0.275, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2521155953407288, "sampling/importance_sampling_ratio/mean": 1.0003145813941956, "sampling/importance_sampling_ratio/min": 0.7654203712940216, "sampling/sampling_logp_difference/max": 0.27801004648208616, "sampling/sampling_logp_difference/mean": 0.021126548759639263, "step": 16590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.1, "completions/max_terminated_length": 325.1, "completions/mean_length": 208.4, "completions/mean_terminated_length": 208.4, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.1132900588214398, "epoch": 1.5261561092212927, "frac_reward_zero_std": 0.4, "grad_norm": 7.807037353515625, "kl": 0.01229794617393054, "learning_rate": 2.6348500109208973e-06, "loss": 0.0007, "num_tokens": 72390482.0, "reward": 0.025, "reward_std": 0.5656446695327759, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5656446814537048, "sampling/importance_sampling_ratio/max": 1.291003704071045, "sampling/importance_sampling_ratio/mean": 1.0006662607192993, "sampling/importance_sampling_ratio/min": 0.7200029969215394, "sampling/sampling_logp_difference/max": 0.3835058033466339, "sampling/sampling_logp_difference/mean": 0.021287894062697887, "step": 16600 }, { "epoch": 1.5261561092212927, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 288.825479930192, "eval_completions/max_terminated_length": 287.9319371727749, "eval_completions/mean_length": 203.7107329842932, "eval_completions/mean_terminated_length": 203.4857268008886, "eval_completions/min_length": 131.07678883071554, "eval_completions/min_terminated_length": 131.07678883071554, "eval_entropy": 1.0723274339244004, "eval_frac_reward_zero_std": 0.5165794066317626, "eval_kl": 0.01047332834185834, "eval_loss": 0.0005794085445813835, "eval_num_tokens": 72390482.0, "eval_reward": -0.09576788830715532, "eval_reward_std": 0.43701467266465566, "eval_rewards/reward_fn/mean": -0.09576788830715532, "eval_rewards/reward_fn/std": 0.4370146889961203, "eval_runtime": 1689.7526, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2693237349208946, "eval_sampling/importance_sampling_ratio/mean": 1.0000454592039032, "eval_sampling/importance_sampling_ratio/min": 0.7765635570306428, "eval_sampling/sampling_logp_difference/max": 0.27876646996168564, "eval_sampling/sampling_logp_difference/mean": 0.021007081162164973, "eval_steps_per_second": 0.043, "step": 16600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 223.525, "completions/mean_terminated_length": 223.525, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.040897612273693, "epoch": 1.527075480371426, "frac_reward_zero_std": 0.5, "grad_norm": 6.156789779663086, "kl": 0.010233050829265266, "learning_rate": 2.6323200064158916e-06, "loss": 0.0003, "num_tokens": 72435132.0, "reward": -0.525, "reward_std": 0.3535533845424652, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.3535533845424652, "sampling/importance_sampling_ratio/max": 1.2613816022872926, "sampling/importance_sampling_ratio/mean": 0.9997498452663421, "sampling/importance_sampling_ratio/min": 0.7681142389774323, "sampling/sampling_logp_difference/max": 0.29162733554840087, "sampling/sampling_logp_difference/mean": 0.020607213862240313, "step": 16610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.1, "completions/max_terminated_length": 330.1, "completions/mean_length": 219.925, "completions/mean_terminated_length": 219.925, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 1.0355173751711846, "epoch": 1.5279948515215591, "frac_reward_zero_std": 0.7, "grad_norm": 0.10660817474126816, "kl": 0.009633790794759988, "learning_rate": 2.629789866007633e-06, "loss": 0.0005, "num_tokens": 72480318.0, "reward": 0.025, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2432792782783508, "sampling/importance_sampling_ratio/mean": 0.9997248530387879, "sampling/importance_sampling_ratio/min": 0.7809884965419769, "sampling/sampling_logp_difference/max": 0.26561373472213745, "sampling/sampling_logp_difference/mean": 0.020524269528687, "step": 16620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.9, "completions/max_terminated_length": 308.9, "completions/mean_length": 213.35, "completions/mean_terminated_length": 213.35, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.0772842593491077, "epoch": 1.5289142226716925, "frac_reward_zero_std": 0.7, "grad_norm": 0.12894417345523834, "kl": 0.011281842517200857, "learning_rate": 2.627259592294778e-06, "loss": 0.0005, "num_tokens": 72525074.0, "reward": -0.25, "reward_std": 0.2777460336685181, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.2777460336685181, "sampling/importance_sampling_ratio/max": 1.2460055828094483, "sampling/importance_sampling_ratio/mean": 1.0000024616718293, "sampling/importance_sampling_ratio/min": 0.7396996319293976, "sampling/sampling_logp_difference/max": 0.31281079053878785, "sampling/sampling_logp_difference/mean": 0.02194384951144457, "step": 16630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 193.325, "completions/mean_terminated_length": 193.325, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0921062059700488, "epoch": 1.529833593821826, "frac_reward_zero_std": 0.6, "grad_norm": 0.08804421871900558, "kl": 0.010586033464642241, "learning_rate": 2.624729187876121e-06, "loss": 0.0007, "num_tokens": 72569516.0, "reward": -0.425, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.3175776958465577, "sampling/importance_sampling_ratio/mean": 1.0004490196704865, "sampling/importance_sampling_ratio/min": 0.7767576575279236, "sampling/sampling_logp_difference/max": 0.3325608134269714, "sampling/sampling_logp_difference/mean": 0.021282119490206242, "step": 16640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 343.5, "completions/max_terminated_length": 324.6, "completions/mean_length": 231.525, "completions/mean_terminated_length": 228.30357208251954, "completions/min_length": 148.7, "completions/min_terminated_length": 148.7, "entropy": 1.1028221189975738, "epoch": 1.5307529649719593, "frac_reward_zero_std": 0.1, "grad_norm": 0.0949563980102539, "kl": 0.009613523114239797, "learning_rate": 2.622198655350589e-06, "loss": 0.0001, "num_tokens": 72618046.0, "reward": 0.16875, "reward_std": 0.6960987254977227, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.6960987374186516, "sampling/importance_sampling_ratio/max": 1.252353346347809, "sampling/importance_sampling_ratio/mean": 0.9999673128128052, "sampling/importance_sampling_ratio/min": 0.7363775849342347, "sampling/sampling_logp_difference/max": 0.3176700949668884, "sampling/sampling_logp_difference/mean": 0.02117004692554474, "step": 16650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.0, "completions/max_terminated_length": 280.0, "completions/mean_length": 203.7625, "completions/mean_terminated_length": 203.7625, "completions/min_length": 121.3, "completions/min_terminated_length": 121.3, "entropy": 0.986411689594388, "epoch": 1.5316723361220925, "frac_reward_zero_std": 0.6, "grad_norm": 0.07864877581596375, "kl": 0.012264217043411917, "learning_rate": 2.619667997317243e-06, "loss": 0.0002, "num_tokens": 72660419.0, "reward": -0.3, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.258502984046936, "sampling/importance_sampling_ratio/mean": 1.0002430975437164, "sampling/importance_sampling_ratio/min": 0.7610738158226014, "sampling/sampling_logp_difference/max": 0.2884418487548828, "sampling/sampling_logp_difference/mean": 0.02024955376982689, "step": 16660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 198.6875, "completions/mean_terminated_length": 198.6875, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0966752290725708, "epoch": 1.5325917072722257, "frac_reward_zero_std": 0.3, "grad_norm": 0.09304729849100113, "kl": 0.011054784513544292, "learning_rate": 2.6171372163752714e-06, "loss": 0.001, "num_tokens": 72703650.0, "reward": 0.45, "reward_std": 0.6808418452739715, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.6808419048786163, "sampling/importance_sampling_ratio/max": 1.2604209780693054, "sampling/importance_sampling_ratio/mean": 1.0000013709068298, "sampling/importance_sampling_ratio/min": 0.7803591370582581, "sampling/sampling_logp_difference/max": 0.28128299713134763, "sampling/sampling_logp_difference/mean": 0.02163936644792557, "step": 16670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.3, "completions/max_terminated_length": 317.3, "completions/mean_length": 229.0, "completions/mean_terminated_length": 229.0, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.1306369431316852, "epoch": 1.5335110784223591, "frac_reward_zero_std": 0.4, "grad_norm": 6.041866779327393, "kl": 0.010492147944751196, "learning_rate": 2.6146063151239887e-06, "loss": 0.0004, "num_tokens": 72752122.0, "reward": -0.475, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.3779594659805299, "sampling/importance_sampling_ratio/mean": 1.0001673519611358, "sampling/importance_sampling_ratio/min": 0.7824758589267731, "sampling/sampling_logp_difference/max": 0.3393793821334839, "sampling/sampling_logp_difference/mean": 0.022452763095498086, "step": 16680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.8, "completions/max_terminated_length": 273.8, "completions/mean_length": 199.6375, "completions/mean_terminated_length": 199.6375, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.1357528150081635, "epoch": 1.5344304495724925, "frac_reward_zero_std": 0.5, "grad_norm": 4.932580947875977, "kl": 0.010081220045685768, "learning_rate": 2.6120752961628333e-06, "loss": 0.0003, "num_tokens": 72794501.0, "reward": -0.50625, "reward_std": 0.45566692352294924, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.45566692352294924, "sampling/importance_sampling_ratio/max": 1.2830132842063904, "sampling/importance_sampling_ratio/mean": 0.9996098279953003, "sampling/importance_sampling_ratio/min": 0.7789575934410096, "sampling/sampling_logp_difference/max": 0.26364338397979736, "sampling/sampling_logp_difference/mean": 0.022212732397019864, "step": 16690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.9, "completions/max_terminated_length": 314.9, "completions/mean_length": 221.3625, "completions/mean_terminated_length": 221.3625, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.1216576851904392, "epoch": 1.5353498207226257, "frac_reward_zero_std": 0.5, "grad_norm": 0.13184203207492828, "kl": 0.00998596487333998, "learning_rate": 2.609544162091365e-06, "loss": 0.0002, "num_tokens": 72839434.0, "reward": 0.29375, "reward_std": 0.33331953138113024, "rewards/reward_fn/mean": 0.29375, "rewards/reward_fn/std": 0.33331954330205915, "sampling/importance_sampling_ratio/max": 1.2462260603904725, "sampling/importance_sampling_ratio/mean": 0.999914288520813, "sampling/importance_sampling_ratio/min": 0.7873400866985321, "sampling/sampling_logp_difference/max": 0.2497868537902832, "sampling/sampling_logp_difference/mean": 0.02145320288836956, "step": 16700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 317.0, "completions/max_terminated_length": 306.1, "completions/mean_length": 214.3375, "completions/mean_terminated_length": 211.55357360839844, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 0.9955198496580124, "epoch": 1.536269191872759, "frac_reward_zero_std": 0.5, "grad_norm": 0.24395166337490082, "kl": 0.012190107890637592, "learning_rate": 2.6070129155092618e-06, "loss": 0.0007, "num_tokens": 72884557.0, "reward": -0.23125, "reward_std": 0.35519086569547653, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.3551908776164055, "sampling/importance_sampling_ratio/max": 1.291152822971344, "sampling/importance_sampling_ratio/mean": 0.9996948838233948, "sampling/importance_sampling_ratio/min": 0.7732052505016327, "sampling/sampling_logp_difference/max": 0.2864715337753296, "sampling/sampling_logp_difference/mean": 0.020601993799209593, "step": 16710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 214.85, "completions/mean_terminated_length": 214.85, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.0214983135461808, "epoch": 1.5371885630228923, "frac_reward_zero_std": 0.6, "grad_norm": 5.227814674377441, "kl": 0.010870473505929113, "learning_rate": 2.6044815590163146e-06, "loss": 0.0006, "num_tokens": 72930977.0, "reward": -0.075, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2674880504608155, "sampling/importance_sampling_ratio/mean": 1.0002521574497223, "sampling/importance_sampling_ratio/min": 0.7302087619900703, "sampling/sampling_logp_difference/max": 0.39674527645111085, "sampling/sampling_logp_difference/mean": 0.019962528720498085, "step": 16720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 228.5375, "completions/mean_terminated_length": 228.5375, "completions/min_length": 159.9, "completions/min_terminated_length": 159.9, "entropy": 1.0832982137799263, "epoch": 1.5381079341730257, "frac_reward_zero_std": 0.2, "grad_norm": 6.716733455657959, "kl": 0.01136978857102804, "learning_rate": 2.6019500952124326e-06, "loss": 0.0006, "num_tokens": 72976164.0, "reward": -0.13125, "reward_std": 0.7082734048366547, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.7082734107971191, "sampling/importance_sampling_ratio/max": 1.3653596997261048, "sampling/importance_sampling_ratio/mean": 0.9996957421302796, "sampling/importance_sampling_ratio/min": 0.792349511384964, "sampling/sampling_logp_difference/max": 0.3527353286743164, "sampling/sampling_logp_difference/mean": 0.020598231256008147, "step": 16730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.4, "completions/max_terminated_length": 306.4, "completions/mean_length": 211.1125, "completions/mean_terminated_length": 211.1125, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0990032456815242, "epoch": 1.5390273053231591, "frac_reward_zero_std": 0.5, "grad_norm": 0.07590218633413315, "kl": 0.012239386443980038, "learning_rate": 2.59941852669763e-06, "loss": 0.0008, "num_tokens": 73022805.0, "reward": -0.15, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.3169493913650512, "sampling/importance_sampling_ratio/mean": 1.0005953073501588, "sampling/importance_sampling_ratio/min": 0.7975568890571594, "sampling/sampling_logp_difference/max": 0.2905677556991577, "sampling/sampling_logp_difference/mean": 0.02097071725875139, "step": 16740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.4, "completions/max_terminated_length": 299.4, "completions/mean_length": 212.6875, "completions/mean_terminated_length": 212.6875, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.1477789469063282, "epoch": 1.5399466764732923, "frac_reward_zero_std": 0.3, "grad_norm": 0.2864082157611847, "kl": 0.01380392470746301, "learning_rate": 2.596886856072031e-06, "loss": 0.0002, "num_tokens": 73067716.0, "reward": 0.0, "reward_std": 0.6152435362339019, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6152435719966889, "sampling/importance_sampling_ratio/max": 1.3306254267692565, "sampling/importance_sampling_ratio/mean": 1.0000433564186095, "sampling/importance_sampling_ratio/min": 0.6962214186787605, "sampling/sampling_logp_difference/max": 0.46050070524215697, "sampling/sampling_logp_difference/mean": 0.022215556539595126, "step": 16750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.3, "completions/max_terminated_length": 287.3, "completions/mean_length": 204.6625, "completions/mean_terminated_length": 204.6625, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.080811071395874, "epoch": 1.5408660476234255, "frac_reward_zero_std": 0.6, "grad_norm": 0.2937110662460327, "kl": 0.010966183082200587, "learning_rate": 2.594355085935865e-06, "loss": 0.0005, "num_tokens": 73111505.0, "reward": -0.25, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.340281617641449, "sampling/importance_sampling_ratio/mean": 0.9998137176036834, "sampling/importance_sampling_ratio/min": 0.7758258879184723, "sampling/sampling_logp_difference/max": 0.31062068939208987, "sampling/sampling_logp_difference/mean": 0.021167846769094466, "step": 16760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.5, "completions/max_terminated_length": 271.5, "completions/mean_length": 192.55, "completions/mean_terminated_length": 192.55, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.0267972104251384, "epoch": 1.541785418773559, "frac_reward_zero_std": 0.6, "grad_norm": 0.19472673535346985, "kl": 0.010419910258497112, "learning_rate": 2.591823218889461e-06, "loss": 0.0005, "num_tokens": 73153373.0, "reward": 0.25, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2414708375930785, "sampling/importance_sampling_ratio/mean": 0.9994127213954925, "sampling/importance_sampling_ratio/min": 0.7663185656070709, "sampling/sampling_logp_difference/max": 0.2901234030723572, "sampling/sampling_logp_difference/mean": 0.02068828959017992, "step": 16770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 350.8, "completions/max_terminated_length": 350.8, "completions/mean_length": 251.3125, "completions/mean_terminated_length": 251.3125, "completions/min_length": 168.2, "completions/min_terminated_length": 168.2, "entropy": 1.1263020783662796, "epoch": 1.5427047899236923, "frac_reward_zero_std": 0.4, "grad_norm": 5.044294834136963, "kl": 0.010543894686270506, "learning_rate": 2.589291257533251e-06, "loss": 0.0008, "num_tokens": 73203710.0, "reward": -0.6, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2431037425994873, "sampling/importance_sampling_ratio/mean": 1.0001436293125152, "sampling/importance_sampling_ratio/min": 0.7806419372558594, "sampling/sampling_logp_difference/max": 0.2564504146575928, "sampling/sampling_logp_difference/mean": 0.021498542092740536, "step": 16780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.6, "completions/max_terminated_length": 340.6, "completions/mean_length": 238.7625, "completions/mean_terminated_length": 238.7625, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0421414352953433, "epoch": 1.5436241610738255, "frac_reward_zero_std": 0.7, "grad_norm": 4.0005598068237305, "kl": 0.010298020689515398, "learning_rate": 2.5867592044677636e-06, "loss": 0.0006, "num_tokens": 73254243.0, "reward": 0.275, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2871226906776427, "sampling/importance_sampling_ratio/mean": 0.9999960124492645, "sampling/importance_sampling_ratio/min": 0.7734188735485077, "sampling/sampling_logp_difference/max": 0.289318722486496, "sampling/sampling_logp_difference/mean": 0.020335920713841917, "step": 16790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.0, "completions/max_terminated_length": 264.0, "completions/mean_length": 200.4875, "completions/mean_terminated_length": 200.4875, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.080998270958662, "epoch": 1.5445435322239587, "frac_reward_zero_std": 0.2, "grad_norm": 7.79194450378418, "kl": 0.01389294322580099, "learning_rate": 2.5842270622936166e-06, "loss": 0.0009, "num_tokens": 73296682.0, "reward": 0.15, "reward_std": 0.7330915451049804, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.7330915689468384, "sampling/importance_sampling_ratio/max": 1.2628666281700134, "sampling/importance_sampling_ratio/mean": 0.9998667418956757, "sampling/importance_sampling_ratio/min": 0.7534092873334884, "sampling/sampling_logp_difference/max": 0.3382482886314392, "sampling/sampling_logp_difference/mean": 0.021349780447781085, "step": 16800 }, { "epoch": 1.5445435322239587, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 297.9406631762653, "eval_completions/max_terminated_length": 295.4589877835951, "eval_completions/mean_length": 209.4465532286213, "eval_completions/mean_terminated_length": 208.79670502294837, "eval_completions/min_length": 136.26876090750437, "eval_completions/min_terminated_length": 136.26876090750437, "eval_entropy": 1.0804047400934742, "eval_frac_reward_zero_std": 0.4816753926701571, "eval_kl": 0.010315192101058432, "eval_loss": 0.0005092109786346555, "eval_num_tokens": 73296682.0, "eval_reward": -0.11420157068062828, "eval_reward_std": 0.45739017728631615, "eval_rewards/reward_fn/mean": -0.11420157068062828, "eval_rewards/reward_fn/std": 0.45739019216147186, "eval_runtime": 1734.4455, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.266455545891434, "eval_sampling/importance_sampling_ratio/mean": 0.9999467318594768, "eval_sampling/importance_sampling_ratio/min": 0.7745828851147264, "eval_sampling/sampling_logp_difference/max": 0.28390249225065556, "eval_sampling/sampling_logp_difference/mean": 0.020761011750234152, "eval_steps_per_second": 0.042, "step": 16800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 213.3125, "completions/mean_terminated_length": 213.3125, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.0955089613795281, "epoch": 1.545462903374092, "frac_reward_zero_std": 0.5, "grad_norm": 0.10201343148946762, "kl": 0.009480284643359482, "learning_rate": 2.5816948336115256e-06, "loss": 0.0003, "num_tokens": 73337747.0, "reward": -0.075, "reward_std": 0.4587401747703552, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4587401866912842, "sampling/importance_sampling_ratio/max": 1.2801054477691651, "sampling/importance_sampling_ratio/mean": 1.0001685261726379, "sampling/importance_sampling_ratio/min": 0.781002938747406, "sampling/sampling_logp_difference/max": 0.2812681317329407, "sampling/sampling_logp_difference/mean": 0.021778489463031292, "step": 16810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.9, "completions/max_terminated_length": 338.9, "completions/mean_length": 222.4875, "completions/mean_terminated_length": 222.4875, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.0309893421828746, "epoch": 1.5463822745242255, "frac_reward_zero_std": 0.4, "grad_norm": 4.950870037078857, "kl": 0.009897819708567113, "learning_rate": 2.5791625210222928e-06, "loss": 0.0007, "num_tokens": 73382714.0, "reward": 0.2, "reward_std": 0.4823292136192322, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4823292136192322, "sampling/importance_sampling_ratio/max": 1.265967893600464, "sampling/importance_sampling_ratio/mean": 0.9999690055847168, "sampling/importance_sampling_ratio/min": 0.7873335719108582, "sampling/sampling_logp_difference/max": 0.2571202039718628, "sampling/sampling_logp_difference/mean": 0.020730344764888286, "step": 16820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.6, "completions/max_terminated_length": 311.6, "completions/mean_length": 225.1875, "completions/mean_terminated_length": 225.1875, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.0239100120961666, "epoch": 1.5473016456743587, "frac_reward_zero_std": 0.5, "grad_norm": 0.1001964807510376, "kl": 0.00981434978893958, "learning_rate": 2.5766301271268035e-06, "loss": 0.0004, "num_tokens": 73427393.0, "reward": -0.275, "reward_std": 0.5100160002708435, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5100160360336303, "sampling/importance_sampling_ratio/max": 1.3783881187438964, "sampling/importance_sampling_ratio/mean": 0.9997141122817993, "sampling/importance_sampling_ratio/min": 0.7738250076770783, "sampling/sampling_logp_difference/max": 0.3730035305023193, "sampling/sampling_logp_difference/mean": 0.020806416869163513, "step": 16830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 202.25, "completions/mean_terminated_length": 202.25, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.0579759448766708, "epoch": 1.548221016824492, "frac_reward_zero_std": 0.2, "grad_norm": 19.202394485473633, "kl": 0.010886485985247418, "learning_rate": 2.5740976545260327e-06, "loss": 0.0006, "num_tokens": 73467821.0, "reward": -0.18125, "reward_std": 0.6960987269878387, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.6960987448692322, "sampling/importance_sampling_ratio/max": 1.2673996329307555, "sampling/importance_sampling_ratio/mean": 1.0002009987831115, "sampling/importance_sampling_ratio/min": 0.7617019534111023, "sampling/sampling_logp_difference/max": 0.3152890205383301, "sampling/sampling_logp_difference/mean": 0.020651103369891642, "step": 16840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 199.2625, "completions/mean_terminated_length": 199.2625, "completions/min_length": 113.6, "completions/min_terminated_length": 113.6, "entropy": 1.006683249771595, "epoch": 1.5491403879746253, "frac_reward_zero_std": 0.6, "grad_norm": 0.13339491188526154, "kl": 0.010452369292033836, "learning_rate": 2.57156510582103e-06, "loss": 0.0009, "num_tokens": 73511050.0, "reward": -0.3, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2543177485466004, "sampling/importance_sampling_ratio/mean": 1.0000922620296477, "sampling/importance_sampling_ratio/min": 0.7747901082038879, "sampling/sampling_logp_difference/max": 0.26392088532447816, "sampling/sampling_logp_difference/mean": 0.020765797607600688, "step": 16850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 208.425, "completions/mean_terminated_length": 208.425, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.167381077259779, "epoch": 1.5500597591247587, "frac_reward_zero_std": 0.8, "grad_norm": 0.1264001876115799, "kl": 0.010507323866477236, "learning_rate": 2.569032483612927e-06, "loss": 0.0007, "num_tokens": 73555756.0, "reward": -0.275, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.2693105697631837, "sampling/importance_sampling_ratio/mean": 0.999891871213913, "sampling/importance_sampling_ratio/min": 0.7584223449230194, "sampling/sampling_logp_difference/max": 0.2959718704223633, "sampling/sampling_logp_difference/mean": 0.022266220301389694, "step": 16860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.9, "completions/max_terminated_length": 331.9, "completions/mean_length": 224.4, "completions/mean_terminated_length": 224.4, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 1.1929179109632968, "epoch": 1.550979130274892, "frac_reward_zero_std": 0.5, "grad_norm": 5.743251800537109, "kl": 0.011432210850762204, "learning_rate": 2.5664997905029303e-06, "loss": 0.0003, "num_tokens": 73599092.0, "reward": 0.0, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2594004034996034, "sampling/importance_sampling_ratio/mean": 0.9997868120670319, "sampling/importance_sampling_ratio/min": 0.8008754134178162, "sampling/sampling_logp_difference/max": 0.251802921295166, "sampling/sampling_logp_difference/mean": 0.022160600870847702, "step": 16870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.6, "completions/max_terminated_length": 275.6, "completions/mean_length": 204.95, "completions/mean_terminated_length": 204.95, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.0600777387619018, "epoch": 1.5518985014250253, "frac_reward_zero_std": 0.2, "grad_norm": 0.15603625774383545, "kl": 0.011238772032083944, "learning_rate": 2.5639670290923178e-06, "loss": 0.0002, "num_tokens": 73642632.0, "reward": -0.2, "reward_std": 0.6859699070453644, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6859699189662933, "sampling/importance_sampling_ratio/max": 1.256494903564453, "sampling/importance_sampling_ratio/mean": 1.0000605404376983, "sampling/importance_sampling_ratio/min": 0.7823904097080231, "sampling/sampling_logp_difference/max": 0.2712180733680725, "sampling/sampling_logp_difference/mean": 0.020509238913655282, "step": 16880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.4, "completions/max_terminated_length": 266.4, "completions/mean_length": 179.625, "completions/mean_terminated_length": 179.625, "completions/min_length": 116.9, "completions/min_terminated_length": 116.9, "entropy": 0.8679010892286897, "epoch": 1.5528178725751585, "frac_reward_zero_std": 0.5, "grad_norm": 5.445323944091797, "kl": 0.010296419309452177, "learning_rate": 2.561434201982439e-06, "loss": 0.0002, "num_tokens": 73680394.0, "reward": -0.15, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2897787928581237, "sampling/importance_sampling_ratio/mean": 0.9997512221336364, "sampling/importance_sampling_ratio/min": 0.7447587490081787, "sampling/sampling_logp_difference/max": 0.33591421842575075, "sampling/sampling_logp_difference/mean": 0.018406875943765045, "step": 16890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 217.9125, "completions/mean_terminated_length": 217.9125, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.104869993776083, "epoch": 1.5537372437252919, "frac_reward_zero_std": 0.3, "grad_norm": 5.697848320007324, "kl": 0.010072454137844033, "learning_rate": 2.55890131177471e-06, "loss": 0.0006, "num_tokens": 73727571.0, "reward": -0.25, "reward_std": 0.6295660197734833, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6295660555362701, "sampling/importance_sampling_ratio/max": 1.2486378312110902, "sampling/importance_sampling_ratio/mean": 1.000050312280655, "sampling/importance_sampling_ratio/min": 0.722878485918045, "sampling/sampling_logp_difference/max": 0.35061222314834595, "sampling/sampling_logp_difference/mean": 0.021524957753717898, "step": 16900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 204.3375, "completions/mean_terminated_length": 204.3375, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.0395354323089123, "epoch": 1.5546566148754253, "frac_reward_zero_std": 0.5, "grad_norm": 0.13977345824241638, "kl": 0.01243323814123869, "learning_rate": 2.5563683610706117e-06, "loss": 0.0005, "num_tokens": 73770318.0, "reward": -0.275, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2437144637107849, "sampling/importance_sampling_ratio/mean": 1.0000718235969543, "sampling/importance_sampling_ratio/min": 0.7823580801486969, "sampling/sampling_logp_difference/max": 0.2538545370101929, "sampling/sampling_logp_difference/mean": 0.020926418714225292, "step": 16910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 190.1625, "completions/mean_terminated_length": 190.1625, "completions/min_length": 115.1, "completions/min_terminated_length": 115.1, "entropy": 1.013915527611971, "epoch": 1.5555759860255585, "frac_reward_zero_std": 0.7, "grad_norm": 9.215277671813965, "kl": 0.011399173218524083, "learning_rate": 2.5538353524716893e-06, "loss": 0.0007, "num_tokens": 73808411.0, "reward": -0.58125, "reward_std": 0.2604610502719879, "rewards/reward_fn/mean": -0.58125, "rewards/reward_fn/std": 0.2604610562324524, "sampling/importance_sampling_ratio/max": 1.2826431155204774, "sampling/importance_sampling_ratio/mean": 0.9999982714653015, "sampling/importance_sampling_ratio/min": 0.7138243943452836, "sampling/sampling_logp_difference/max": 0.38432042598724364, "sampling/sampling_logp_difference/mean": 0.021710808761417864, "step": 16920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.9, "completions/max_terminated_length": 277.9, "completions/mean_length": 194.6375, "completions/mean_terminated_length": 194.6375, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.025950586795807, "epoch": 1.5564953571756919, "frac_reward_zero_std": 0.5, "grad_norm": 5.158813953399658, "kl": 0.014892077149124817, "learning_rate": 2.551302288579544e-06, "loss": 0.0009, "num_tokens": 73850358.0, "reward": -0.025, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2397889614105224, "sampling/importance_sampling_ratio/mean": 0.9996105670928955, "sampling/importance_sampling_ratio/min": 0.7768073797225952, "sampling/sampling_logp_difference/max": 0.2593496084213257, "sampling/sampling_logp_difference/mean": 0.02070724032819271, "step": 16930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.2, "completions/max_terminated_length": 249.2, "completions/mean_length": 178.9125, "completions/mean_terminated_length": 178.9125, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.1057158261537552, "epoch": 1.557414728325825, "frac_reward_zero_std": 0.4, "grad_norm": 0.10109630972146988, "kl": 0.011253409169148654, "learning_rate": 2.5487691719958356e-06, "loss": 0.001, "num_tokens": 73891423.0, "reward": 0.15, "reward_std": 0.49665169715881347, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.49665169715881347, "sampling/importance_sampling_ratio/max": 1.286923897266388, "sampling/importance_sampling_ratio/mean": 1.000109851360321, "sampling/importance_sampling_ratio/min": 0.7977999567985534, "sampling/sampling_logp_difference/max": 0.27023270130157473, "sampling/sampling_logp_difference/mean": 0.02187460884451866, "step": 16940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 213.375, "completions/mean_terminated_length": 213.375, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0912955395877362, "epoch": 1.5583340994759585, "frac_reward_zero_std": 0.5, "grad_norm": 5.63629674911499, "kl": 0.010761512006865814, "learning_rate": 2.546236005322279e-06, "loss": 0.0004, "num_tokens": 73936725.0, "reward": -0.075, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2525306582450866, "sampling/importance_sampling_ratio/mean": 1.0006527721881866, "sampling/importance_sampling_ratio/min": 0.7625417053699494, "sampling/sampling_logp_difference/max": 0.28010082244873047, "sampling/sampling_logp_difference/mean": 0.020989503152668475, "step": 16950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.9, "completions/max_terminated_length": 306.9, "completions/mean_length": 226.4625, "completions/mean_terminated_length": 226.4625, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.064613050967455, "epoch": 1.5592534706260919, "frac_reward_zero_std": 0.4, "grad_norm": 0.11371380090713501, "kl": 0.011282635544193908, "learning_rate": 2.543702791160638e-06, "loss": 0.0004, "num_tokens": 73982402.0, "reward": -0.475, "reward_std": 0.5479432165622711, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5479432165622711, "sampling/importance_sampling_ratio/max": 1.265514898300171, "sampling/importance_sampling_ratio/mean": 1.0003485143184663, "sampling/importance_sampling_ratio/min": 0.7833836913108826, "sampling/sampling_logp_difference/max": 0.25796573162078856, "sampling/sampling_logp_difference/mean": 0.02064573671668768, "step": 16960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.5, "completions/max_terminated_length": 306.5, "completions/mean_length": 208.85, "completions/mean_terminated_length": 208.85, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.1175811730325222, "epoch": 1.560172841776225, "frac_reward_zero_std": 0.2, "grad_norm": 5.481800079345703, "kl": 0.010335585143184289, "learning_rate": 2.5411695321127273e-06, "loss": 0.0004, "num_tokens": 74024030.0, "reward": -0.1, "reward_std": 0.7617365121841431, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.761736536026001, "sampling/importance_sampling_ratio/max": 1.284582781791687, "sampling/importance_sampling_ratio/mean": 1.0000475645065308, "sampling/importance_sampling_ratio/min": 0.781630688905716, "sampling/sampling_logp_difference/max": 0.279850047826767, "sampling/sampling_logp_difference/mean": 0.0217990480363369, "step": 16970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.8, "completions/max_terminated_length": 240.8, "completions/mean_length": 184.85, "completions/mean_terminated_length": 184.85, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "entropy": 1.0580819867551328, "epoch": 1.5610922129263582, "frac_reward_zero_std": 0.8, "grad_norm": 7.466050624847412, "kl": 0.0115311958885286, "learning_rate": 2.5386362307804075e-06, "loss": 0.0006, "num_tokens": 74064834.0, "reward": -0.075, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2851599335670472, "sampling/importance_sampling_ratio/mean": 1.0009180724620819, "sampling/importance_sampling_ratio/min": 0.7331594407558442, "sampling/sampling_logp_difference/max": 0.33237653970718384, "sampling/sampling_logp_difference/mean": 0.021505187451839446, "step": 16980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 285.3, "completions/max_terminated_length": 261.4, "completions/mean_length": 191.5, "completions/mean_terminated_length": 187.59464416503906, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.09167238175869, "epoch": 1.5620115840764917, "frac_reward_zero_std": 0.5, "grad_norm": 0.10629607737064362, "kl": 0.012277163873659448, "learning_rate": 2.536102889765581e-06, "loss": 0.0006, "num_tokens": 74105786.0, "reward": 0.01875, "reward_std": 0.4534652173519135, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.4534652173519135, "sampling/importance_sampling_ratio/max": 1.2654942154884339, "sampling/importance_sampling_ratio/mean": 0.999615877866745, "sampling/importance_sampling_ratio/min": 0.7819227993488311, "sampling/sampling_logp_difference/max": 0.2765907347202301, "sampling/sampling_logp_difference/mean": 0.0221824137493968, "step": 16990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.5, "completions/max_terminated_length": 267.5, "completions/mean_length": 186.2375, "completions/mean_terminated_length": 186.2375, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0315252989530563, "epoch": 1.562930955226625, "frac_reward_zero_std": 0.5, "grad_norm": 5.086122035980225, "kl": 0.010166024044156074, "learning_rate": 2.533569511670193e-06, "loss": 0.0005, "num_tokens": 74144877.0, "reward": 0.65, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.65, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.3611849308013917, "sampling/importance_sampling_ratio/mean": 1.000670200586319, "sampling/importance_sampling_ratio/min": 0.7660346806049347, "sampling/sampling_logp_difference/max": 0.3319714665412903, "sampling/sampling_logp_difference/mean": 0.02179201375693083, "step": 17000 }, { "epoch": 1.562930955226625, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 289.0994764397906, "eval_completions/max_terminated_length": 287.630017452007, "eval_completions/mean_length": 199.98625654450262, "eval_completions/mean_terminated_length": 199.64633721955784, "eval_completions/min_length": 126.47818499127399, "eval_completions/min_terminated_length": 126.47818499127399, "eval_entropy": 1.0655415262019239, "eval_frac_reward_zero_std": 0.4799301919720768, "eval_kl": 0.01086984091904827, "eval_loss": 0.000578974315430969, "eval_num_tokens": 74144877.0, "eval_reward": -0.11180191972076789, "eval_reward_std": 0.46227460628924777, "eval_rewards/reward_fn/mean": -0.11180191972076789, "eval_rewards/reward_fn/std": 0.46227462033222694, "eval_runtime": 1688.2928, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2706150771435643, "eval_sampling/importance_sampling_ratio/mean": 1.000000986961378, "eval_sampling/importance_sampling_ratio/min": 0.7770083689928887, "eval_sampling/sampling_logp_difference/max": 0.28627716376727375, "eval_sampling/sampling_logp_difference/mean": 0.020757249858795974, "eval_steps_per_second": 0.043, "step": 17000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.5, "completions/max_terminated_length": 294.0, "completions/mean_length": 205.575, "completions/mean_terminated_length": 202.60535888671876, "completions/min_length": 109.4, "completions/min_terminated_length": 109.4, "entropy": 1.0483258709311485, "epoch": 1.5638503263767582, "frac_reward_zero_std": 0.3, "grad_norm": 0.11212623864412308, "kl": 0.012092884886078537, "learning_rate": 2.5310360990962252e-06, "loss": 0.0003, "num_tokens": 74184891.0, "reward": 0.26875, "reward_std": 0.656330692768097, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.6563307046890259, "sampling/importance_sampling_ratio/max": 1.335577094554901, "sampling/importance_sampling_ratio/mean": 1.0003123223781585, "sampling/importance_sampling_ratio/min": 0.767602053284645, "sampling/sampling_logp_difference/max": 0.2983793020248413, "sampling/sampling_logp_difference/mean": 0.02174245212227106, "step": 17010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.3, "completions/max_terminated_length": 285.3, "completions/mean_length": 202.3625, "completions/mean_terminated_length": 202.3625, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.0352749206125735, "epoch": 1.5647696975268917, "frac_reward_zero_std": 0.7, "grad_norm": 0.11493466794490814, "kl": 0.011468250845791773, "learning_rate": 2.5285026546456963e-06, "loss": 0.0006, "num_tokens": 74226312.0, "reward": -0.375, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.261633837223053, "sampling/importance_sampling_ratio/mean": 1.0002434611320496, "sampling/importance_sampling_ratio/min": 0.7663898706436157, "sampling/sampling_logp_difference/max": 0.28896979093551634, "sampling/sampling_logp_difference/mean": 0.020878109149634838, "step": 17020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 241.2, "completions/max_terminated_length": 241.2, "completions/mean_length": 179.6, "completions/mean_terminated_length": 179.6, "completions/min_length": 115.5, "completions/min_terminated_length": 115.5, "entropy": 0.9916694045066834, "epoch": 1.5656890686770248, "frac_reward_zero_std": 0.5, "grad_norm": 0.10974392294883728, "kl": 0.011814694089116529, "learning_rate": 2.525969180920656e-06, "loss": 0.0009, "num_tokens": 74263952.0, "reward": -0.25, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.3019091844558717, "sampling/importance_sampling_ratio/mean": 0.9999330639839172, "sampling/importance_sampling_ratio/min": 0.7616707384586334, "sampling/sampling_logp_difference/max": 0.2999305009841919, "sampling/sampling_logp_difference/mean": 0.020805468410253526, "step": 17030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 195.9875, "completions/mean_terminated_length": 195.9875, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0211693219840527, "epoch": 1.5666084398271583, "frac_reward_zero_std": 0.3, "grad_norm": 0.14824658632278442, "kl": 0.010790407651802525, "learning_rate": 2.5234356805231857e-06, "loss": 0.0008, "num_tokens": 74307199.0, "reward": 0.15, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2375637531280517, "sampling/importance_sampling_ratio/mean": 1.0002159655094147, "sampling/importance_sampling_ratio/min": 0.753184512257576, "sampling/sampling_logp_difference/max": 0.3028333067893982, "sampling/sampling_logp_difference/mean": 0.020655626989901067, "step": 17040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.9, "completions/max_terminated_length": 258.9, "completions/mean_length": 183.375, "completions/mean_terminated_length": 183.375, "completions/min_length": 112.7, "completions/min_terminated_length": 112.7, "entropy": 1.0781574472784996, "epoch": 1.5675278109772917, "frac_reward_zero_std": 0.6, "grad_norm": 4.983147621154785, "kl": 0.012324341083876788, "learning_rate": 2.520902156055392e-06, "loss": 0.0008, "num_tokens": 74347725.0, "reward": -0.15, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.3148167133331299, "sampling/importance_sampling_ratio/mean": 0.9996751964092254, "sampling/importance_sampling_ratio/min": 0.7749910414218902, "sampling/sampling_logp_difference/max": 0.352596640586853, "sampling/sampling_logp_difference/mean": 0.022224815003573893, "step": 17050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 190.325, "completions/mean_terminated_length": 190.325, "completions/min_length": 115.5, "completions/min_terminated_length": 115.5, "entropy": 1.1134859569370748, "epoch": 1.5684471821274248, "frac_reward_zero_std": 0.5, "grad_norm": 6.794419288635254, "kl": 0.012483915500342845, "learning_rate": 2.5183686101194094e-06, "loss": 0.0007, "num_tokens": 74388967.0, "reward": -0.45625, "reward_std": 0.4587006986141205, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.4587007284164429, "sampling/importance_sampling_ratio/max": 1.2619499444961548, "sampling/importance_sampling_ratio/mean": 1.0001834511756897, "sampling/importance_sampling_ratio/min": 0.7687192678451538, "sampling/sampling_logp_difference/max": 0.276708847284317, "sampling/sampling_logp_difference/mean": 0.02297646887600422, "step": 17060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.5, "completions/max_terminated_length": 266.5, "completions/mean_length": 199.1875, "completions/mean_terminated_length": 199.1875, "completions/min_length": 146.2, "completions/min_terminated_length": 146.2, "entropy": 1.1351660683751106, "epoch": 1.569366553277558, "frac_reward_zero_std": 0.6, "grad_norm": 0.10314060002565384, "kl": 0.010739247733727097, "learning_rate": 2.515835045317392e-06, "loss": 0.0005, "num_tokens": 74431166.0, "reward": -0.125, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2530516982078552, "sampling/importance_sampling_ratio/mean": 1.0001908659934997, "sampling/importance_sampling_ratio/min": 0.7556060791015625, "sampling/sampling_logp_difference/max": 0.29103312492370603, "sampling/sampling_logp_difference/mean": 0.022670429944992066, "step": 17070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.5, "completions/max_terminated_length": 257.5, "completions/mean_length": 184.9375, "completions/mean_terminated_length": 184.9375, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.0594869829714297, "epoch": 1.5702859244276914, "frac_reward_zero_std": 0.6, "grad_norm": 0.14500769972801208, "kl": 0.009011720825219527, "learning_rate": 2.513301464251513e-06, "loss": 0.0003, "num_tokens": 74471129.0, "reward": -0.15, "reward_std": 0.2828427076339722, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.2828427076339722, "sampling/importance_sampling_ratio/max": 1.2415046453475953, "sampling/importance_sampling_ratio/mean": 1.0000331401824951, "sampling/importance_sampling_ratio/min": 0.7676136434078217, "sampling/sampling_logp_difference/max": 0.2735949277877808, "sampling/sampling_logp_difference/mean": 0.02139015682041645, "step": 17080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.2, "completions/max_terminated_length": 327.2, "completions/mean_length": 216.1875, "completions/mean_terminated_length": 216.1875, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0260148249566554, "epoch": 1.5712052955778248, "frac_reward_zero_std": 0.3, "grad_norm": 5.019344806671143, "kl": 0.011085342013393529, "learning_rate": 2.510767869523964e-06, "loss": 0.0009, "num_tokens": 74515888.0, "reward": -0.3, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.3169500827789307, "sampling/importance_sampling_ratio/mean": 1.000043135881424, "sampling/importance_sampling_ratio/min": 0.7636088848114013, "sampling/sampling_logp_difference/max": 0.3099943518638611, "sampling/sampling_logp_difference/mean": 0.021391979418694973, "step": 17090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 205.1875, "completions/mean_terminated_length": 205.1875, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.0353844836354256, "epoch": 1.572124666727958, "frac_reward_zero_std": 0.4, "grad_norm": 4.781015872955322, "kl": 0.010746089156600646, "learning_rate": 2.50823426373695e-06, "loss": 0.0007, "num_tokens": 74557703.0, "reward": -0.025, "reward_std": 0.6093873381614685, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6093873500823974, "sampling/importance_sampling_ratio/max": 1.3768822789192199, "sampling/importance_sampling_ratio/mean": 1.0000488340854645, "sampling/importance_sampling_ratio/min": 0.7726404309272766, "sampling/sampling_logp_difference/max": 0.3609391927719116, "sampling/sampling_logp_difference/mean": 0.020626351796090604, "step": 17100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.1, "completions/max_terminated_length": 280.1, "completions/mean_length": 190.375, "completions/mean_terminated_length": 190.375, "completions/min_length": 121.2, "completions/min_terminated_length": 121.2, "entropy": 1.0942388616502285, "epoch": 1.5730440378780914, "frac_reward_zero_std": 0.4, "grad_norm": 6.375375747680664, "kl": 0.01031974065117538, "learning_rate": 2.5057006494926866e-06, "loss": 0.0008, "num_tokens": 74600165.0, "reward": -0.125, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2405920147895813, "sampling/importance_sampling_ratio/mean": 0.9997305631637573, "sampling/importance_sampling_ratio/min": 0.7935682713985444, "sampling/sampling_logp_difference/max": 0.2510651290416718, "sampling/sampling_logp_difference/mean": 0.02177895475178957, "step": 17110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.2, "completions/max_terminated_length": 240.2, "completions/mean_length": 187.5875, "completions/mean_terminated_length": 187.5875, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.03742441162467, "epoch": 1.5739634090282246, "frac_reward_zero_std": 0.7, "grad_norm": 6.157254219055176, "kl": 0.00995024224394001, "learning_rate": 2.5031670293933995e-06, "loss": 0.0004, "num_tokens": 74644748.0, "reward": 0.1, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2511233329772948, "sampling/importance_sampling_ratio/mean": 1.000422292947769, "sampling/importance_sampling_ratio/min": 0.7886392533779144, "sampling/sampling_logp_difference/max": 0.26237191557884215, "sampling/sampling_logp_difference/mean": 0.020971486531198024, "step": 17120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 201.025, "completions/mean_terminated_length": 201.025, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0756278336048126, "epoch": 1.574882780178358, "frac_reward_zero_std": 0.5, "grad_norm": 5.3890557289123535, "kl": 0.010632907442050055, "learning_rate": 2.50063340604132e-06, "loss": 0.0008, "num_tokens": 74688142.0, "reward": -0.1, "reward_std": 0.3897472023963928, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3897472023963928, "sampling/importance_sampling_ratio/max": 1.3292248606681825, "sampling/importance_sampling_ratio/mean": 1.0003465592861176, "sampling/importance_sampling_ratio/min": 0.7721399247646332, "sampling/sampling_logp_difference/max": 0.31645424365997316, "sampling/sampling_logp_difference/mean": 0.021609233878552913, "step": 17130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.8, "completions/max_terminated_length": 282.8, "completions/mean_length": 206.55, "completions/mean_terminated_length": 206.55, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0574284553527833, "epoch": 1.5758021513284914, "frac_reward_zero_std": 0.3, "grad_norm": 6.387618064880371, "kl": 0.00978255842346698, "learning_rate": 2.4980997820386816e-06, "loss": 0.0009, "num_tokens": 74734986.0, "reward": 0.225, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2365646600723266, "sampling/importance_sampling_ratio/mean": 0.9997708439826966, "sampling/importance_sampling_ratio/min": 0.769487726688385, "sampling/sampling_logp_difference/max": 0.2866827964782715, "sampling/sampling_logp_difference/mean": 0.021083101257681848, "step": 17140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.8, "completions/max_terminated_length": 293.8, "completions/mean_length": 220.5375, "completions/mean_terminated_length": 220.5375, "completions/min_length": 155.4, "completions/min_terminated_length": 155.4, "entropy": 1.0804807133972645, "epoch": 1.5767215224786246, "frac_reward_zero_std": 0.6, "grad_norm": 0.23281027376651764, "kl": 0.011340736551210285, "learning_rate": 2.495566159987719e-06, "loss": 0.0007, "num_tokens": 74782037.0, "reward": -0.1, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.250153398513794, "sampling/importance_sampling_ratio/mean": 0.9996920824050903, "sampling/importance_sampling_ratio/min": 0.7796042501926422, "sampling/sampling_logp_difference/max": 0.26532639265060426, "sampling/sampling_logp_difference/mean": 0.020908291172236203, "step": 17150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 217.875, "completions/mean_terminated_length": 217.875, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.0318295910954476, "epoch": 1.5776408936287578, "frac_reward_zero_std": 0.7, "grad_norm": 0.12784257531166077, "kl": 0.010632238344987855, "learning_rate": 2.4930325424906648e-06, "loss": 0.0007, "num_tokens": 74826683.0, "reward": -0.325, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.2950375199317932, "sampling/importance_sampling_ratio/mean": 1.0007639169692992, "sampling/importance_sampling_ratio/min": 0.763396942615509, "sampling/sampling_logp_difference/max": 0.29148423075675967, "sampling/sampling_logp_difference/mean": 0.02073886673897505, "step": 17160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.8, "completions/max_terminated_length": 306.8, "completions/mean_length": 221.3125, "completions/mean_terminated_length": 221.3125, "completions/min_length": 147.9, "completions/min_terminated_length": 147.9, "entropy": 1.0961280331015586, "epoch": 1.5785602647788912, "frac_reward_zero_std": 0.3, "grad_norm": 0.08561582863330841, "kl": 0.009320121153723448, "learning_rate": 2.4904989321497487e-06, "loss": 0.0004, "num_tokens": 74872988.0, "reward": -0.075, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.3412516593933106, "sampling/importance_sampling_ratio/mean": 0.9999609112739563, "sampling/importance_sampling_ratio/min": 0.7958272576332093, "sampling/sampling_logp_difference/max": 0.3050907254219055, "sampling/sampling_logp_difference/mean": 0.021743106842041015, "step": 17170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.2, "completions/max_terminated_length": 288.2, "completions/mean_length": 202.3375, "completions/mean_terminated_length": 202.3375, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 0.9992210246622563, "epoch": 1.5794796359290246, "frac_reward_zero_std": 0.6, "grad_norm": 8.062387466430664, "kl": 0.009782737371278927, "learning_rate": 2.4879653315671913e-06, "loss": 0.0006, "num_tokens": 74914951.0, "reward": -0.3, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.27038516998291, "sampling/importance_sampling_ratio/mean": 1.0005879044532775, "sampling/importance_sampling_ratio/min": 0.7712176203727722, "sampling/sampling_logp_difference/max": 0.2785829544067383, "sampling/sampling_logp_difference/mean": 0.020427744835615158, "step": 17180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.8, "completions/max_terminated_length": 273.8, "completions/mean_length": 204.7875, "completions/mean_terminated_length": 204.7875, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 0.9920888714492321, "epoch": 1.5803990070791578, "frac_reward_zero_std": 0.2, "grad_norm": 6.512203693389893, "kl": 0.011662494973279536, "learning_rate": 2.4854317433452028e-06, "loss": 0.0005, "num_tokens": 74956030.0, "reward": -0.1, "reward_std": 0.7187690615653992, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.718769085407257, "sampling/importance_sampling_ratio/max": 1.315407967567444, "sampling/importance_sampling_ratio/mean": 1.0001496613025664, "sampling/importance_sampling_ratio/min": 0.7894260883331299, "sampling/sampling_logp_difference/max": 0.29440599083900454, "sampling/sampling_logp_difference/mean": 0.02041926085948944, "step": 17190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 325.8, "completions/max_terminated_length": 308.9, "completions/mean_length": 209.8375, "completions/mean_terminated_length": 206.20000152587892, "completions/min_length": 117.1, "completions/min_terminated_length": 117.1, "entropy": 1.122204127162695, "epoch": 1.581318378229291, "frac_reward_zero_std": 0.3, "grad_norm": 6.691413879394531, "kl": 0.011773879494285211, "learning_rate": 2.4828981700859814e-06, "loss": 0.0009, "num_tokens": 75000321.0, "reward": -0.0375, "reward_std": 0.6153841197490693, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.6153841376304626, "sampling/importance_sampling_ratio/max": 1.256502592563629, "sampling/importance_sampling_ratio/mean": 1.0001595437526702, "sampling/importance_sampling_ratio/min": 0.7531147241592407, "sampling/sampling_logp_difference/max": 0.29234901666641233, "sampling/sampling_logp_difference/mean": 0.021532236970961093, "step": 17200 }, { "epoch": 1.581318378229291, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 293.13438045375216, "eval_completions/max_terminated_length": 290.93542757417106, "eval_completions/mean_length": 204.5835514834206, "eval_completions/mean_terminated_length": 204.10654046023703, "eval_completions/min_length": 131.32984293193718, "eval_completions/min_terminated_length": 131.32984293193718, "eval_entropy": 1.0737542171440824, "eval_frac_reward_zero_std": 0.450261780104712, "eval_kl": 0.010338456287043881, "eval_loss": 0.00048494088696315885, "eval_num_tokens": 75000321.0, "eval_reward": -0.09991273996509599, "eval_reward_std": 0.48168507470614413, "eval_rewards/reward_fn/mean": -0.09991273996509599, "eval_rewards/reward_fn/std": 0.48168508521237297, "eval_runtime": 1710.4388, "eval_samples_per_second": 0.335, "eval_sampling/importance_sampling_ratio/max": 1.2846351961814921, "eval_sampling/importance_sampling_ratio/mean": 1.000054737140163, "eval_sampling/importance_sampling_ratio/min": 0.7777149577675482, "eval_sampling/sampling_logp_difference/max": 0.29254526237958806, "eval_sampling/sampling_logp_difference/mean": 0.021124584686472672, "eval_steps_per_second": 0.042, "step": 17200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 202.5625, "completions/mean_terminated_length": 202.5625, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0341089457273482, "epoch": 1.5822377493794244, "frac_reward_zero_std": 0.4, "grad_norm": 5.405783176422119, "kl": 0.009699473407818005, "learning_rate": 2.4803646143917106e-06, "loss": 0.0008, "num_tokens": 75044886.0, "reward": -0.2, "reward_std": 0.5554606795310975, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5554607272148132, "sampling/importance_sampling_ratio/max": 1.2236013531684875, "sampling/importance_sampling_ratio/mean": 1.0000609278678894, "sampling/importance_sampling_ratio/min": 0.7886333286762237, "sampling/sampling_logp_difference/max": 0.2411583185195923, "sampling/sampling_logp_difference/mean": 0.021269513480365276, "step": 17210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.9, "completions/max_terminated_length": 300.5, "completions/mean_length": 225.7625, "completions/mean_terminated_length": 222.04285736083983, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.1474539890885354, "epoch": 1.5831571205295578, "frac_reward_zero_std": 0.2, "grad_norm": 5.098205089569092, "kl": 0.010837884456850588, "learning_rate": 2.477831078864554e-06, "loss": 0.0001, "num_tokens": 75092899.0, "reward": -0.13125, "reward_std": 0.6720638930797577, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.6720639228820801, "sampling/importance_sampling_ratio/max": 1.2548571944236755, "sampling/importance_sampling_ratio/mean": 1.000271987915039, "sampling/importance_sampling_ratio/min": 0.7625889629125595, "sampling/sampling_logp_difference/max": 0.2913512885570526, "sampling/sampling_logp_difference/mean": 0.022076086699962617, "step": 17220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 200.75, "completions/mean_terminated_length": 200.75, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.0900609634816647, "epoch": 1.5840764916796912, "frac_reward_zero_std": 0.4, "grad_norm": 6.334776401519775, "kl": 0.011318316264078022, "learning_rate": 2.4752975661066576e-06, "loss": 0.0004, "num_tokens": 75137199.0, "reward": -0.1, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.3227348566055297, "sampling/importance_sampling_ratio/mean": 0.9999297976493835, "sampling/importance_sampling_ratio/min": 0.7589171409606934, "sampling/sampling_logp_difference/max": 0.3415316104888916, "sampling/sampling_logp_difference/mean": 0.021734486520290374, "step": 17230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 205.55, "completions/mean_terminated_length": 205.55, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.0994403041899203, "epoch": 1.5849958628298244, "frac_reward_zero_std": 0.6, "grad_norm": 5.272764205932617, "kl": 0.014163337578065694, "learning_rate": 2.47276407872014e-06, "loss": 0.0004, "num_tokens": 75186835.0, "reward": 0.0, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2846365094184875, "sampling/importance_sampling_ratio/mean": 1.000557804107666, "sampling/importance_sampling_ratio/min": 0.7074646860361099, "sampling/sampling_logp_difference/max": 0.3920369207859039, "sampling/sampling_logp_difference/mean": 0.021826169453561306, "step": 17240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 185.9, "completions/mean_terminated_length": 185.9, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "entropy": 1.0415544301271438, "epoch": 1.5859152339799576, "frac_reward_zero_std": 0.8, "grad_norm": 0.0833018347620964, "kl": 0.01118270111619495, "learning_rate": 2.4702306193070968e-06, "loss": 0.0005, "num_tokens": 75227595.0, "reward": -0.525, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2675629615783692, "sampling/importance_sampling_ratio/mean": 1.0001290321350098, "sampling/importance_sampling_ratio/min": 0.7389724791049957, "sampling/sampling_logp_difference/max": 0.34483609199523924, "sampling/sampling_logp_difference/mean": 0.02126842625439167, "step": 17250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.2, "completions/max_terminated_length": 319.2, "completions/mean_length": 226.8875, "completions/mean_terminated_length": 226.8875, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 1.0609073959290982, "epoch": 1.586834605130091, "frac_reward_zero_std": 0.4, "grad_norm": 6.431919097900391, "kl": 0.0118887304503005, "learning_rate": 2.467697190469594e-06, "loss": 0.0004, "num_tokens": 75272938.0, "reward": 0.05, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.3107258319854735, "sampling/importance_sampling_ratio/mean": 1.0000627219676972, "sampling/importance_sampling_ratio/min": 0.7751010954380035, "sampling/sampling_logp_difference/max": 0.301312130689621, "sampling/sampling_logp_difference/mean": 0.021479826793074606, "step": 17260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.4, "completions/max_terminated_length": 265.4, "completions/mean_length": 190.925, "completions/mean_terminated_length": 190.925, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 0.997811135649681, "epoch": 1.5877539762802244, "frac_reward_zero_std": 0.5, "grad_norm": 7.245407581329346, "kl": 0.011269352020462975, "learning_rate": 2.465163794809667e-06, "loss": 0.0007, "num_tokens": 75313604.0, "reward": -0.15, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.2716651558876038, "sampling/importance_sampling_ratio/mean": 0.9995772659778595, "sampling/importance_sampling_ratio/min": 0.7362855792045593, "sampling/sampling_logp_difference/max": 0.3249755591154099, "sampling/sampling_logp_difference/mean": 0.02077462188899517, "step": 17270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.0, "completions/max_terminated_length": 266.0, "completions/mean_length": 184.25, "completions/mean_terminated_length": 184.25, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 0.9992605462670326, "epoch": 1.5886733474303576, "frac_reward_zero_std": 0.4, "grad_norm": 0.08959323912858963, "kl": 0.00970555660314858, "learning_rate": 2.462630434929314e-06, "loss": 0.0007, "num_tokens": 75351584.0, "reward": 0.15, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.2376163125038147, "sampling/importance_sampling_ratio/mean": 0.9992085576057435, "sampling/importance_sampling_ratio/min": 0.7809464752674102, "sampling/sampling_logp_difference/max": 0.25897889137268065, "sampling/sampling_logp_difference/mean": 0.0206793162971735, "step": 17280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.4, "completions/max_terminated_length": 297.4, "completions/mean_length": 209.325, "completions/mean_terminated_length": 209.325, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0598471663892268, "epoch": 1.5895927185804908, "frac_reward_zero_std": 0.4, "grad_norm": 6.640732288360596, "kl": 0.00946643311472144, "learning_rate": 2.4600971134305007e-06, "loss": 0.0007, "num_tokens": 75395042.0, "reward": 0.0, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.2601947784423828, "sampling/importance_sampling_ratio/mean": 0.9999654173851014, "sampling/importance_sampling_ratio/min": 0.7576471984386444, "sampling/sampling_logp_difference/max": 0.3094895601272583, "sampling/sampling_logp_difference/mean": 0.021119718626141547, "step": 17290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.4, "completions/max_terminated_length": 276.4, "completions/mean_length": 202.025, "completions/mean_terminated_length": 202.025, "completions/min_length": 117.6, "completions/min_terminated_length": 117.6, "entropy": 1.0749392360448837, "epoch": 1.5905120897306242, "frac_reward_zero_std": 0.7, "grad_norm": 0.09722275286912918, "kl": 0.009266685572220013, "learning_rate": 2.4575638329151493e-06, "loss": 0.0006, "num_tokens": 75435460.0, "reward": -0.4, "reward_std": 0.29206851720809934, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.29206851720809934, "sampling/importance_sampling_ratio/max": 1.289747405052185, "sampling/importance_sampling_ratio/mean": 0.9997811377048492, "sampling/importance_sampling_ratio/min": 0.7867707550525666, "sampling/sampling_logp_difference/max": 0.2767244577407837, "sampling/sampling_logp_difference/mean": 0.021808239072561263, "step": 17300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 222.2875, "completions/mean_terminated_length": 222.2875, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.1447705946862698, "epoch": 1.5914314608807576, "frac_reward_zero_std": 0.4, "grad_norm": 0.12397199869155884, "kl": 0.011891905299853534, "learning_rate": 2.4550305959851445e-06, "loss": 0.0009, "num_tokens": 75478163.0, "reward": -0.35625, "reward_std": 0.5163357496261597, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.5163357555866241, "sampling/importance_sampling_ratio/max": 1.3239615321159364, "sampling/importance_sampling_ratio/mean": 0.9994157016277313, "sampling/importance_sampling_ratio/min": 0.7467662036418915, "sampling/sampling_logp_difference/max": 0.35241461992263795, "sampling/sampling_logp_difference/mean": 0.02299468107521534, "step": 17310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 200.825, "completions/mean_terminated_length": 200.825, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.0861418455839158, "epoch": 1.592350832030891, "frac_reward_zero_std": 0.5, "grad_norm": 0.09819075465202332, "kl": 0.009029847889905795, "learning_rate": 2.4524974052423213e-06, "loss": 0.0007, "num_tokens": 75521197.0, "reward": -0.4, "reward_std": 0.49908818006515504, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4990882039070129, "sampling/importance_sampling_ratio/max": 1.2396297454833984, "sampling/importance_sampling_ratio/mean": 1.000377643108368, "sampling/importance_sampling_ratio/min": 0.76946542263031, "sampling/sampling_logp_difference/max": 0.2663127303123474, "sampling/sampling_logp_difference/mean": 0.021517151035368443, "step": 17320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 208.85, "completions/mean_terminated_length": 208.85, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "entropy": 0.9819321505725384, "epoch": 1.5932702031810242, "frac_reward_zero_std": 0.6, "grad_norm": 5.506608963012695, "kl": 0.011167040269356221, "learning_rate": 2.4499642632884714e-06, "loss": 0.0002, "num_tokens": 75562617.0, "reward": 0.0, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.2514825820922852, "sampling/importance_sampling_ratio/mean": 0.9998144388198853, "sampling/importance_sampling_ratio/min": 0.726829606294632, "sampling/sampling_logp_difference/max": 0.36021665334701536, "sampling/sampling_logp_difference/mean": 0.020084093697369097, "step": 17330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 211.775, "completions/mean_terminated_length": 211.775, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.0815343327820301, "epoch": 1.5941895743311574, "frac_reward_zero_std": 0.5, "grad_norm": 0.0913160890340805, "kl": 0.008956736393156462, "learning_rate": 2.4474311727253334e-06, "loss": 0.0007, "num_tokens": 75608983.0, "reward": -0.325, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2653539776802063, "sampling/importance_sampling_ratio/mean": 0.9995956659317017, "sampling/importance_sampling_ratio/min": 0.7817139327526093, "sampling/sampling_logp_difference/max": 0.2767000675201416, "sampling/sampling_logp_difference/mean": 0.021595696359872817, "step": 17340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.4, "completions/max_terminated_length": 277.4, "completions/mean_length": 191.7625, "completions/mean_terminated_length": 191.7625, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.0303653553128242, "epoch": 1.5951089454812908, "frac_reward_zero_std": 0.6, "grad_norm": 0.0923214852809906, "kl": 0.009483834105776624, "learning_rate": 2.444898136154595e-06, "loss": -0.0, "num_tokens": 75650660.0, "reward": -0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.3743261456489564, "sampling/importance_sampling_ratio/mean": 1.000193613767624, "sampling/importance_sampling_ratio/min": 0.7734797298908234, "sampling/sampling_logp_difference/max": 0.3758107304573059, "sampling/sampling_logp_difference/mean": 0.021504209004342557, "step": 17350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 217.0125, "completions/mean_terminated_length": 217.0125, "completions/min_length": 155.7, "completions/min_terminated_length": 155.7, "entropy": 1.0408941142261028, "epoch": 1.5960283166314242, "frac_reward_zero_std": 0.3, "grad_norm": 4.9037580490112305, "kl": 0.009809373132884502, "learning_rate": 2.442365156177889e-06, "loss": 0.0003, "num_tokens": 75695661.0, "reward": 0.125, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2565796375274658, "sampling/importance_sampling_ratio/mean": 0.9999751150608063, "sampling/importance_sampling_ratio/min": 0.7574158906936646, "sampling/sampling_logp_difference/max": 0.2902714729309082, "sampling/sampling_logp_difference/mean": 0.020529414899647234, "step": 17360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.9, "completions/max_terminated_length": 302.0, "completions/mean_length": 225.5375, "completions/mean_terminated_length": 221.4232147216797, "completions/min_length": 148.8, "completions/min_terminated_length": 148.8, "entropy": 1.0938634052872658, "epoch": 1.5969476877815574, "frac_reward_zero_std": 0.5, "grad_norm": 4.989368438720703, "kl": 0.009586157338344493, "learning_rate": 2.4398322353967867e-06, "loss": 0.0005, "num_tokens": 75743832.0, "reward": -0.18125, "reward_std": 0.4850714147090912, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.4850714385509491, "sampling/importance_sampling_ratio/max": 1.2467227578163147, "sampling/importance_sampling_ratio/mean": 0.9997844576835633, "sampling/importance_sampling_ratio/min": 0.730374950170517, "sampling/sampling_logp_difference/max": 0.32647587060928346, "sampling/sampling_logp_difference/mean": 0.02166050635278225, "step": 17370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 203.4375, "completions/mean_terminated_length": 203.4375, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.0862860403954984, "epoch": 1.5978670589316906, "frac_reward_zero_std": 0.2, "grad_norm": 6.136575222015381, "kl": 0.011186882530455477, "learning_rate": 2.437299376412801e-06, "loss": 0.0009, "num_tokens": 75788483.0, "reward": 0.075, "reward_std": 0.7296968817710876, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.7296969175338746, "sampling/importance_sampling_ratio/max": 1.3217824816703796, "sampling/importance_sampling_ratio/mean": 0.999644011259079, "sampling/importance_sampling_ratio/min": 0.753337150812149, "sampling/sampling_logp_difference/max": 0.3304384112358093, "sampling/sampling_logp_difference/mean": 0.021391783840954304, "step": 17380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.2, "completions/max_terminated_length": 297.2, "completions/mean_length": 207.475, "completions/mean_terminated_length": 207.475, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.1142319291830063, "epoch": 1.598786430081824, "frac_reward_zero_std": 0.3, "grad_norm": 0.1206173449754715, "kl": 0.013423995915218256, "learning_rate": 2.4347665818273824e-06, "loss": 0.0005, "num_tokens": 75828529.0, "reward": 0.225, "reward_std": 0.6480583846569061, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.6480584084987641, "sampling/importance_sampling_ratio/max": 1.2787728309631348, "sampling/importance_sampling_ratio/mean": 1.0005550742149354, "sampling/importance_sampling_ratio/min": 0.7626580536365509, "sampling/sampling_logp_difference/max": 0.29463871717453005, "sampling/sampling_logp_difference/mean": 0.02202967032790184, "step": 17390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.9, "completions/max_terminated_length": 306.9, "completions/mean_length": 213.95, "completions/mean_terminated_length": 213.95, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.083611872047186, "epoch": 1.5997058012319574, "frac_reward_zero_std": 0.7, "grad_norm": 0.09426695853471756, "kl": 0.010437507543247192, "learning_rate": 2.4322338542419134e-06, "loss": 0.0007, "num_tokens": 75872605.0, "reward": 0.2, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.242941963672638, "sampling/importance_sampling_ratio/mean": 1.0001284062862397, "sampling/importance_sampling_ratio/min": 0.7782416582107544, "sampling/sampling_logp_difference/max": 0.25531563758850095, "sampling/sampling_logp_difference/mean": 0.021521290205419062, "step": 17400 }, { "epoch": 1.5997058012319574, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 300.45200698080276, "eval_completions/max_terminated_length": 299.0157068062827, "eval_completions/mean_length": 209.1500872600349, "eval_completions/mean_terminated_length": 208.74463979777659, "eval_completions/min_length": 135.9022687609075, "eval_completions/min_terminated_length": 135.9022687609075, "eval_entropy": 1.097301421885299, "eval_frac_reward_zero_std": 0.4956369982547993, "eval_kl": 0.01060100609313105, "eval_loss": 0.0005139505956321955, "eval_num_tokens": 75872605.0, "eval_reward": -0.12358202443280977, "eval_reward_std": 0.45052538016391674, "eval_rewards/reward_fn/mean": -0.12358202443280977, "eval_rewards/reward_fn/std": 0.4505253971195138, "eval_runtime": 1746.5209, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.2753209768165468, "eval_sampling/importance_sampling_ratio/mean": 0.9999616210581328, "eval_sampling/importance_sampling_ratio/min": 0.7749254246604796, "eval_sampling/sampling_logp_difference/max": 0.2877000222343425, "eval_sampling/sampling_logp_difference/mean": 0.020926993081824007, "eval_steps_per_second": 0.041, "step": 17400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.0, "completions/max_terminated_length": 321.0, "completions/mean_length": 225.9125, "completions/mean_terminated_length": 225.9125, "completions/min_length": 158.7, "completions/min_terminated_length": 158.7, "entropy": 1.091290581971407, "epoch": 1.6006251723820908, "frac_reward_zero_std": 0.7, "grad_norm": 5.458862781524658, "kl": 0.00971702440874651, "learning_rate": 2.4297011962577084e-06, "loss": 0.0005, "num_tokens": 75914158.0, "reward": -0.025, "reward_std": 0.2845196664333344, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2845196664333344, "sampling/importance_sampling_ratio/max": 1.3063085675239563, "sampling/importance_sampling_ratio/mean": 1.0002945601940154, "sampling/importance_sampling_ratio/min": 0.7874757289886475, "sampling/sampling_logp_difference/max": 0.2905935406684875, "sampling/sampling_logp_difference/mean": 0.02134048640727997, "step": 17410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 209.45, "completions/mean_terminated_length": 209.45, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.1071626350283623, "epoch": 1.601544543532224, "frac_reward_zero_std": 0.6, "grad_norm": 0.0902906134724617, "kl": 0.009778759407345206, "learning_rate": 2.4271686104760093e-06, "loss": 0.0005, "num_tokens": 75957186.0, "reward": -0.225, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.366746711730957, "sampling/importance_sampling_ratio/mean": 0.9998187065124512, "sampling/importance_sampling_ratio/min": 0.7687068343162536, "sampling/sampling_logp_difference/max": 0.35963332653045654, "sampling/sampling_logp_difference/mean": 0.021377666853368282, "step": 17420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 198.825, "completions/mean_terminated_length": 198.825, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0597802236676217, "epoch": 1.6024639146823572, "frac_reward_zero_std": 0.7, "grad_norm": 5.125431537628174, "kl": 0.010045392875326797, "learning_rate": 2.424636099497986e-06, "loss": 0.0004, "num_tokens": 75997084.0, "reward": -0.23125, "reward_std": 0.28421239256858827, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.28421240448951723, "sampling/importance_sampling_ratio/max": 1.2486456155776977, "sampling/importance_sampling_ratio/mean": 0.9996542096138, "sampling/importance_sampling_ratio/min": 0.803339147567749, "sampling/sampling_logp_difference/max": 0.25248140692710874, "sampling/sampling_logp_difference/mean": 0.02082744175568223, "step": 17430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.1, "completions/max_terminated_length": 304.1, "completions/mean_length": 211.0875, "completions/mean_terminated_length": 211.0875, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.1086172342300415, "epoch": 1.6033832858324906, "frac_reward_zero_std": 0.6, "grad_norm": 0.082950659096241, "kl": 0.010720175714232028, "learning_rate": 2.422103665924729e-06, "loss": 0.0009, "num_tokens": 76041251.0, "reward": -0.075, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.2857080340385436, "sampling/importance_sampling_ratio/mean": 0.9996699333190918, "sampling/importance_sampling_ratio/min": 0.7769720077514648, "sampling/sampling_logp_difference/max": 0.2838940262794495, "sampling/sampling_logp_difference/mean": 0.02130107693374157, "step": 17440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.2, "completions/max_terminated_length": 246.2, "completions/mean_length": 187.4875, "completions/mean_terminated_length": 187.4875, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.0359888724982738, "epoch": 1.604302656982624, "frac_reward_zero_std": 0.8, "grad_norm": 6.562263488769531, "kl": 0.008832267438992857, "learning_rate": 2.419571312357253e-06, "loss": 0.0003, "num_tokens": 76082594.0, "reward": 0.225, "reward_std": 0.1960918426513672, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.19609185457229614, "sampling/importance_sampling_ratio/max": 1.2512728333473206, "sampling/importance_sampling_ratio/mean": 1.0001989662647248, "sampling/importance_sampling_ratio/min": 0.7921730399131774, "sampling/sampling_logp_difference/max": 0.2453530550003052, "sampling/sampling_logp_difference/mean": 0.02074533924460411, "step": 17450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.5, "completions/max_terminated_length": 267.5, "completions/mean_length": 196.7125, "completions/mean_terminated_length": 196.7125, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.146894884109497, "epoch": 1.6052220281327572, "frac_reward_zero_std": 0.3, "grad_norm": 0.13183735311031342, "kl": 0.011652562167728319, "learning_rate": 2.4170390413964842e-06, "loss": 0.0004, "num_tokens": 76127163.0, "reward": 0.075, "reward_std": 0.6623808681964874, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.6623808920383454, "sampling/importance_sampling_ratio/max": 1.2844727754592895, "sampling/importance_sampling_ratio/mean": 0.9998882949352265, "sampling/importance_sampling_ratio/min": 0.7826194584369659, "sampling/sampling_logp_difference/max": 0.27267405986785886, "sampling/sampling_logp_difference/mean": 0.02258011233061552, "step": 17460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.7, "completions/max_terminated_length": 288.7, "completions/mean_length": 199.0875, "completions/mean_terminated_length": 199.0875, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.1115497343242169, "epoch": 1.6061413992828903, "frac_reward_zero_std": 0.6, "grad_norm": 9.67706298828125, "kl": 0.009786214379710145, "learning_rate": 2.414506855643271e-06, "loss": 0.0005, "num_tokens": 76167266.0, "reward": 0.175, "reward_std": 0.3484567105770111, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3484567105770111, "sampling/importance_sampling_ratio/max": 1.33396315574646, "sampling/importance_sampling_ratio/mean": 0.9997413575649261, "sampling/importance_sampling_ratio/min": 0.7922511637210846, "sampling/sampling_logp_difference/max": 0.30004817247390747, "sampling/sampling_logp_difference/mean": 0.022010089084506034, "step": 17470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.8, "completions/max_terminated_length": 322.8, "completions/mean_length": 229.125, "completions/mean_terminated_length": 229.125, "completions/min_length": 156.2, "completions/min_terminated_length": 156.2, "entropy": 1.0641048341989516, "epoch": 1.6070607704330238, "frac_reward_zero_std": 0.6, "grad_norm": 4.330241680145264, "kl": 0.009667522914242, "learning_rate": 2.41197475769837e-06, "loss": 0.0004, "num_tokens": 76211052.0, "reward": -0.175, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2581575155258178, "sampling/importance_sampling_ratio/mean": 0.9998457431793213, "sampling/importance_sampling_ratio/min": 0.7272122770547866, "sampling/sampling_logp_difference/max": 0.3378265142440796, "sampling/sampling_logp_difference/mean": 0.02070564441382885, "step": 17480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.3, "completions/max_terminated_length": 286.3, "completions/mean_length": 200.225, "completions/mean_terminated_length": 200.225, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0803141757845878, "epoch": 1.6079801415831572, "frac_reward_zero_std": 0.4, "grad_norm": 6.628087997436523, "kl": 0.010339988284977153, "learning_rate": 2.4094427501624495e-06, "loss": 0.0006, "num_tokens": 76252558.0, "reward": -0.025, "reward_std": 0.5185230314731598, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5185230314731598, "sampling/importance_sampling_ratio/max": 1.315491831302643, "sampling/importance_sampling_ratio/mean": 0.9998575568199157, "sampling/importance_sampling_ratio/min": 0.7475334703922272, "sampling/sampling_logp_difference/max": 0.3895518362522125, "sampling/sampling_logp_difference/mean": 0.021655864268541335, "step": 17490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.7, "completions/max_terminated_length": 265.7, "completions/mean_length": 191.225, "completions/mean_terminated_length": 191.225, "completions/min_length": 119.9, "completions/min_terminated_length": 119.9, "entropy": 1.0403435602784157, "epoch": 1.6088995127332906, "frac_reward_zero_std": 0.5, "grad_norm": 0.10986063629388809, "kl": 0.010007054306333885, "learning_rate": 2.406910835636083e-06, "loss": 0.0004, "num_tokens": 76293568.0, "reward": -0.025, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.252482295036316, "sampling/importance_sampling_ratio/mean": 1.0000801384449005, "sampling/importance_sampling_ratio/min": 0.7609431147575378, "sampling/sampling_logp_difference/max": 0.2950954705476761, "sampling/sampling_logp_difference/mean": 0.02133528757840395, "step": 17500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.3, "completions/max_terminated_length": 289.3, "completions/mean_length": 211.1625, "completions/mean_terminated_length": 211.1625, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.0150983288884163, "epoch": 1.6098188838834238, "frac_reward_zero_std": 0.5, "grad_norm": 4.9830756187438965, "kl": 0.009414176869904622, "learning_rate": 2.4043790167197505e-06, "loss": 0.0009, "num_tokens": 76336877.0, "reward": 0.2, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.3714171528816224, "sampling/importance_sampling_ratio/mean": 1.0003409445285798, "sampling/importance_sampling_ratio/min": 0.7416674196720123, "sampling/sampling_logp_difference/max": 0.3369631767272949, "sampling/sampling_logp_difference/mean": 0.020905864611268043, "step": 17510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.3, "completions/max_terminated_length": 286.3, "completions/mean_length": 196.9875, "completions/mean_terminated_length": 196.9875, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0367900609970093, "epoch": 1.610738255033557, "frac_reward_zero_std": 0.5, "grad_norm": 0.10271842032670975, "kl": 0.010408749390626327, "learning_rate": 2.401847296013832e-06, "loss": 0.0006, "num_tokens": 76376916.0, "reward": 0.025, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2466217875480652, "sampling/importance_sampling_ratio/mean": 1.000379753112793, "sampling/importance_sampling_ratio/min": 0.7955690622329712, "sampling/sampling_logp_difference/max": 0.25156459808349607, "sampling/sampling_logp_difference/mean": 0.021298001892864703, "step": 17520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 204.6375, "completions/mean_terminated_length": 204.6375, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.0578311942517757, "epoch": 1.6116576261836904, "frac_reward_zero_std": 0.5, "grad_norm": 0.09836583584547043, "kl": 0.009469786117551848, "learning_rate": 2.3993156761186093e-06, "loss": 0.0003, "num_tokens": 76422151.0, "reward": 0.025, "reward_std": 0.4410387217998505, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4410387217998505, "sampling/importance_sampling_ratio/max": 1.209650194644928, "sampling/importance_sampling_ratio/mean": 0.9998314559459687, "sampling/importance_sampling_ratio/min": 0.7691229164600373, "sampling/sampling_logp_difference/max": 0.27368152141571045, "sampling/sampling_logp_difference/mean": 0.021412193216383456, "step": 17530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.5, "completions/max_terminated_length": 258.5, "completions/mean_length": 193.175, "completions/mean_terminated_length": 193.175, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0615277260541915, "epoch": 1.6125769973338238, "frac_reward_zero_std": 0.7, "grad_norm": 6.978992462158203, "kl": 0.009455813738168218, "learning_rate": 2.3967841596342586e-06, "loss": 0.0007, "num_tokens": 76460557.0, "reward": 0.225, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.222791850566864, "sampling/importance_sampling_ratio/mean": 0.9996574878692627, "sampling/importance_sampling_ratio/min": 0.7336759954690933, "sampling/sampling_logp_difference/max": 0.32779244184494016, "sampling/sampling_logp_difference/mean": 0.021414113231003284, "step": 17540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 211.2375, "completions/mean_terminated_length": 211.2375, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.1627620734274386, "epoch": 1.613496368483957, "frac_reward_zero_std": 0.3, "grad_norm": 10.217707633972168, "kl": 0.010889745753956959, "learning_rate": 2.3942527491608487e-06, "loss": 0.0009, "num_tokens": 76505624.0, "reward": 0.375, "reward_std": 0.643904197216034, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.6439042091369629, "sampling/importance_sampling_ratio/max": 1.3013635873794556, "sampling/importance_sampling_ratio/mean": 1.0004910826683044, "sampling/importance_sampling_ratio/min": 0.7339659631252289, "sampling/sampling_logp_difference/max": 0.36592050790786745, "sampling/sampling_logp_difference/mean": 0.022513049095869063, "step": 17550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.6, "completions/max_terminated_length": 272.6, "completions/mean_length": 200.2875, "completions/mean_terminated_length": 200.2875, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.0374085322022437, "epoch": 1.6144157396340901, "frac_reward_zero_std": 0.5, "grad_norm": 0.0863497406244278, "kl": 0.009460289351409301, "learning_rate": 2.3917214472983427e-06, "loss": 0.0007, "num_tokens": 76548159.0, "reward": -0.025, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2580137372016906, "sampling/importance_sampling_ratio/mean": 0.9997367143630982, "sampling/importance_sampling_ratio/min": 0.7851546883583069, "sampling/sampling_logp_difference/max": 0.2632436990737915, "sampling/sampling_logp_difference/mean": 0.020933002047240735, "step": 17560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 197.525, "completions/mean_terminated_length": 197.525, "completions/min_length": 116.7, "completions/min_terminated_length": 116.7, "entropy": 0.9455666724592448, "epoch": 1.6153351107842235, "frac_reward_zero_std": 0.5, "grad_norm": 6.137202739715576, "kl": 0.008916637959191575, "learning_rate": 2.3891902566465887e-06, "loss": 0.0007, "num_tokens": 76589009.0, "reward": 0.275, "reward_std": 0.4956935167312622, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.49569355249404906, "sampling/importance_sampling_ratio/max": 1.274504578113556, "sampling/importance_sampling_ratio/mean": 0.999993622303009, "sampling/importance_sampling_ratio/min": 0.7467150181531906, "sampling/sampling_logp_difference/max": 0.3200668454170227, "sampling/sampling_logp_difference/mean": 0.019522672146558763, "step": 17570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 199.625, "completions/mean_terminated_length": 199.625, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.1000275656580925, "epoch": 1.616254481934357, "frac_reward_zero_std": 0.4, "grad_norm": 5.557172775268555, "kl": 0.008729090355336666, "learning_rate": 2.386659179805326e-06, "loss": 0.0003, "num_tokens": 76630947.0, "reward": -0.275, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.3043774843215943, "sampling/importance_sampling_ratio/mean": 0.9997171401977539, "sampling/importance_sampling_ratio/min": 0.7667892575263977, "sampling/sampling_logp_difference/max": 0.3126756250858307, "sampling/sampling_logp_difference/mean": 0.021698906645178796, "step": 17580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.8, "completions/max_terminated_length": 278.8, "completions/mean_length": 189.925, "completions/mean_terminated_length": 189.925, "completions/min_length": 117.2, "completions/min_terminated_length": 117.2, "entropy": 1.0793165490031242, "epoch": 1.6171738530844904, "frac_reward_zero_std": 0.7, "grad_norm": 0.11556874215602875, "kl": 0.009516733168857172, "learning_rate": 2.3841282193741693e-06, "loss": 0.0004, "num_tokens": 76670901.0, "reward": 0.25, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.3219775676727294, "sampling/importance_sampling_ratio/mean": 1.0004238188266754, "sampling/importance_sampling_ratio/min": 0.7638165473937988, "sampling/sampling_logp_difference/max": 0.3180082321166992, "sampling/sampling_logp_difference/mean": 0.021835457906126975, "step": 17590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.1, "completions/max_terminated_length": 270.1, "completions/mean_length": 197.2625, "completions/mean_terminated_length": 197.2625, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0493264675140381, "epoch": 1.6180932242346235, "frac_reward_zero_std": 0.9, "grad_norm": 0.10594183206558228, "kl": 0.009503921450232156, "learning_rate": 2.3815973779526193e-06, "loss": 0.0003, "num_tokens": 76713114.0, "reward": -0.85, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": -0.85, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.3284902334213258, "sampling/importance_sampling_ratio/mean": 1.0000539422035217, "sampling/importance_sampling_ratio/min": 0.776656574010849, "sampling/sampling_logp_difference/max": 0.3122968316078186, "sampling/sampling_logp_difference/mean": 0.02070640493184328, "step": 17600 }, { "epoch": 1.6180932242346235, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 299.02966841186736, "eval_completions/max_terminated_length": 296.76614310645726, "eval_completions/mean_length": 208.45811518324606, "eval_completions/mean_terminated_length": 207.8615372210065, "eval_completions/min_length": 134.47993019197207, "eval_completions/min_terminated_length": 134.47993019197207, "eval_entropy": 1.0813776320187833, "eval_frac_reward_zero_std": 0.4973821989528796, "eval_kl": 0.009902644686296869, "eval_loss": 0.0005361461080610752, "eval_num_tokens": 76713114.0, "eval_reward": -0.10340314136125654, "eval_reward_std": 0.4433018160532074, "eval_rewards/reward_fn/mean": -0.10340314136125654, "eval_rewards/reward_fn/std": 0.4433018293680321, "eval_runtime": 1739.5659, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.2757050928645108, "eval_sampling/importance_sampling_ratio/mean": 0.9999209012036548, "eval_sampling/importance_sampling_ratio/min": 0.7758978799011487, "eval_sampling/sampling_logp_difference/max": 0.2878815715849712, "eval_sampling/sampling_logp_difference/mean": 0.020594832763712965, "eval_steps_per_second": 0.041, "step": 17600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.9, "completions/max_terminated_length": 264.9, "completions/mean_length": 198.1375, "completions/mean_terminated_length": 198.1375, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.076672737300396, "epoch": 1.6190125953847567, "frac_reward_zero_std": 0.7, "grad_norm": 0.11817565560340881, "kl": 0.009140083455713465, "learning_rate": 2.3790666581400537e-06, "loss": 0.0005, "num_tokens": 76755917.0, "reward": 0.125, "reward_std": 0.3105294942855835, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.31052953004837036, "sampling/importance_sampling_ratio/max": 1.270512306690216, "sampling/importance_sampling_ratio/mean": 0.9999937295913697, "sampling/importance_sampling_ratio/min": 0.7728993058204651, "sampling/sampling_logp_difference/max": 0.26617065668106077, "sampling/sampling_logp_difference/mean": 0.021613417379558086, "step": 17610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 205.8125, "completions/mean_terminated_length": 205.8125, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0400977961719036, "epoch": 1.6199319665348901, "frac_reward_zero_std": 0.3, "grad_norm": 6.53712272644043, "kl": 0.010070735856425017, "learning_rate": 2.376536062535725e-06, "loss": 0.0004, "num_tokens": 76799278.0, "reward": 0.025, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2371646285057067, "sampling/importance_sampling_ratio/mean": 1.0002325832843781, "sampling/importance_sampling_ratio/min": 0.7887971222400665, "sampling/sampling_logp_difference/max": 0.2513124108314514, "sampling/sampling_logp_difference/mean": 0.0206080861389637, "step": 17620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.9, "completions/max_terminated_length": 319.9, "completions/mean_length": 221.4125, "completions/mean_terminated_length": 221.4125, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.1214825950562954, "epoch": 1.6208513376850235, "frac_reward_zero_std": 0.1, "grad_norm": 6.738011837005615, "kl": 0.009784831729484722, "learning_rate": 2.3740055937387558e-06, "loss": 0.0009, "num_tokens": 76845927.0, "reward": -0.45625, "reward_std": 0.8054240092635154, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.8054240688681602, "sampling/importance_sampling_ratio/max": 1.287937581539154, "sampling/importance_sampling_ratio/mean": 0.9997939527034759, "sampling/importance_sampling_ratio/min": 0.7163054406642914, "sampling/sampling_logp_difference/max": 0.3459011226892471, "sampling/sampling_logp_difference/mean": 0.02201937548816204, "step": 17630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 217.1875, "completions/mean_terminated_length": 217.1875, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.0134164899587632, "epoch": 1.6217707088351567, "frac_reward_zero_std": 0.1, "grad_norm": 0.08953075110912323, "kl": 0.00877972278976813, "learning_rate": 2.371475254348142e-06, "loss": 0.0008, "num_tokens": 76888486.0, "reward": 0.125, "reward_std": 0.7566805839538574, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.7566805958747864, "sampling/importance_sampling_ratio/max": 1.2501243948936462, "sampling/importance_sampling_ratio/mean": 0.9999528288841247, "sampling/importance_sampling_ratio/min": 0.7858585178852081, "sampling/sampling_logp_difference/max": 0.2529030740261078, "sampling/sampling_logp_difference/mean": 0.02007988318800926, "step": 17640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.5, "completions/max_terminated_length": 281.5, "completions/mean_length": 200.2125, "completions/mean_terminated_length": 200.2125, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.0162309244275094, "epoch": 1.62269007998529, "frac_reward_zero_std": 0.5, "grad_norm": 0.1069503203034401, "kl": 0.014797501612338237, "learning_rate": 2.368945046962744e-06, "loss": 0.0006, "num_tokens": 76928775.0, "reward": -0.025, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.331772220134735, "sampling/importance_sampling_ratio/mean": 1.0002005100250244, "sampling/importance_sampling_ratio/min": 0.76026691198349, "sampling/sampling_logp_difference/max": 0.32232309579849244, "sampling/sampling_logp_difference/mean": 0.021405785344541073, "step": 17650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.8, "completions/max_terminated_length": 326.8, "completions/mean_length": 227.975, "completions/mean_terminated_length": 227.975, "completions/min_length": 157.9, "completions/min_terminated_length": 157.9, "entropy": 1.1590615287423134, "epoch": 1.6236094511354233, "frac_reward_zero_std": 0.4, "grad_norm": 0.2863815724849701, "kl": 0.010781078494619577, "learning_rate": 2.366414974181288e-06, "loss": 0.0009, "num_tokens": 76974765.0, "reward": -0.50625, "reward_std": 0.5522001430392265, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.5522001430392265, "sampling/importance_sampling_ratio/max": 1.281568455696106, "sampling/importance_sampling_ratio/mean": 1.0001379370689392, "sampling/importance_sampling_ratio/min": 0.7725229978561401, "sampling/sampling_logp_difference/max": 0.27144365310668944, "sampling/sampling_logp_difference/mean": 0.02260287869721651, "step": 17660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.3, "completions/max_terminated_length": 333.3, "completions/mean_length": 217.9, "completions/mean_terminated_length": 217.9, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.098167682439089, "epoch": 1.6245288222855567, "frac_reward_zero_std": 0.4, "grad_norm": 6.08392858505249, "kl": 0.010326721909223124, "learning_rate": 2.3638850386023637e-06, "loss": 0.0002, "num_tokens": 77017653.0, "reward": -0.075, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.3326937317848206, "sampling/importance_sampling_ratio/mean": 0.9998773157596588, "sampling/importance_sampling_ratio/min": 0.7586337268352509, "sampling/sampling_logp_difference/max": 0.31870203018188475, "sampling/sampling_logp_difference/mean": 0.021946945413947105, "step": 17670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.6, "completions/max_terminated_length": 315.6, "completions/mean_length": 218.45, "completions/mean_terminated_length": 218.45, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0878883443772793, "epoch": 1.6254481934356901, "frac_reward_zero_std": 0.6, "grad_norm": 0.09985239058732986, "kl": 0.010701093840179966, "learning_rate": 2.3613552428244134e-06, "loss": 0.0005, "num_tokens": 77062713.0, "reward": -0.1, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.3051358819007874, "sampling/importance_sampling_ratio/mean": 0.9995378911495209, "sampling/importance_sampling_ratio/min": 0.7508586287498474, "sampling/sampling_logp_difference/max": 0.32528077960014345, "sampling/sampling_logp_difference/mean": 0.022456702217459677, "step": 17680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.6, "completions/max_terminated_length": 329.6, "completions/mean_length": 230.9875, "completions/mean_terminated_length": 230.9875, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.0666367679834365, "epoch": 1.6263675645858233, "frac_reward_zero_std": 0.4, "grad_norm": 6.310216426849365, "kl": 0.009528563037747517, "learning_rate": 2.3588255894457436e-06, "loss": 0.0003, "num_tokens": 77108536.0, "reward": -0.35, "reward_std": 0.5554606795310975, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5554607272148132, "sampling/importance_sampling_ratio/max": 1.261524486541748, "sampling/importance_sampling_ratio/mean": 0.9992310166358948, "sampling/importance_sampling_ratio/min": 0.7556246519088745, "sampling/sampling_logp_difference/max": 0.3018079251050949, "sampling/sampling_logp_difference/mean": 0.02155262418091297, "step": 17690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 203.6625, "completions/mean_terminated_length": 203.6625, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.1148892015218734, "epoch": 1.6272869357359565, "frac_reward_zero_std": 0.6, "grad_norm": 4.885404586791992, "kl": 0.010545367107260973, "learning_rate": 2.3562960810645095e-06, "loss": 0.0003, "num_tokens": 77148157.0, "reward": -0.3, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.257027530670166, "sampling/importance_sampling_ratio/mean": 1.0004752933979035, "sampling/importance_sampling_ratio/min": 0.8086339354515075, "sampling/sampling_logp_difference/max": 0.23980360031127929, "sampling/sampling_logp_difference/mean": 0.022156060114502905, "step": 17700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.3, "completions/max_terminated_length": 281.2, "completions/mean_length": 208.1125, "completions/mean_terminated_length": 204.7517868041992, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.090186507254839, "epoch": 1.62820630688609, "frac_reward_zero_std": 0.5, "grad_norm": 6.014030456542969, "kl": 0.010130929993465543, "learning_rate": 2.3537667202787202e-06, "loss": 0.0005, "num_tokens": 77193486.0, "reward": -0.18125, "reward_std": 0.46937641501426697, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.46937642693519593, "sampling/importance_sampling_ratio/max": 1.2693976879119873, "sampling/importance_sampling_ratio/mean": 0.9998913466930389, "sampling/importance_sampling_ratio/min": 0.7629740357398986, "sampling/sampling_logp_difference/max": 0.28572673797607423, "sampling/sampling_logp_difference/mean": 0.02251989524811506, "step": 17710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.5, "completions/max_terminated_length": 302.5, "completions/mean_length": 210.3, "completions/mean_terminated_length": 210.3, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.024077306687832, "epoch": 1.6291256780362233, "frac_reward_zero_std": 0.4, "grad_norm": 5.0540876388549805, "kl": 0.010023273632396013, "learning_rate": 2.35123750968623e-06, "loss": 0.0005, "num_tokens": 77237294.0, "reward": -0.075, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.3585466504096986, "sampling/importance_sampling_ratio/mean": 0.9999908149242401, "sampling/importance_sampling_ratio/min": 0.7679339826107026, "sampling/sampling_logp_difference/max": 0.34028669595718386, "sampling/sampling_logp_difference/mean": 0.020862051844596864, "step": 17720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 211.6375, "completions/mean_terminated_length": 211.6375, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.0766979143023492, "epoch": 1.6300450491863565, "frac_reward_zero_std": 0.3, "grad_norm": 5.576570987701416, "kl": 0.008498966178740375, "learning_rate": 2.3487084518847423e-06, "loss": 0.0008, "num_tokens": 77283169.0, "reward": -0.2, "reward_std": 0.6220328629016876, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6220328748226166, "sampling/importance_sampling_ratio/max": 1.2449389100074768, "sampling/importance_sampling_ratio/mean": 1.0001241862773895, "sampling/importance_sampling_ratio/min": 0.7379796355962753, "sampling/sampling_logp_difference/max": 0.3248395562171936, "sampling/sampling_logp_difference/mean": 0.02125536035746336, "step": 17730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.3, "completions/max_terminated_length": 272.3, "completions/mean_length": 206.225, "completions/mean_terminated_length": 206.225, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0825366526842117, "epoch": 1.6309644203364897, "frac_reward_zero_std": 0.7, "grad_norm": 0.09854549914598465, "kl": 0.010966304832254536, "learning_rate": 2.3461795494718028e-06, "loss": 0.0006, "num_tokens": 77325411.0, "reward": -0.4, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2376651287078857, "sampling/importance_sampling_ratio/mean": 0.9999138414859772, "sampling/importance_sampling_ratio/min": 0.7520466446876526, "sampling/sampling_logp_difference/max": 0.29491489231586454, "sampling/sampling_logp_difference/mean": 0.02217294368892908, "step": 17740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.4, "completions/max_terminated_length": 337.4, "completions/mean_length": 237.8375, "completions/mean_terminated_length": 237.8375, "completions/min_length": 162.7, "completions/min_terminated_length": 162.7, "entropy": 1.0680227946490048, "epoch": 1.631883791486623, "frac_reward_zero_std": 0.3, "grad_norm": 0.08560957759618759, "kl": 0.010791221942054108, "learning_rate": 2.343650805044795e-06, "loss": 0.001, "num_tokens": 77373374.0, "reward": -0.15625, "reward_std": 0.5950253769755364, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.5950254008173943, "sampling/importance_sampling_ratio/max": 1.2750226140022278, "sampling/importance_sampling_ratio/mean": 1.0001985788345338, "sampling/importance_sampling_ratio/min": 0.7887675940990448, "sampling/sampling_logp_difference/max": 0.27223293781280516, "sampling/sampling_logp_difference/mean": 0.021163217164576052, "step": 17750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 204.6625, "completions/mean_terminated_length": 204.6625, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.0904608719050883, "epoch": 1.6328031626367565, "frac_reward_zero_std": 0.5, "grad_norm": 6.752020835876465, "kl": 0.010579762421548367, "learning_rate": 2.3411222212009454e-06, "loss": 0.0009, "num_tokens": 77422323.0, "reward": -0.125, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.270054292678833, "sampling/importance_sampling_ratio/mean": 0.9996828615665436, "sampling/importance_sampling_ratio/min": 0.7733093142509461, "sampling/sampling_logp_difference/max": 0.28919593095779417, "sampling/sampling_logp_difference/mean": 0.021715078502893448, "step": 17760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 205.9875, "completions/mean_terminated_length": 205.9875, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.1254981994628905, "epoch": 1.63372253378689, "frac_reward_zero_std": 0.6, "grad_norm": 5.3190083503723145, "kl": 0.010476738828583621, "learning_rate": 2.338593800537309e-06, "loss": 0.0006, "num_tokens": 77465850.0, "reward": 0.1, "reward_std": 0.3880294978618622, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.38802950978279116, "sampling/importance_sampling_ratio/max": 1.251678192615509, "sampling/importance_sampling_ratio/mean": 0.999956876039505, "sampling/importance_sampling_ratio/min": 0.7782407402992249, "sampling/sampling_logp_difference/max": 0.2630902290344238, "sampling/sampling_logp_difference/mean": 0.021457417495548727, "step": 17770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.4, "completions/max_terminated_length": 269.4, "completions/mean_length": 191.675, "completions/mean_terminated_length": 191.675, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0745037890970708, "epoch": 1.634641904937023, "frac_reward_zero_std": 0.5, "grad_norm": 7.761893272399902, "kl": 0.010155013552866876, "learning_rate": 2.3360655456507773e-06, "loss": 0.0005, "num_tokens": 77505056.0, "reward": 0.0, "reward_std": 0.49493399262428284, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4949340045452118, "sampling/importance_sampling_ratio/max": 1.237851095199585, "sampling/importance_sampling_ratio/mean": 1.0002526938915253, "sampling/importance_sampling_ratio/min": 0.789675360918045, "sampling/sampling_logp_difference/max": 0.25643353462219237, "sampling/sampling_logp_difference/mean": 0.02147747203707695, "step": 17780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.1, "completions/max_terminated_length": 338.1, "completions/mean_length": 224.325, "completions/mean_terminated_length": 224.325, "completions/min_length": 155.3, "completions/min_terminated_length": 155.3, "entropy": 1.1025904536247253, "epoch": 1.6355612760871563, "frac_reward_zero_std": 0.3, "grad_norm": 4.98966121673584, "kl": 0.009884703333955257, "learning_rate": 2.3335374591380713e-06, "loss": 0.0004, "num_tokens": 77551090.0, "reward": -0.1625, "reward_std": 0.5659266039729118, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.5659266039729118, "sampling/importance_sampling_ratio/max": 1.3078085780143738, "sampling/importance_sampling_ratio/mean": 0.9998272061347961, "sampling/importance_sampling_ratio/min": 0.79916552901268, "sampling/sampling_logp_difference/max": 0.2813318490982056, "sampling/sampling_logp_difference/mean": 0.021176269650459288, "step": 17790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 206.925, "completions/mean_terminated_length": 206.925, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0496532317250966, "epoch": 1.6364806472372897, "frac_reward_zero_std": 0.4, "grad_norm": 4.9014763832092285, "kl": 0.010804770002141594, "learning_rate": 2.3310095435957392e-06, "loss": 0.0002, "num_tokens": 77595292.0, "reward": 0.0, "reward_std": 0.5117493987083435, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5117493987083435, "sampling/importance_sampling_ratio/max": 1.2655568957328795, "sampling/importance_sampling_ratio/mean": 0.9995510041713714, "sampling/importance_sampling_ratio/min": 0.7881701171398163, "sampling/sampling_logp_difference/max": 0.27670290470123293, "sampling/sampling_logp_difference/mean": 0.020695522148162126, "step": 17800 }, { "epoch": 1.6364806472372897, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 296.8568935427574, "eval_completions/max_terminated_length": 293.9476439790576, "eval_completions/mean_length": 205.64550610820245, "eval_completions/mean_terminated_length": 204.92396976810474, "eval_completions/min_length": 133.79406631762652, "eval_completions/min_terminated_length": 133.79406631762652, "eval_entropy": 1.0770749350908955, "eval_frac_reward_zero_std": 0.49214659685863876, "eval_kl": 0.010005207122815529, "eval_loss": 0.0004769090737681836, "eval_num_tokens": 77595292.0, "eval_reward": -0.0824607329842932, "eval_reward_std": 0.4425290392851538, "eval_rewards/reward_fn/mean": -0.0824607329842932, "eval_rewards/reward_fn/std": 0.4425290545763978, "eval_runtime": 1727.93, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.274916970084891, "eval_sampling/importance_sampling_ratio/mean": 1.0000052905623618, "eval_sampling/importance_sampling_ratio/min": 0.7726156207071341, "eval_sampling/sampling_logp_difference/max": 0.28977115428884614, "eval_sampling/sampling_logp_difference/mean": 0.021092323004471277, "eval_steps_per_second": 0.042, "step": 17800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 276.6, "completions/max_terminated_length": 257.5, "completions/mean_length": 193.275, "completions/mean_terminated_length": 189.8750015258789, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.1980206094682218, "epoch": 1.637400018387423, "frac_reward_zero_std": 0.6, "grad_norm": 3.681450128555298, "kl": 0.01034168220940046, "learning_rate": 2.328481801620151e-06, "loss": 0.0007, "num_tokens": 77640970.0, "reward": -0.03125, "reward_std": 0.273552368581295, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.273552368581295, "sampling/importance_sampling_ratio/max": 1.2687278270721436, "sampling/importance_sampling_ratio/mean": 0.999175763130188, "sampling/importance_sampling_ratio/min": 0.711947537958622, "sampling/sampling_logp_difference/max": 0.48818355798721313, "sampling/sampling_logp_difference/mean": 0.023449908010661602, "step": 17810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.1, "completions/max_terminated_length": 285.1, "completions/mean_length": 208.9375, "completions/mean_terminated_length": 208.9375, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.0456743575632572, "epoch": 1.6383193895375563, "frac_reward_zero_std": 0.4, "grad_norm": 5.356698989868164, "kl": 0.008972714832634666, "learning_rate": 2.3259542358075023e-06, "loss": -0.0002, "num_tokens": 77682669.0, "reward": 0.35, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.266387927532196, "sampling/importance_sampling_ratio/mean": 0.9993927299976348, "sampling/importance_sampling_ratio/min": 0.7794513404369354, "sampling/sampling_logp_difference/max": 0.2688866376876831, "sampling/sampling_logp_difference/mean": 0.02159441765397787, "step": 17820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.0, "completions/max_terminated_length": 296.0, "completions/mean_length": 211.2125, "completions/mean_terminated_length": 211.2125, "completions/min_length": 148.2, "completions/min_terminated_length": 148.2, "entropy": 1.0315821416676045, "epoch": 1.6392387606876895, "frac_reward_zero_std": 0.5, "grad_norm": 0.12792696058750153, "kl": 0.009836023338721134, "learning_rate": 2.3234268487538043e-06, "loss": 0.0008, "num_tokens": 77726950.0, "reward": -0.4, "reward_std": 0.4772325396537781, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4772325396537781, "sampling/importance_sampling_ratio/max": 1.3424318790435792, "sampling/importance_sampling_ratio/mean": 0.9995646357536316, "sampling/importance_sampling_ratio/min": 0.7750022947788239, "sampling/sampling_logp_difference/max": 0.3260743677616119, "sampling/sampling_logp_difference/mean": 0.020783541537821294, "step": 17830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 208.175, "completions/mean_terminated_length": 208.175, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0838793084025382, "epoch": 1.6401581318378229, "frac_reward_zero_std": 0.7, "grad_norm": 0.09270501136779785, "kl": 0.009045620734104887, "learning_rate": 2.320899643054887e-06, "loss": 0.0004, "num_tokens": 77770620.0, "reward": -0.3, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.230869472026825, "sampling/importance_sampling_ratio/mean": 0.9997459650039673, "sampling/importance_sampling_ratio/min": 0.8024027049541473, "sampling/sampling_logp_difference/max": 0.22654207944869995, "sampling/sampling_logp_difference/mean": 0.021346067264676093, "step": 17840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 216.2625, "completions/mean_terminated_length": 216.2625, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.1040927536785603, "epoch": 1.6410775029879563, "frac_reward_zero_std": 0.4, "grad_norm": 0.12777680158615112, "kl": 0.009092086192686111, "learning_rate": 2.318372621306393e-06, "loss": 0.0004, "num_tokens": 77815889.0, "reward": -0.35, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.2934927701950074, "sampling/importance_sampling_ratio/mean": 1.0002983152866363, "sampling/importance_sampling_ratio/min": 0.7785047113895416, "sampling/sampling_logp_difference/max": 0.28239849805831907, "sampling/sampling_logp_difference/mean": 0.021453853510320187, "step": 17850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 220.4625, "completions/mean_terminated_length": 220.4625, "completions/min_length": 151.1, "completions/min_terminated_length": 151.1, "entropy": 1.0507981553673744, "epoch": 1.6419968741380897, "frac_reward_zero_std": 0.3, "grad_norm": 4.310059070587158, "kl": 0.009973036430892535, "learning_rate": 2.3158457861037766e-06, "loss": 0.0008, "num_tokens": 77861814.0, "reward": -0.18125, "reward_std": 0.6276857256889343, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.6276857376098632, "sampling/importance_sampling_ratio/max": 1.2741055727005004, "sampling/importance_sampling_ratio/mean": 0.9998876690864563, "sampling/importance_sampling_ratio/min": 0.7782795488834381, "sampling/sampling_logp_difference/max": 0.2776454448699951, "sampling/sampling_logp_difference/mean": 0.02079388815909624, "step": 17860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.2, "completions/max_terminated_length": 300.2, "completions/mean_length": 200.175, "completions/mean_terminated_length": 200.175, "completions/min_length": 134.1, "completions/min_terminated_length": 134.1, "entropy": 1.1016972228884696, "epoch": 1.6429162452882229, "frac_reward_zero_std": 0.4, "grad_norm": 0.07273315638303757, "kl": 0.010060309391701595, "learning_rate": 2.313319140042298e-06, "loss": 0.0006, "num_tokens": 77904356.0, "reward": -0.475, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.317233717441559, "sampling/importance_sampling_ratio/mean": 0.9999729037284851, "sampling/importance_sampling_ratio/min": 0.650747437775135, "sampling/sampling_logp_difference/max": 0.5896847605705261, "sampling/sampling_logp_difference/mean": 0.021869450993835925, "step": 17870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.5, "completions/max_terminated_length": 303.5, "completions/mean_length": 206.9375, "completions/mean_terminated_length": 206.9375, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.1058786787092685, "epoch": 1.643835616438356, "frac_reward_zero_std": 0.3, "grad_norm": 0.42397448420524597, "kl": 0.011538985729566775, "learning_rate": 2.310792685717028e-06, "loss": 0.0008, "num_tokens": 77952391.0, "reward": -0.275, "reward_std": 0.6152592301368713, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6152592420578002, "sampling/importance_sampling_ratio/max": 1.3326586842536927, "sampling/importance_sampling_ratio/mean": 0.9999745845794678, "sampling/importance_sampling_ratio/min": 0.7634446144104003, "sampling/sampling_logp_difference/max": 0.3267823100090027, "sampling/sampling_logp_difference/mean": 0.021637787111103535, "step": 17880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.6, "completions/max_terminated_length": 292.6, "completions/mean_length": 214.4625, "completions/mean_terminated_length": 214.4625, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.0339158542454243, "epoch": 1.6447549875884895, "frac_reward_zero_std": 0.9, "grad_norm": 0.11031918227672577, "kl": 0.010517419644747861, "learning_rate": 2.3082664257228372e-06, "loss": 0.0007, "num_tokens": 77996292.0, "reward": 0.25, "reward_std": 0.09258201122283935, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.09258201122283935, "sampling/importance_sampling_ratio/max": 1.2479847192764282, "sampling/importance_sampling_ratio/mean": 1.0000067174434661, "sampling/importance_sampling_ratio/min": 0.7987302958965301, "sampling/sampling_logp_difference/max": 0.24385786056518555, "sampling/sampling_logp_difference/mean": 0.020447950437664987, "step": 17890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.4, "completions/max_terminated_length": 301.4, "completions/mean_length": 207.7125, "completions/mean_terminated_length": 207.7125, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 1.07267851755023, "epoch": 1.6456743587386229, "frac_reward_zero_std": 0.5, "grad_norm": 4.755181312561035, "kl": 0.009629569749813527, "learning_rate": 2.3057403626543955e-06, "loss": 0.0004, "num_tokens": 78038957.0, "reward": 0.3, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2386142492294312, "sampling/importance_sampling_ratio/mean": 1.0001162707805633, "sampling/importance_sampling_ratio/min": 0.774039912223816, "sampling/sampling_logp_difference/max": 0.25907522439956665, "sampling/sampling_logp_difference/mean": 0.02173363883048296, "step": 17900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 205.65, "completions/mean_terminated_length": 205.65, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.0629710242152215, "epoch": 1.646593729888756, "frac_reward_zero_std": 0.3, "grad_norm": 5.34403657913208, "kl": 0.01065746836247854, "learning_rate": 2.3032144991061735e-06, "loss": 0.0003, "num_tokens": 78082473.0, "reward": 0.1, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.2275256514549255, "sampling/importance_sampling_ratio/mean": 1.0001774966716765, "sampling/importance_sampling_ratio/min": 0.7712134599685669, "sampling/sampling_logp_difference/max": 0.2627787828445435, "sampling/sampling_logp_difference/mean": 0.02143404334783554, "step": 17910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.7, "completions/max_terminated_length": 266.7, "completions/mean_length": 197.8, "completions/mean_terminated_length": 197.8, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0124514263123274, "epoch": 1.6475131010388893, "frac_reward_zero_std": 0.5, "grad_norm": 8.121606826782227, "kl": 0.00829592409718316, "learning_rate": 2.3006888376724352e-06, "loss": 0.0004, "num_tokens": 78127193.0, "reward": -0.025, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2488903284072876, "sampling/importance_sampling_ratio/mean": 1.0001203298568726, "sampling/importance_sampling_ratio/min": 0.7787292718887329, "sampling/sampling_logp_difference/max": 0.26261247992515563, "sampling/sampling_logp_difference/mean": 0.02102948762476444, "step": 17920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.3, "completions/max_terminated_length": 265.3, "completions/mean_length": 185.2375, "completions/mean_terminated_length": 185.2375, "completions/min_length": 108.3, "completions/min_terminated_length": 108.3, "entropy": 1.0109515517950058, "epoch": 1.6484324721890227, "frac_reward_zero_std": 0.5, "grad_norm": 0.08171464502811432, "kl": 0.009517618050449528, "learning_rate": 2.298163380947238e-06, "loss": 0.0007, "num_tokens": 78166564.0, "reward": 0.275, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2925055146217346, "sampling/importance_sampling_ratio/mean": 0.9999152064323426, "sampling/importance_sampling_ratio/min": 0.7875445783138275, "sampling/sampling_logp_difference/max": 0.2724154531955719, "sampling/sampling_logp_difference/mean": 0.020837948098778723, "step": 17930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.9, "completions/max_terminated_length": 316.9, "completions/mean_length": 225.8125, "completions/mean_terminated_length": 225.8125, "completions/min_length": 157.1, "completions/min_terminated_length": 157.1, "entropy": 0.953479508496821, "epoch": 1.649351843339156, "frac_reward_zero_std": 0.5, "grad_norm": 5.192276954650879, "kl": 0.008768425481684971, "learning_rate": 2.295638131524427e-06, "loss": 0.0006, "num_tokens": 78211621.0, "reward": -0.45, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2769248127937316, "sampling/importance_sampling_ratio/mean": 0.9999314486980438, "sampling/importance_sampling_ratio/min": 0.7639974534511567, "sampling/sampling_logp_difference/max": 0.28955717384815216, "sampling/sampling_logp_difference/mean": 0.01970397774130106, "step": 17940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.4, "completions/max_terminated_length": 281.4, "completions/mean_length": 198.5, "completions/mean_terminated_length": 198.5, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.1712536834180356, "epoch": 1.6502712144892895, "frac_reward_zero_std": 0.5, "grad_norm": 0.12380547821521759, "kl": 0.01081035468087066, "learning_rate": 2.293113091997636e-06, "loss": 0.0009, "num_tokens": 78254853.0, "reward": 0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.3170486330986022, "sampling/importance_sampling_ratio/mean": 0.9998917818069458, "sampling/importance_sampling_ratio/min": 0.7741102755069733, "sampling/sampling_logp_difference/max": 0.3300697565078735, "sampling/sampling_logp_difference/mean": 0.02355287503451109, "step": 17950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.3, "completions/max_terminated_length": 281.3, "completions/mean_length": 196.45, "completions/mean_terminated_length": 196.45, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0424700565636158, "epoch": 1.6511905856394227, "frac_reward_zero_std": 0.8, "grad_norm": 0.11194979399442673, "kl": 0.009523244632873684, "learning_rate": 2.290588264960284e-06, "loss": 0.0006, "num_tokens": 78295425.0, "reward": 0.125, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2597252964973449, "sampling/importance_sampling_ratio/mean": 0.9998632729053497, "sampling/importance_sampling_ratio/min": 0.7428047716617584, "sampling/sampling_logp_difference/max": 0.31410592794418335, "sampling/sampling_logp_difference/mean": 0.02081190552562475, "step": 17960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.5, "completions/max_terminated_length": 325.5, "completions/mean_length": 222.25, "completions/mean_terminated_length": 222.25, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.123719296604395, "epoch": 1.6521099567895559, "frac_reward_zero_std": 0.4, "grad_norm": 5.798583030700684, "kl": 0.01026604725047946, "learning_rate": 2.2880636530055704e-06, "loss": 0.0003, "num_tokens": 78338789.0, "reward": -0.28125, "reward_std": 0.49489451497793197, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.4948945388197899, "sampling/importance_sampling_ratio/max": 1.2706136584281922, "sampling/importance_sampling_ratio/mean": 0.9995575845241547, "sampling/importance_sampling_ratio/min": 0.7888101279735565, "sampling/sampling_logp_difference/max": 0.2815037965774536, "sampling/sampling_logp_difference/mean": 0.02141082864254713, "step": 17970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 367.8, "completions/max_terminated_length": 348.2, "completions/mean_length": 255.5375, "completions/mean_terminated_length": 252.12142944335938, "completions/min_length": 160.0, "completions/min_terminated_length": 160.0, "entropy": 1.147061814367771, "epoch": 1.6530293279396893, "frac_reward_zero_std": 0.5, "grad_norm": 4.903158187866211, "kl": 0.010455524973804132, "learning_rate": 2.2855392587264746e-06, "loss": 0.0006, "num_tokens": 78387296.0, "reward": -0.53125, "reward_std": 0.4607496678829193, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.4607496738433838, "sampling/importance_sampling_ratio/max": 1.2900460481643676, "sampling/importance_sampling_ratio/mean": 1.0002363681793214, "sampling/importance_sampling_ratio/min": 0.7557786583900452, "sampling/sampling_logp_difference/max": 0.31602734327316284, "sampling/sampling_logp_difference/mean": 0.022508405707776548, "step": 17980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 202.8125, "completions/mean_terminated_length": 202.8125, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0502284720540047, "epoch": 1.6539486990898227, "frac_reward_zero_std": 0.5, "grad_norm": 0.10030568391084671, "kl": 0.008804139512358233, "learning_rate": 2.2830150847157505e-06, "loss": 0.0001, "num_tokens": 78428577.0, "reward": -0.125, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2659681916236878, "sampling/importance_sampling_ratio/mean": 1.0006165981292725, "sampling/importance_sampling_ratio/min": 0.7512517929077148, "sampling/sampling_logp_difference/max": 0.2969036340713501, "sampling/sampling_logp_difference/mean": 0.020833934657275677, "step": 17990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.9, "completions/max_terminated_length": 295.9, "completions/mean_length": 207.625, "completions/mean_terminated_length": 207.625, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0909955210983753, "epoch": 1.6548680702399559, "frac_reward_zero_std": 0.7, "grad_norm": 0.1049441546201706, "kl": 0.009916726281517185, "learning_rate": 2.280491133565928e-06, "loss": 0.0003, "num_tokens": 78474291.0, "reward": 0.125, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2456366539001464, "sampling/importance_sampling_ratio/mean": 0.999729311466217, "sampling/importance_sampling_ratio/min": 0.7445263147354126, "sampling/sampling_logp_difference/max": 0.3110589385032654, "sampling/sampling_logp_difference/mean": 0.02210405096411705, "step": 18000 }, { "epoch": 1.6548680702399559, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 296.1116928446771, "eval_completions/max_terminated_length": 292.81151832460733, "eval_completions/mean_length": 205.4960732984293, "eval_completions/mean_terminated_length": 204.71615576619254, "eval_completions/min_length": 133.13961605584643, "eval_completions/min_terminated_length": 133.13961605584643, "eval_entropy": 1.083491291831301, "eval_frac_reward_zero_std": 0.4869109947643979, "eval_kl": 0.009782452404460046, "eval_loss": 0.00047939643263816833, "eval_num_tokens": 78474291.0, "eval_reward": -0.10045811518324607, "eval_reward_std": 0.4503086950714468, "eval_rewards/reward_fn/mean": -0.10045811518324607, "eval_rewards/reward_fn/std": 0.4503087090104038, "eval_runtime": 1725.3008, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.2750172317548156, "eval_sampling/importance_sampling_ratio/mean": 0.9999837696448255, "eval_sampling/importance_sampling_ratio/min": 0.7681653669679352, "eval_sampling/sampling_logp_difference/max": 0.2920472144456434, "eval_sampling/sampling_logp_difference/mean": 0.021537598059959643, "eval_steps_per_second": 0.042, "step": 18000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.6, "completions/max_terminated_length": 240.6, "completions/mean_length": 188.6625, "completions/mean_terminated_length": 188.6625, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 0.9572590790688992, "epoch": 1.655787441390089, "frac_reward_zero_std": 0.4, "grad_norm": 0.11409711092710495, "kl": 0.010366619273554534, "learning_rate": 2.2779674078693066e-06, "loss": 0.0003, "num_tokens": 78513056.0, "reward": 0.19375, "reward_std": 0.5701045751571655, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.5701045989990234, "sampling/importance_sampling_ratio/max": 1.2586786985397338, "sampling/importance_sampling_ratio/mean": 1.000082516670227, "sampling/importance_sampling_ratio/min": 0.7729264378547669, "sampling/sampling_logp_difference/max": 0.2734403669834137, "sampling/sampling_logp_difference/mean": 0.02000009175390005, "step": 18010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 208.8375, "completions/mean_terminated_length": 208.8375, "completions/min_length": 152.6, "completions/min_terminated_length": 152.6, "entropy": 1.1361251384019853, "epoch": 1.6567068125402225, "frac_reward_zero_std": 0.6, "grad_norm": 6.212041854858398, "kl": 0.01115161709021777, "learning_rate": 2.275443910217957e-06, "loss": 0.0004, "num_tokens": 78558139.0, "reward": -0.175, "reward_std": 0.3812558650970459, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.38125587701797486, "sampling/importance_sampling_ratio/max": 1.2690095782279969, "sampling/importance_sampling_ratio/mean": 0.9998415827751159, "sampling/importance_sampling_ratio/min": 0.767910772562027, "sampling/sampling_logp_difference/max": 0.27382272481918335, "sampling/sampling_logp_difference/mean": 0.022094055078923702, "step": 18020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.8, "completions/max_terminated_length": 294.2, "completions/mean_length": 198.4125, "completions/mean_terminated_length": 194.39464416503907, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0844616904854774, "epoch": 1.6576261836903559, "frac_reward_zero_std": 0.6, "grad_norm": 8.274609565734863, "kl": 0.009720679692691192, "learning_rate": 2.2729206432037105e-06, "loss": 0.0003, "num_tokens": 78601388.0, "reward": -0.7, "reward_std": 0.2828427076339722, "rewards/reward_fn/mean": -0.7, "rewards/reward_fn/std": 0.2828427076339722, "sampling/importance_sampling_ratio/max": 1.3040319561958313, "sampling/importance_sampling_ratio/mean": 0.9998144388198853, "sampling/importance_sampling_ratio/min": 0.7479640066623687, "sampling/sampling_logp_difference/max": 0.31880295276641846, "sampling/sampling_logp_difference/mean": 0.02191206868737936, "step": 18030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 209.6375, "completions/mean_terminated_length": 209.6375, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.1384009160101414, "epoch": 1.6585455548404893, "frac_reward_zero_std": 0.5, "grad_norm": 5.868476867675781, "kl": 0.009782867931062356, "learning_rate": 2.270397609418166e-06, "loss": 0.0006, "num_tokens": 78644647.0, "reward": -0.225, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2401066899299622, "sampling/importance_sampling_ratio/mean": 1.0001443862915038, "sampling/importance_sampling_ratio/min": 0.7818807363510132, "sampling/sampling_logp_difference/max": 0.25451197028160094, "sampling/sampling_logp_difference/mean": 0.021522062830626965, "step": 18040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.3, "completions/max_terminated_length": 267.3, "completions/mean_length": 196.5875, "completions/mean_terminated_length": 196.5875, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0367233358323573, "epoch": 1.6594649259906225, "frac_reward_zero_std": 0.5, "grad_norm": 5.8631181716918945, "kl": 0.010529593253158965, "learning_rate": 2.267874811452682e-06, "loss": 0.0006, "num_tokens": 78685134.0, "reward": 0.15, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.3051498651504516, "sampling/importance_sampling_ratio/mean": 0.9995530366897583, "sampling/importance_sampling_ratio/min": 0.7557758867740632, "sampling/sampling_logp_difference/max": 0.315248966217041, "sampling/sampling_logp_difference/mean": 0.021817994117736817, "step": 18050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.9, "completions/max_terminated_length": 263.9, "completions/mean_length": 190.35, "completions/mean_terminated_length": 190.35, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 0.9128828030079603, "epoch": 1.6603842971407556, "frac_reward_zero_std": 0.5, "grad_norm": 0.14355897903442383, "kl": 0.009077654968132265, "learning_rate": 2.265352251898374e-06, "loss": 0.0005, "num_tokens": 78722714.0, "reward": 0.075, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2630086421966553, "sampling/importance_sampling_ratio/mean": 1.0004187047481536, "sampling/importance_sampling_ratio/min": 0.7809902966022492, "sampling/sampling_logp_difference/max": 0.2672960519790649, "sampling/sampling_logp_difference/mean": 0.01961746457964182, "step": 18060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 205.5125, "completions/mean_terminated_length": 205.5125, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0979117378592491, "epoch": 1.661303668290889, "frac_reward_zero_std": 0.4, "grad_norm": 7.718113422393799, "kl": 0.011287921920302323, "learning_rate": 2.2628299333461127e-06, "loss": 0.0004, "num_tokens": 78764243.0, "reward": -0.125, "reward_std": 0.5882755279541015, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5882755637168884, "sampling/importance_sampling_ratio/max": 1.3102652430534363, "sampling/importance_sampling_ratio/mean": 0.9997464835643768, "sampling/importance_sampling_ratio/min": 0.7550436377525329, "sampling/sampling_logp_difference/max": 0.3357470750808716, "sampling/sampling_logp_difference/mean": 0.022137140855193138, "step": 18070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 202.4375, "completions/mean_terminated_length": 202.4375, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.1023616909980773, "epoch": 1.6622230394410225, "frac_reward_zero_std": 0.8, "grad_norm": 0.12079291045665741, "kl": 0.009520220017293468, "learning_rate": 2.260307858386521e-06, "loss": 0.0007, "num_tokens": 78806142.0, "reward": -0.5, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2491139769554138, "sampling/importance_sampling_ratio/mean": 1.000505083799362, "sampling/importance_sampling_ratio/min": 0.7843545436859131, "sampling/sampling_logp_difference/max": 0.25154775381088257, "sampling/sampling_logp_difference/mean": 0.021515671722590925, "step": 18080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.3, "completions/max_terminated_length": 285.7, "completions/mean_length": 195.225, "completions/mean_terminated_length": 191.8625015258789, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.1649117194116116, "epoch": 1.6631424105911556, "frac_reward_zero_std": 0.6, "grad_norm": 6.079902648925781, "kl": 0.009152643696870655, "learning_rate": 2.257786029609972e-06, "loss": 0.0004, "num_tokens": 78849056.0, "reward": -0.45625, "reward_std": 0.32022821307182314, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.32022823095321656, "sampling/importance_sampling_ratio/max": 1.2849591732025147, "sampling/importance_sampling_ratio/mean": 0.9997131407260895, "sampling/importance_sampling_ratio/min": 0.7827137768268585, "sampling/sampling_logp_difference/max": 0.284208619594574, "sampling/sampling_logp_difference/mean": 0.02282832358032465, "step": 18090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.8, "completions/max_terminated_length": 303.8, "completions/mean_length": 214.725, "completions/mean_terminated_length": 214.725, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0258719071745872, "epoch": 1.6640617817412888, "frac_reward_zero_std": 0.3, "grad_norm": 0.09393831342458725, "kl": 0.009649302973411978, "learning_rate": 2.255264449606587e-06, "loss": 0.0006, "num_tokens": 78892642.0, "reward": 0.025, "reward_std": 0.5824600756168365, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5824600756168365, "sampling/importance_sampling_ratio/max": 1.2763447761535645, "sampling/importance_sampling_ratio/mean": 0.999771797657013, "sampling/importance_sampling_ratio/min": 0.7699718952178956, "sampling/sampling_logp_difference/max": 0.27141025066375735, "sampling/sampling_logp_difference/mean": 0.020803667418658734, "step": 18100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.9, "completions/max_terminated_length": 315.9, "completions/mean_length": 215.6875, "completions/mean_terminated_length": 215.6875, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.0385150834918022, "epoch": 1.6649811528914222, "frac_reward_zero_std": 0.6, "grad_norm": 0.08883132040500641, "kl": 0.01050924314185977, "learning_rate": 2.25274312096623e-06, "loss": 0.0005, "num_tokens": 78936001.0, "reward": -0.325, "reward_std": 0.3771016776561737, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.3771016776561737, "sampling/importance_sampling_ratio/max": 1.2613521575927735, "sampling/importance_sampling_ratio/mean": 0.9997688055038452, "sampling/importance_sampling_ratio/min": 0.7573096334934235, "sampling/sampling_logp_difference/max": 0.30801020860671996, "sampling/sampling_logp_difference/mean": 0.021332785487174988, "step": 18110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.3, "completions/max_terminated_length": 322.3, "completions/mean_length": 216.025, "completions/mean_terminated_length": 216.025, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0337418161332608, "epoch": 1.6659005240415556, "frac_reward_zero_std": 0.7, "grad_norm": 4.274101257324219, "kl": 0.00953448585642036, "learning_rate": 2.250222046278508e-06, "loss": 0.0004, "num_tokens": 78981147.0, "reward": -0.425, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.292393112182617, "sampling/importance_sampling_ratio/mean": 0.9998527765274048, "sampling/importance_sampling_ratio/min": 0.7920303344726562, "sampling/sampling_logp_difference/max": 0.2768330335617065, "sampling/sampling_logp_difference/mean": 0.02064917851239443, "step": 18120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 213.4125, "completions/mean_terminated_length": 213.4125, "completions/min_length": 141.1, "completions/min_terminated_length": 141.1, "entropy": 1.118889543414116, "epoch": 1.666819895191689, "frac_reward_zero_std": 0.5, "grad_norm": 6.952661991119385, "kl": 0.012112473617889918, "learning_rate": 2.2477012281327664e-06, "loss": 0.0011, "num_tokens": 79024468.0, "reward": -0.675, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.675, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.267266833782196, "sampling/importance_sampling_ratio/mean": 0.999752938747406, "sampling/importance_sampling_ratio/min": 0.7470804780721665, "sampling/sampling_logp_difference/max": 0.32791012823581694, "sampling/sampling_logp_difference/mean": 0.022258286736905573, "step": 18130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.3, "completions/max_terminated_length": 294.3, "completions/mean_length": 214.4375, "completions/mean_terminated_length": 214.4375, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.048734775185585, "epoch": 1.6677392663418222, "frac_reward_zero_std": 0.6, "grad_norm": 4.290496826171875, "kl": 0.010235325086978265, "learning_rate": 2.245180669118088e-06, "loss": 0.0005, "num_tokens": 79068423.0, "reward": 0.025, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.2719462037086486, "sampling/importance_sampling_ratio/mean": 0.9997754693031311, "sampling/importance_sampling_ratio/min": 0.6901095986366272, "sampling/sampling_logp_difference/max": 0.4106931149959564, "sampling/sampling_logp_difference/mean": 0.021434022486209868, "step": 18140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.8, "completions/max_terminated_length": 273.8, "completions/mean_length": 196.725, "completions/mean_terminated_length": 196.725, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 1.1324665978550912, "epoch": 1.6686586374919554, "frac_reward_zero_std": 0.4, "grad_norm": 6.179656028747559, "kl": 0.009983035761979408, "learning_rate": 2.2426603718232877e-06, "loss": 0.0006, "num_tokens": 79110137.0, "reward": -0.30625, "reward_std": 0.5533159852027894, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5533160150051117, "sampling/importance_sampling_ratio/max": 1.2938971281051637, "sampling/importance_sampling_ratio/mean": 0.9999755084514618, "sampling/importance_sampling_ratio/min": 0.7641196429729462, "sampling/sampling_logp_difference/max": 0.3016135931015015, "sampling/sampling_logp_difference/mean": 0.022713159024715424, "step": 18150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.5, "completions/max_terminated_length": 272.5, "completions/mean_length": 176.8375, "completions/mean_terminated_length": 176.8375, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "entropy": 1.0343358032405376, "epoch": 1.6695780086420888, "frac_reward_zero_std": 0.6, "grad_norm": 7.240379333496094, "kl": 0.01002033380791545, "learning_rate": 2.240140338836916e-06, "loss": 0.0006, "num_tokens": 79150028.0, "reward": 0.35, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.253578007221222, "sampling/importance_sampling_ratio/mean": 1.0000294983386993, "sampling/importance_sampling_ratio/min": 0.782736212015152, "sampling/sampling_logp_difference/max": 0.270962119102478, "sampling/sampling_logp_difference/mean": 0.021041741594672203, "step": 18160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.3, "completions/max_terminated_length": 278.3, "completions/mean_length": 207.175, "completions/mean_terminated_length": 207.175, "completions/min_length": 151.8, "completions/min_terminated_length": 151.8, "entropy": 1.1491720087826252, "epoch": 1.6704973797922222, "frac_reward_zero_std": 0.5, "grad_norm": 5.3238701820373535, "kl": 0.011537869920721278, "learning_rate": 2.2376205727472445e-06, "loss": 0.0005, "num_tokens": 79193330.0, "reward": 0.35, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.2374794721603393, "sampling/importance_sampling_ratio/mean": 1.0003402650356292, "sampling/importance_sampling_ratio/min": 0.7772765696048737, "sampling/sampling_logp_difference/max": 0.2615154981613159, "sampling/sampling_logp_difference/mean": 0.022521641850471497, "step": 18170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 196.575, "completions/mean_terminated_length": 196.575, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.1153589099645616, "epoch": 1.6714167509423554, "frac_reward_zero_std": 0.1, "grad_norm": 7.836032867431641, "kl": 0.010939937940565869, "learning_rate": 2.235101076142278e-06, "loss": 0.0, "num_tokens": 79232496.0, "reward": 0.09375, "reward_std": 0.8071417152881623, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.8071417689323426, "sampling/importance_sampling_ratio/max": 1.2584147334098816, "sampling/importance_sampling_ratio/mean": 1.0008285760879516, "sampling/importance_sampling_ratio/min": 0.7524790644645691, "sampling/sampling_logp_difference/max": 0.29652066230773927, "sampling/sampling_logp_difference/mean": 0.022822916321456433, "step": 18180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 201.1125, "completions/mean_terminated_length": 201.1125, "completions/min_length": 122.7, "completions/min_terminated_length": 122.7, "entropy": 1.2013341270387172, "epoch": 1.6723361220924886, "frac_reward_zero_std": 0.5, "grad_norm": 5.472229480743408, "kl": 0.011099651327822358, "learning_rate": 2.2325818516097404e-06, "loss": 0.0009, "num_tokens": 79276929.0, "reward": 0.05, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3512856125831605, "sampling/importance_sampling_ratio/mean": 0.9999232232570648, "sampling/importance_sampling_ratio/min": 0.7579983234405517, "sampling/sampling_logp_difference/max": 0.3428728699684143, "sampling/sampling_logp_difference/mean": 0.023212940990924837, "step": 18190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.3, "completions/max_terminated_length": 288.3, "completions/mean_length": 191.8375, "completions/mean_terminated_length": 191.8375, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.0374938778579235, "epoch": 1.673255493242622, "frac_reward_zero_std": 0.7, "grad_norm": 0.08803010731935501, "kl": 0.00983313723700121, "learning_rate": 2.2300629017370775e-06, "loss": 0.0005, "num_tokens": 79316804.0, "reward": -0.125, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2530880451202393, "sampling/importance_sampling_ratio/mean": 1.0000390768051148, "sampling/importance_sampling_ratio/min": 0.7956163763999939, "sampling/sampling_logp_difference/max": 0.25456713438034057, "sampling/sampling_logp_difference/mean": 0.020894282311201096, "step": 18200 }, { "epoch": 1.673255493242622, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 286.3874345549738, "eval_completions/max_terminated_length": 284.56195462478183, "eval_completions/mean_length": 198.62543630017453, "eval_completions/mean_terminated_length": 198.12922599219942, "eval_completions/min_length": 126.195462478185, "eval_completions/min_terminated_length": 126.195462478185, "eval_entropy": 1.0862946488990417, "eval_frac_reward_zero_std": 0.4781849912739965, "eval_kl": 0.010077801021872442, "eval_loss": 0.0004909345298074186, "eval_num_tokens": 79316804.0, "eval_reward": -0.11365619546247818, "eval_reward_std": 0.45059729530029063, "eval_rewards/reward_fn/mean": -0.11365619546247818, "eval_rewards/reward_fn/std": 0.4505973087191374, "eval_runtime": 1674.5965, "eval_samples_per_second": 0.342, "eval_sampling/importance_sampling_ratio/max": 1.2680612794183728, "eval_sampling/importance_sampling_ratio/mean": 1.000035453425236, "eval_sampling/importance_sampling_ratio/min": 0.7703115859216837, "eval_sampling/sampling_logp_difference/max": 0.28878540195927777, "eval_sampling/sampling_logp_difference/mean": 0.021145142247513536, "eval_steps_per_second": 0.043, "step": 18200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 199.1, "completions/mean_terminated_length": 199.1, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 1.0876631036400795, "epoch": 1.6741748643927554, "frac_reward_zero_std": 0.6, "grad_norm": 7.5836639404296875, "kl": 0.010848283243831247, "learning_rate": 2.2275442291114527e-06, "loss": 0.0006, "num_tokens": 79361292.0, "reward": -0.4, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2765352368354796, "sampling/importance_sampling_ratio/mean": 1.000105357170105, "sampling/importance_sampling_ratio/min": 0.7621901750564575, "sampling/sampling_logp_difference/max": 0.29054372310638427, "sampling/sampling_logp_difference/mean": 0.021783160418272017, "step": 18210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.0, "completions/max_terminated_length": 300.0, "completions/mean_length": 204.2625, "completions/mean_terminated_length": 204.2625, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.1728282988071441, "epoch": 1.6750942355428886, "frac_reward_zero_std": 0.6, "grad_norm": 0.12161549925804138, "kl": 0.012358330143615603, "learning_rate": 2.2250258363197443e-06, "loss": 0.0007, "num_tokens": 79408961.0, "reward": -0.325, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.3107959389686585, "sampling/importance_sampling_ratio/mean": 1.0005188584327698, "sampling/importance_sampling_ratio/min": 0.7628697693347931, "sampling/sampling_logp_difference/max": 0.31310114860534666, "sampling/sampling_logp_difference/mean": 0.022805792465806007, "step": 18220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.8, "completions/max_terminated_length": 326.8, "completions/mean_length": 220.0875, "completions/mean_terminated_length": 220.0875, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.1314513742923737, "epoch": 1.676013606693022, "frac_reward_zero_std": 0.3, "grad_norm": 0.10385499894618988, "kl": 0.00980663081863895, "learning_rate": 2.2225077259485434e-06, "loss": 0.0007, "num_tokens": 79452008.0, "reward": -0.05, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2967979431152343, "sampling/importance_sampling_ratio/mean": 1.0004857301712036, "sampling/importance_sampling_ratio/min": 0.7763583660125732, "sampling/sampling_logp_difference/max": 0.2943085789680481, "sampling/sampling_logp_difference/mean": 0.022713897004723548, "step": 18230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.8, "completions/max_terminated_length": 263.8, "completions/mean_length": 189.3625, "completions/mean_terminated_length": 189.3625, "completions/min_length": 134.6, "completions/min_terminated_length": 134.6, "entropy": 0.9734312947839499, "epoch": 1.6769329778431552, "frac_reward_zero_std": 0.4, "grad_norm": 0.10381275415420532, "kl": 0.011150528804864734, "learning_rate": 2.2199899005841534e-06, "loss": 0.0006, "num_tokens": 79493045.0, "reward": 0.1, "reward_std": 0.5117493987083435, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5117493987083435, "sampling/importance_sampling_ratio/max": 1.2924529910087585, "sampling/importance_sampling_ratio/mean": 0.9998506486415863, "sampling/importance_sampling_ratio/min": 0.7641176521778107, "sampling/sampling_logp_difference/max": 0.28983094692230227, "sampling/sampling_logp_difference/mean": 0.020659170206636192, "step": 18240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.7, "completions/max_terminated_length": 317.7, "completions/mean_length": 200.7375, "completions/mean_terminated_length": 200.7375, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.0426888741552829, "epoch": 1.6778523489932886, "frac_reward_zero_std": 0.4, "grad_norm": 5.733188629150391, "kl": 0.010384106857236475, "learning_rate": 2.2174723628125793e-06, "loss": 0.0006, "num_tokens": 79533848.0, "reward": 0.325, "reward_std": 0.49325703382492064, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.4932570457458496, "sampling/importance_sampling_ratio/max": 1.2673051834106446, "sampling/importance_sampling_ratio/mean": 1.0003533482551574, "sampling/importance_sampling_ratio/min": 0.7745347380638122, "sampling/sampling_logp_difference/max": 0.27318221926689146, "sampling/sampling_logp_difference/mean": 0.021501114778220653, "step": 18250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 303.2, "completions/max_terminated_length": 298.3, "completions/mean_length": 229.9875, "completions/mean_terminated_length": 224.3375, "completions/min_length": 168.1, "completions/min_terminated_length": 168.1, "entropy": 1.0395020343363286, "epoch": 1.678771720143422, "frac_reward_zero_std": 0.5, "grad_norm": 0.10885463654994965, "kl": 0.010142852406715974, "learning_rate": 2.2149551152195364e-06, "loss": 0.0006, "num_tokens": 79581871.0, "reward": 0.2, "reward_std": 0.42542328834533694, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.42542328834533694, "sampling/importance_sampling_ratio/max": 1.2946672916412354, "sampling/importance_sampling_ratio/mean": 1.000141167640686, "sampling/importance_sampling_ratio/min": 0.7773457884788513, "sampling/sampling_logp_difference/max": 0.2908567786216736, "sampling/sampling_logp_difference/mean": 0.020498538576066493, "step": 18260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 214.6625, "completions/mean_terminated_length": 214.6625, "completions/min_length": 115.6, "completions/min_terminated_length": 115.6, "entropy": 1.0502311758697034, "epoch": 1.6796910912935552, "frac_reward_zero_std": 0.3, "grad_norm": 5.464218616485596, "kl": 0.010260432731593027, "learning_rate": 2.2124381603904395e-06, "loss": 0.0005, "num_tokens": 79626348.0, "reward": 0.19375, "reward_std": 0.5950253769755364, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.5950254008173943, "sampling/importance_sampling_ratio/max": 1.3281798720359803, "sampling/importance_sampling_ratio/mean": 0.9998574495315552, "sampling/importance_sampling_ratio/min": 0.7353073298931122, "sampling/sampling_logp_difference/max": 0.35217142701148985, "sampling/sampling_logp_difference/mean": 0.02030557869002223, "step": 18270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 198.8625, "completions/mean_terminated_length": 198.8625, "completions/min_length": 126.9, "completions/min_terminated_length": 126.9, "entropy": 1.0351539127528668, "epoch": 1.6806104624436884, "frac_reward_zero_std": 0.2, "grad_norm": 5.80466365814209, "kl": 0.011111519887344911, "learning_rate": 2.2099215009104027e-06, "loss": 0.0005, "num_tokens": 79665641.0, "reward": -0.15625, "reward_std": 0.7723727524280548, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.7723727762699127, "sampling/importance_sampling_ratio/max": 1.31126469373703, "sampling/importance_sampling_ratio/mean": 1.0001386523246765, "sampling/importance_sampling_ratio/min": 0.7842384040355682, "sampling/sampling_logp_difference/max": 0.29336408376693723, "sampling/sampling_logp_difference/mean": 0.021001345291733742, "step": 18280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.0, "completions/max_terminated_length": 263.0, "completions/mean_length": 192.4, "completions/mean_terminated_length": 192.4, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 0.9446449667215348, "epoch": 1.6815298335938218, "frac_reward_zero_std": 0.5, "grad_norm": 0.09184888750314713, "kl": 0.008247601275797933, "learning_rate": 2.2074051393642377e-06, "loss": 0.0004, "num_tokens": 79705865.0, "reward": 0.375, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2557476878166198, "sampling/importance_sampling_ratio/mean": 0.999639630317688, "sampling/importance_sampling_ratio/min": 0.7853701770305633, "sampling/sampling_logp_difference/max": 0.25977976322174073, "sampling/sampling_logp_difference/mean": 0.019428456528112294, "step": 18290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 203.45, "completions/mean_terminated_length": 203.45, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0049181766808033, "epoch": 1.6824492047439552, "frac_reward_zero_std": 0.6, "grad_norm": 0.17075756192207336, "kl": 0.010334883659379557, "learning_rate": 2.2048890783364485e-06, "loss": 0.0009, "num_tokens": 79749493.0, "reward": -0.3, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2259426712989807, "sampling/importance_sampling_ratio/mean": 0.9999685406684875, "sampling/importance_sampling_ratio/min": 0.7815629601478576, "sampling/sampling_logp_difference/max": 0.25418243408203123, "sampling/sampling_logp_difference/mean": 0.020114910416305067, "step": 18300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.9, "completions/max_terminated_length": 307.9, "completions/mean_length": 205.7375, "completions/mean_terminated_length": 205.7375, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.1931091763079167, "epoch": 1.6833685758940884, "frac_reward_zero_std": 0.6, "grad_norm": 7.439951419830322, "kl": 0.012568830943200738, "learning_rate": 2.202373320411231e-06, "loss": 0.0002, "num_tokens": 79791144.0, "reward": -0.225, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2644727945327758, "sampling/importance_sampling_ratio/mean": 1.000130832195282, "sampling/importance_sampling_ratio/min": 0.7968757927417756, "sampling/sampling_logp_difference/max": 0.2538326621055603, "sampling/sampling_logp_difference/mean": 0.02239612992852926, "step": 18310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.4, "completions/max_terminated_length": 277.4, "completions/mean_length": 192.65, "completions/mean_terminated_length": 192.65, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 0.9736366480588913, "epoch": 1.6842879470442218, "frac_reward_zero_std": 0.5, "grad_norm": 0.08853477984666824, "kl": 0.00889784308092203, "learning_rate": 2.199857868172472e-06, "loss": 0.0005, "num_tokens": 79829708.0, "reward": -0.1, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.280817949771881, "sampling/importance_sampling_ratio/mean": 1.0000581383705138, "sampling/importance_sampling_ratio/min": 0.7482788503170014, "sampling/sampling_logp_difference/max": 0.32376332879066466, "sampling/sampling_logp_difference/mean": 0.020563033409416676, "step": 18320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 198.9125, "completions/mean_terminated_length": 198.9125, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1063916824758053, "epoch": 1.685207318194355, "frac_reward_zero_std": 0.6, "grad_norm": 0.09701178222894669, "kl": 0.010366004245588556, "learning_rate": 2.197342724203742e-06, "loss": 0.0007, "num_tokens": 79868725.0, "reward": -0.4, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2795913100242615, "sampling/importance_sampling_ratio/mean": 1.000440204143524, "sampling/importance_sampling_ratio/min": 0.7687868416309357, "sampling/sampling_logp_difference/max": 0.29358458518981934, "sampling/sampling_logp_difference/mean": 0.02229181807488203, "step": 18330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.6, "completions/max_terminated_length": 307.9, "completions/mean_length": 207.9875, "completions/mean_terminated_length": 204.7125015258789, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.0795529961586, "epoch": 1.6861266893444884, "frac_reward_zero_std": 0.3, "grad_norm": 0.09998677670955658, "kl": 0.010644776414847002, "learning_rate": 2.194827891088295e-06, "loss": 0.0004, "num_tokens": 79912012.0, "reward": -0.15625, "reward_std": 0.6778950393199921, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.6778950810432434, "sampling/importance_sampling_ratio/max": 1.2931087136268615, "sampling/importance_sampling_ratio/mean": 1.0001393496990203, "sampling/importance_sampling_ratio/min": 0.7622307240962982, "sampling/sampling_logp_difference/max": 0.30389105081558226, "sampling/sampling_logp_difference/mean": 0.02167360335588455, "step": 18340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.2, "completions/max_terminated_length": 296.2, "completions/mean_length": 218.0625, "completions/mean_terminated_length": 218.0625, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.1270072750747204, "epoch": 1.6870460604946218, "frac_reward_zero_std": 0.3, "grad_norm": 6.327423095703125, "kl": 0.010617328359512613, "learning_rate": 2.192313371409066e-06, "loss": 0.0002, "num_tokens": 79955425.0, "reward": 0.175, "reward_std": 0.643904197216034, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6439042091369629, "sampling/importance_sampling_ratio/max": 1.2611382126808166, "sampling/importance_sampling_ratio/mean": 1.0001884818077087, "sampling/importance_sampling_ratio/min": 0.7664469480514526, "sampling/sampling_logp_difference/max": 0.2794557332992554, "sampling/sampling_logp_difference/mean": 0.02168669980019331, "step": 18350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 185.05, "completions/mean_terminated_length": 185.05, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.012656345963478, "epoch": 1.687965431644755, "frac_reward_zero_std": 0.7, "grad_norm": 5.157541751861572, "kl": 0.00913298764207866, "learning_rate": 2.189799167748669e-06, "loss": 0.0002, "num_tokens": 79996933.0, "reward": 0.45, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.227632987499237, "sampling/importance_sampling_ratio/mean": 0.999424934387207, "sampling/importance_sampling_ratio/min": 0.779376232624054, "sampling/sampling_logp_difference/max": 0.2661218583583832, "sampling/sampling_logp_difference/mean": 0.020270367711782457, "step": 18360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 208.2375, "completions/mean_terminated_length": 208.2375, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.0069877110421657, "epoch": 1.6888848027948882, "frac_reward_zero_std": 0.5, "grad_norm": 6.451460361480713, "kl": 0.008820876211393625, "learning_rate": 2.1872852826893916e-06, "loss": 0.0006, "num_tokens": 80041400.0, "reward": 0.025, "reward_std": 0.5100160002708435, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5100160360336303, "sampling/importance_sampling_ratio/max": 1.2458301782608032, "sampling/importance_sampling_ratio/mean": 1.000203275680542, "sampling/importance_sampling_ratio/min": 0.7824194431304932, "sampling/sampling_logp_difference/max": 0.25902565717697146, "sampling/sampling_logp_difference/mean": 0.020628088898956776, "step": 18370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 213.0625, "completions/mean_terminated_length": 213.0625, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.1173180807381868, "epoch": 1.6898041739450216, "frac_reward_zero_std": 0.8, "grad_norm": 0.11046172678470612, "kl": 0.009368308761622757, "learning_rate": 2.1847717188131977e-06, "loss": 0.0006, "num_tokens": 80087789.0, "reward": -0.28125, "reward_std": 0.18220152258872985, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.18220152854919433, "sampling/importance_sampling_ratio/max": 1.2871334075927734, "sampling/importance_sampling_ratio/mean": 0.9999387741088868, "sampling/importance_sampling_ratio/min": 0.7702472627162933, "sampling/sampling_logp_difference/max": 0.30468002557754514, "sampling/sampling_logp_difference/mean": 0.021954997163265943, "step": 18380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.5, "completions/max_terminated_length": 268.5, "completions/mean_length": 191.2625, "completions/mean_terminated_length": 191.2625, "completions/min_length": 116.2, "completions/min_terminated_length": 116.2, "entropy": 1.0292094513773917, "epoch": 1.690723545095155, "frac_reward_zero_std": 0.6, "grad_norm": 8.737700462341309, "kl": 0.010690593143226578, "learning_rate": 2.1822584787017158e-06, "loss": 0.0006, "num_tokens": 80130714.0, "reward": 0.05, "reward_std": 0.38124017119407655, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3812402069568634, "sampling/importance_sampling_ratio/max": 1.3437404870986938, "sampling/importance_sampling_ratio/mean": 1.0002726435661315, "sampling/importance_sampling_ratio/min": 0.7553746581077576, "sampling/sampling_logp_difference/max": 0.34859020113945005, "sampling/sampling_logp_difference/mean": 0.020924570225179195, "step": 18390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 213.15, "completions/mean_terminated_length": 213.15, "completions/min_length": 124.8, "completions/min_terminated_length": 124.8, "entropy": 1.0335017919540406, "epoch": 1.6916429162452882, "frac_reward_zero_std": 0.4, "grad_norm": 6.18092679977417, "kl": 0.01143439780571498, "learning_rate": 2.1797455649362465e-06, "loss": 0.0007, "num_tokens": 80172950.0, "reward": 0.025, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2970817804336547, "sampling/importance_sampling_ratio/mean": 1.0000141143798829, "sampling/importance_sampling_ratio/min": 0.7818075239658355, "sampling/sampling_logp_difference/max": 0.29192112684249877, "sampling/sampling_logp_difference/mean": 0.021163762919604777, "step": 18400 }, { "epoch": 1.6916429162452882, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0006544502617801048, "eval_completions/max_length": 292.3542757417103, "eval_completions/max_terminated_length": 291.9947643979058, "eval_completions/mean_length": 204.967277486911, "eval_completions/mean_terminated_length": 204.80517125420963, "eval_completions/min_length": 134.08551483420592, "eval_completions/min_terminated_length": 134.08551483420592, "eval_entropy": 1.0761093475431671, "eval_frac_reward_zero_std": 0.506108202443281, "eval_kl": 0.009529015280137304, "eval_loss": 0.00048045991570688784, "eval_num_tokens": 80172950.0, "eval_reward": -0.1043848167539267, "eval_reward_std": 0.4300110017129888, "eval_rewards/reward_fn/mean": -0.1043848167539267, "eval_rewards/reward_fn/std": 0.43001101315541623, "eval_runtime": 1705.9946, "eval_samples_per_second": 0.336, "eval_sampling/importance_sampling_ratio/max": 1.2774388892072241, "eval_sampling/importance_sampling_ratio/mean": 0.9999937969560606, "eval_sampling/importance_sampling_ratio/min": 0.7776265282697494, "eval_sampling/sampling_logp_difference/max": 0.2884301720593404, "eval_sampling/sampling_logp_difference/mean": 0.020862747429111032, "eval_steps_per_second": 0.042, "step": 18400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 209.0125, "completions/mean_terminated_length": 209.0125, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.1348601296544074, "epoch": 1.6925622873954216, "frac_reward_zero_std": 0.6, "grad_norm": 4.949255466461182, "kl": 0.010044501285301521, "learning_rate": 2.177232980097754e-06, "loss": 0.0004, "num_tokens": 80215207.0, "reward": 0.025, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.2570040464401244, "sampling/importance_sampling_ratio/mean": 1.00044487118721, "sampling/importance_sampling_ratio/min": 0.7878698885440827, "sampling/sampling_logp_difference/max": 0.25649508833885193, "sampling/sampling_logp_difference/mean": 0.022454370744526386, "step": 18410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/max_terminated_length": 275.3, "completions/mean_length": 204.275, "completions/mean_terminated_length": 204.275, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.1631817825138568, "epoch": 1.6934816585455548, "frac_reward_zero_std": 0.5, "grad_norm": 5.218203544616699, "kl": 0.009486268140608444, "learning_rate": 2.174720726766864e-06, "loss": 0.0006, "num_tokens": 80257813.0, "reward": -0.2, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.2281484603881836, "sampling/importance_sampling_ratio/mean": 0.9994787991046905, "sampling/importance_sampling_ratio/min": 0.7995527625083924, "sampling/sampling_logp_difference/max": 0.2379591464996338, "sampling/sampling_logp_difference/mean": 0.023160577937960624, "step": 18420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.5, "completions/max_terminated_length": 259.5, "completions/mean_length": 187.8125, "completions/mean_terminated_length": 187.8125, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "entropy": 0.9889102958142757, "epoch": 1.6944010296956882, "frac_reward_zero_std": 0.5, "grad_norm": 0.12715435028076172, "kl": 0.010163930733688176, "learning_rate": 2.1722088075238615e-06, "loss": 0.0008, "num_tokens": 80296846.0, "reward": 0.325, "reward_std": 0.4587401747703552, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.4587401866912842, "sampling/importance_sampling_ratio/max": 1.253459668159485, "sampling/importance_sampling_ratio/mean": 0.9999570429325104, "sampling/importance_sampling_ratio/min": 0.8000100910663605, "sampling/sampling_logp_difference/max": 0.2502763748168945, "sampling/sampling_logp_difference/mean": 0.020479115843772887, "step": 18430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.1, "completions/max_terminated_length": 309.1, "completions/mean_length": 221.975, "completions/mean_terminated_length": 221.975, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.0765558294951916, "epoch": 1.6953204008458216, "frac_reward_zero_std": 0.3, "grad_norm": 0.08477757126092911, "kl": 0.009427695721387862, "learning_rate": 2.1696972249486894e-06, "loss": 0.0, "num_tokens": 80343092.0, "reward": 0.15, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.2660741806030273, "sampling/importance_sampling_ratio/mean": 1.0000687539577484, "sampling/importance_sampling_ratio/min": 0.7612644910812378, "sampling/sampling_logp_difference/max": 0.282133960723877, "sampling/sampling_logp_difference/mean": 0.02160679381340742, "step": 18440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.8, "completions/max_terminated_length": 323.8, "completions/mean_length": 217.575, "completions/mean_terminated_length": 217.575, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.071342898160219, "epoch": 1.6962397719959548, "frac_reward_zero_std": 0.3, "grad_norm": 5.125617980957031, "kl": 0.010298516921466216, "learning_rate": 2.167185981620944e-06, "loss": 0.0004, "num_tokens": 80389770.0, "reward": -0.2, "reward_std": 0.5496452271938324, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5496452391147614, "sampling/importance_sampling_ratio/max": 1.2693839073181152, "sampling/importance_sampling_ratio/mean": 0.9994803011417389, "sampling/importance_sampling_ratio/min": 0.7556407451629639, "sampling/sampling_logp_difference/max": 0.29888482093811036, "sampling/sampling_logp_difference/mean": 0.021182439662516116, "step": 18450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.4, "completions/max_terminated_length": 301.4, "completions/mean_length": 224.0, "completions/mean_terminated_length": 224.0, "completions/min_length": 158.8, "completions/min_terminated_length": 158.8, "entropy": 1.0346549108624459, "epoch": 1.697159143146088, "frac_reward_zero_std": 0.4, "grad_norm": 6.4018049240112305, "kl": 0.010829049383755774, "learning_rate": 2.164675080119875e-06, "loss": 0.0005, "num_tokens": 80436298.0, "reward": -0.25, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.3042156457901002, "sampling/importance_sampling_ratio/mean": 1.000152689218521, "sampling/importance_sampling_ratio/min": 0.792611813545227, "sampling/sampling_logp_difference/max": 0.28261423110961914, "sampling/sampling_logp_difference/mean": 0.02067002058029175, "step": 18460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.2, "completions/max_terminated_length": 274.2, "completions/mean_length": 187.9875, "completions/mean_terminated_length": 187.9875, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 1.0768606286495923, "epoch": 1.6980785142962214, "frac_reward_zero_std": 0.5, "grad_norm": 5.984437465667725, "kl": 0.010713950084755198, "learning_rate": 2.162164523024377e-06, "loss": 0.0002, "num_tokens": 80478409.0, "reward": 0.2, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.3275415897369385, "sampling/importance_sampling_ratio/mean": 1.0003776252269745, "sampling/importance_sampling_ratio/min": 0.7515882611274719, "sampling/sampling_logp_difference/max": 0.3494132995605469, "sampling/sampling_logp_difference/mean": 0.021654391102492808, "step": 18470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.9, "completions/max_terminated_length": 266.9, "completions/mean_length": 193.8125, "completions/mean_terminated_length": 193.8125, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.106149858236313, "epoch": 1.6989978854463548, "frac_reward_zero_std": 0.6, "grad_norm": 4.625433921813965, "kl": 0.01046188662876375, "learning_rate": 2.159654312912995e-06, "loss": 0.0004, "num_tokens": 80517514.0, "reward": -0.6, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.237607192993164, "sampling/importance_sampling_ratio/mean": 1.0003574907779693, "sampling/importance_sampling_ratio/min": 0.7869525074958801, "sampling/sampling_logp_difference/max": 0.2523950576782227, "sampling/sampling_logp_difference/mean": 0.022060991637408732, "step": 18480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.4, "completions/max_terminated_length": 289.4, "completions/mean_length": 211.65, "completions/mean_terminated_length": 211.65, "completions/min_length": 159.8, "completions/min_terminated_length": 159.8, "entropy": 1.038796876743436, "epoch": 1.699917256596488, "frac_reward_zero_std": 0.5, "grad_norm": 5.608017921447754, "kl": 0.00904127912945114, "learning_rate": 2.157144452363916e-06, "loss": 0.0001, "num_tokens": 80561734.0, "reward": 0.1, "reward_std": 0.5066213369369507, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5066213846206665, "sampling/importance_sampling_ratio/max": 1.3348865032196044, "sampling/importance_sampling_ratio/mean": 0.9999727129936218, "sampling/importance_sampling_ratio/min": 0.7736990809440613, "sampling/sampling_logp_difference/max": 0.33179832696914674, "sampling/sampling_logp_difference/mean": 0.021227336116135122, "step": 18490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 347.3, "completions/max_terminated_length": 347.3, "completions/mean_length": 229.6125, "completions/mean_terminated_length": 229.6125, "completions/min_length": 141.4, "completions/min_terminated_length": 141.4, "entropy": 1.0634235713630915, "epoch": 1.7008366277466214, "frac_reward_zero_std": 0.4, "grad_norm": 4.681619644165039, "kl": 0.009137643946451135, "learning_rate": 2.1546349439549673e-06, "loss": 0.0008, "num_tokens": 80610143.0, "reward": -0.475, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2354856252670288, "sampling/importance_sampling_ratio/mean": 1.0002855479717254, "sampling/importance_sampling_ratio/min": 0.7632579922676086, "sampling/sampling_logp_difference/max": 0.27076451778411864, "sampling/sampling_logp_difference/mean": 0.021023352444171906, "step": 18500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 207.6, "completions/mean_terminated_length": 207.6, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0917164020240306, "epoch": 1.7017559988967546, "frac_reward_zero_std": 0.3, "grad_norm": 5.583169937133789, "kl": 0.012247518170624971, "learning_rate": 2.152125790263616e-06, "loss": 0.0002, "num_tokens": 80654247.0, "reward": 0.11875, "reward_std": 0.610797768831253, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.610797780752182, "sampling/importance_sampling_ratio/max": 1.261184275150299, "sampling/importance_sampling_ratio/mean": 1.0002543926239014, "sampling/importance_sampling_ratio/min": 0.7838892698287964, "sampling/sampling_logp_difference/max": 0.26151685118675233, "sampling/sampling_logp_difference/mean": 0.022015179879963397, "step": 18510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 207.1375, "completions/mean_terminated_length": 207.1375, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.1536242872476579, "epoch": 1.702675370046888, "frac_reward_zero_std": 0.5, "grad_norm": 8.940282821655273, "kl": 0.00941198993823491, "learning_rate": 2.1496169938669635e-06, "loss": 0.0005, "num_tokens": 80698898.0, "reward": -0.125, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2329297065734863, "sampling/importance_sampling_ratio/mean": 1.000187087059021, "sampling/importance_sampling_ratio/min": 0.7881964206695556, "sampling/sampling_logp_difference/max": 0.24528075456619264, "sampling/sampling_logp_difference/mean": 0.022635064087808132, "step": 18520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 305.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 208.8625, "completions/mean_terminated_length": 205.56785888671874, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.027102929353714, "epoch": 1.7035947411970214, "frac_reward_zero_std": 0.6, "grad_norm": 0.12435127794742584, "kl": 0.009055928437737748, "learning_rate": 2.147108557341744e-06, "loss": 0.0002, "num_tokens": 80742463.0, "reward": -0.00625, "reward_std": 0.37290801405906676, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.37290802001953127, "sampling/importance_sampling_ratio/max": 1.3141415238380432, "sampling/importance_sampling_ratio/mean": 1.0003910422325135, "sampling/importance_sampling_ratio/min": 0.793919563293457, "sampling/sampling_logp_difference/max": 0.28186103105545046, "sampling/sampling_logp_difference/mean": 0.020434556528925896, "step": 18530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.8, "completions/max_terminated_length": 292.8, "completions/mean_length": 209.5375, "completions/mean_terminated_length": 209.5375, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.085011651366949, "epoch": 1.7045141123471546, "frac_reward_zero_std": 0.6, "grad_norm": 6.008273601531982, "kl": 0.010497647814918309, "learning_rate": 2.1446004832643246e-06, "loss": 0.0006, "num_tokens": 80784770.0, "reward": -0.6, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2423208832740784, "sampling/importance_sampling_ratio/mean": 0.9999788165092468, "sampling/importance_sampling_ratio/min": 0.7732636511325837, "sampling/sampling_logp_difference/max": 0.2926609754562378, "sampling/sampling_logp_difference/mean": 0.021430033445358276, "step": 18540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 197.1, "completions/mean_terminated_length": 197.1, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.016027346998453, "epoch": 1.7054334834972877, "frac_reward_zero_std": 0.6, "grad_norm": 5.863893032073975, "kl": 0.01184482024400495, "learning_rate": 2.1420927742106982e-06, "loss": 0.0005, "num_tokens": 80824698.0, "reward": 0.15, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.4284992694854737, "sampling/importance_sampling_ratio/mean": 1.00049769282341, "sampling/importance_sampling_ratio/min": 0.7953985631465912, "sampling/sampling_logp_difference/max": 0.3687659025192261, "sampling/sampling_logp_difference/mean": 0.020542584545910358, "step": 18550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 204.9, "completions/mean_terminated_length": 204.9, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0894927494227886, "epoch": 1.7063528546474211, "frac_reward_zero_std": 0.3, "grad_norm": 4.79653787612915, "kl": 0.00980060621513985, "learning_rate": 2.139585432756482e-06, "loss": 0.0001, "num_tokens": 80868858.0, "reward": -0.25, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2662202358245849, "sampling/importance_sampling_ratio/mean": 1.0005124509334564, "sampling/importance_sampling_ratio/min": 0.7404053002595902, "sampling/sampling_logp_difference/max": 0.32384549379348754, "sampling/sampling_logp_difference/mean": 0.02123507224023342, "step": 18560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.5, "completions/max_terminated_length": 336.5, "completions/mean_length": 215.7625, "completions/mean_terminated_length": 215.7625, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 0.9419424902647734, "epoch": 1.7072722257975546, "frac_reward_zero_std": 0.4, "grad_norm": 5.4477763175964355, "kl": 0.010024701950896997, "learning_rate": 2.1370784614769168e-06, "loss": 0.0008, "num_tokens": 80911007.0, "reward": -0.35625, "reward_std": 0.5499370157718658, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.5499370336532593, "sampling/importance_sampling_ratio/max": 1.2649539589881897, "sampling/importance_sampling_ratio/mean": 0.9997080802917481, "sampling/importance_sampling_ratio/min": 0.7844855725765228, "sampling/sampling_logp_difference/max": 0.26506805419921875, "sampling/sampling_logp_difference/mean": 0.019501664908602834, "step": 18570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 351.8, "completions/max_terminated_length": 330.1, "completions/mean_length": 230.7125, "completions/mean_terminated_length": 226.8750015258789, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.0558921672403812, "epoch": 1.7081915969476877, "frac_reward_zero_std": 0.3, "grad_norm": 5.928819179534912, "kl": 0.011193969490705058, "learning_rate": 2.134571862946864e-06, "loss": 0.0005, "num_tokens": 80954536.0, "reward": -0.28125, "reward_std": 0.5294113740324974, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.5294113978743553, "sampling/importance_sampling_ratio/max": 1.34158296585083, "sampling/importance_sampling_ratio/mean": 0.9998818516731263, "sampling/importance_sampling_ratio/min": 0.76779825091362, "sampling/sampling_logp_difference/max": 0.31929835081100466, "sampling/sampling_logp_difference/mean": 0.02120595332235098, "step": 18580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.9, "completions/max_terminated_length": 302.9, "completions/mean_length": 213.7375, "completions/mean_terminated_length": 213.7375, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.02388726323843, "epoch": 1.709110968097821, "frac_reward_zero_std": 0.7, "grad_norm": 0.12649449706077576, "kl": 0.01085827144270297, "learning_rate": 2.1320656397407996e-06, "loss": 0.0003, "num_tokens": 80996867.0, "reward": 0.225, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2510487198829652, "sampling/importance_sampling_ratio/mean": 1.0003283083438874, "sampling/importance_sampling_ratio/min": 0.7783046007156372, "sampling/sampling_logp_difference/max": 0.26257010698318484, "sampling/sampling_logp_difference/mean": 0.020633606985211373, "step": 18590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 196.3125, "completions/mean_terminated_length": 196.3125, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0271939277648925, "epoch": 1.7100303392479543, "frac_reward_zero_std": 0.7, "grad_norm": 0.09776295721530914, "kl": 0.008707906960626132, "learning_rate": 2.1295597944328185e-06, "loss": 0.0004, "num_tokens": 81039348.0, "reward": -0.25625, "reward_std": 0.15909902304410933, "rewards/reward_fn/mean": -0.25625, "rewards/reward_fn/std": 0.15909902304410933, "sampling/importance_sampling_ratio/max": 1.2514334201812745, "sampling/importance_sampling_ratio/mean": 0.9992482602596283, "sampling/importance_sampling_ratio/min": 0.7511537075042725, "sampling/sampling_logp_difference/max": 0.2981335878372192, "sampling/sampling_logp_difference/mean": 0.02091885879635811, "step": 18600 }, { "epoch": 1.7100303392479543, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 293.44851657940666, "eval_completions/max_terminated_length": 291.3071553228621, "eval_completions/mean_length": 204.9585514834206, "eval_completions/mean_terminated_length": 204.38498321586462, "eval_completions/min_length": 132.93542757417103, "eval_completions/min_terminated_length": 132.93542757417103, "eval_entropy": 1.0847846627443456, "eval_frac_reward_zero_std": 0.4886561954624782, "eval_kl": 0.00995233643110874, "eval_loss": 0.0004960981314070523, "eval_num_tokens": 81039348.0, "eval_reward": -0.12085514834205933, "eval_reward_std": 0.45709200206019285, "eval_rewards/reward_fn/mean": -0.12085514834205933, "eval_rewards/reward_fn/std": 0.45709201683132644, "eval_runtime": 1712.4254, "eval_samples_per_second": 0.335, "eval_sampling/importance_sampling_ratio/max": 1.2690743881994517, "eval_sampling/importance_sampling_ratio/mean": 0.9999651416850132, "eval_sampling/importance_sampling_ratio/min": 0.7657652015669392, "eval_sampling/sampling_logp_difference/max": 0.2942859170012449, "eval_sampling/sampling_logp_difference/mean": 0.021470481123495685, "eval_steps_per_second": 0.042, "step": 18600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 204.3, "completions/mean_terminated_length": 204.3, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0807694531977177, "epoch": 1.7109497103980877, "frac_reward_zero_std": 0.5, "grad_norm": 5.29219913482666, "kl": 0.010734912287443877, "learning_rate": 2.127054329596621e-06, "loss": 0.0004, "num_tokens": 81079612.0, "reward": -0.275, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2746169805526733, "sampling/importance_sampling_ratio/mean": 1.0001780211925506, "sampling/importance_sampling_ratio/min": 0.7703535854816437, "sampling/sampling_logp_difference/max": 0.28158825635910034, "sampling/sampling_logp_difference/mean": 0.021088015846908094, "step": 18610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.6, "completions/max_terminated_length": 278.6, "completions/mean_length": 200.6625, "completions/mean_terminated_length": 200.6625, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.1172574765980243, "epoch": 1.7118690815482211, "frac_reward_zero_std": 0.4, "grad_norm": 0.0928555503487587, "kl": 0.01084886979078874, "learning_rate": 2.124549247805523e-06, "loss": 0.0008, "num_tokens": 81121129.0, "reward": -0.25, "reward_std": 0.5916701912879944, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5916702151298523, "sampling/importance_sampling_ratio/max": 1.343020462989807, "sampling/importance_sampling_ratio/mean": 1.0001794755458833, "sampling/importance_sampling_ratio/min": 0.794276750087738, "sampling/sampling_logp_difference/max": 0.29836896657943723, "sampling/sampling_logp_difference/mean": 0.022589870914816856, "step": 18620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.3, "completions/max_terminated_length": 314.3, "completions/mean_length": 221.8375, "completions/mean_terminated_length": 221.8375, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0618756581097841, "epoch": 1.7127884526983543, "frac_reward_zero_std": 0.7, "grad_norm": 6.490370273590088, "kl": 0.010613532061688602, "learning_rate": 2.1220445516324436e-06, "loss": 0.0003, "num_tokens": 81166188.0, "reward": -0.225, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.362973141670227, "sampling/importance_sampling_ratio/mean": 0.9996197283267975, "sampling/importance_sampling_ratio/min": 0.7607781708240509, "sampling/sampling_logp_difference/max": 0.33750752806663514, "sampling/sampling_logp_difference/mean": 0.02149868756532669, "step": 18630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.5, "completions/max_terminated_length": 312.5, "completions/mean_length": 219.65, "completions/mean_terminated_length": 219.65, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0178046278655528, "epoch": 1.7137078238484875, "frac_reward_zero_std": 0.3, "grad_norm": 5.506438255310059, "kl": 0.010475381015567109, "learning_rate": 2.119540243649907e-06, "loss": 0.0008, "num_tokens": 81207008.0, "reward": -0.1, "reward_std": 0.5824443817138671, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5824444055557251, "sampling/importance_sampling_ratio/max": 1.2323011755943298, "sampling/importance_sampling_ratio/mean": 0.9999956667423249, "sampling/importance_sampling_ratio/min": 0.7746289074420929, "sampling/sampling_logp_difference/max": 0.260089111328125, "sampling/sampling_logp_difference/mean": 0.020513475872576237, "step": 18640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.3, "completions/max_terminated_length": 243.3, "completions/mean_length": 185.5875, "completions/mean_terminated_length": 185.5875, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 0.9799044221639633, "epoch": 1.714627194998621, "frac_reward_zero_std": 0.7, "grad_norm": 0.10557873547077179, "kl": 0.008976552085368895, "learning_rate": 2.1170363264300374e-06, "loss": 0.0008, "num_tokens": 81248735.0, "reward": -0.1, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2440956830978394, "sampling/importance_sampling_ratio/mean": 0.9996157944202423, "sampling/importance_sampling_ratio/min": 0.7673050284385681, "sampling/sampling_logp_difference/max": 0.27261877059936523, "sampling/sampling_logp_difference/mean": 0.0200386106967926, "step": 18650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 207.7125, "completions/mean_terminated_length": 207.7125, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.0074374001473188, "epoch": 1.7155465661487543, "frac_reward_zero_std": 0.6, "grad_norm": 6.714776039123535, "kl": 0.009439090569503605, "learning_rate": 2.1145328025445584e-06, "loss": 0.0007, "num_tokens": 81292304.0, "reward": 0.275, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2973198771476746, "sampling/importance_sampling_ratio/mean": 1.0002149641513824, "sampling/importance_sampling_ratio/min": 0.7484363079071045, "sampling/sampling_logp_difference/max": 0.3108679294586182, "sampling/sampling_logp_difference/mean": 0.021132871881127356, "step": 18660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.3, "completions/max_terminated_length": 313.3, "completions/mean_length": 208.2625, "completions/mean_terminated_length": 208.2625, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.1128377862274648, "epoch": 1.7164659372988875, "frac_reward_zero_std": 0.1, "grad_norm": 5.16945219039917, "kl": 0.010164760780753568, "learning_rate": 2.11202967456479e-06, "loss": 0.0006, "num_tokens": 81335149.0, "reward": -0.125, "reward_std": 0.8147457361221313, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.8147457480430603, "sampling/importance_sampling_ratio/max": 1.2971254229545592, "sampling/importance_sampling_ratio/mean": 0.9999335289001465, "sampling/importance_sampling_ratio/min": 0.7643786847591401, "sampling/sampling_logp_difference/max": 0.316396164894104, "sampling/sampling_logp_difference/mean": 0.02155518587678671, "step": 18670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.2, "completions/max_terminated_length": 333.2, "completions/mean_length": 211.925, "completions/mean_terminated_length": 211.925, "completions/min_length": 118.4, "completions/min_terminated_length": 118.4, "entropy": 1.0630662687122823, "epoch": 1.7173853084490207, "frac_reward_zero_std": 0.3, "grad_norm": 5.83039665222168, "kl": 0.00964620905870106, "learning_rate": 2.109526945061646e-06, "loss": 0.0004, "num_tokens": 81377039.0, "reward": -0.15, "reward_std": 0.6152435362339019, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6152435719966889, "sampling/importance_sampling_ratio/max": 1.3606741666793822, "sampling/importance_sampling_ratio/mean": 1.00054789185524, "sampling/importance_sampling_ratio/min": 0.7545235276222229, "sampling/sampling_logp_difference/max": 0.38164920806884767, "sampling/sampling_logp_difference/mean": 0.021344459615647793, "step": 18680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 215.875, "completions/mean_terminated_length": 215.875, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.1400953657925128, "epoch": 1.7183046795991541, "frac_reward_zero_std": 0.4, "grad_norm": 7.485304355621338, "kl": 0.009407825337257236, "learning_rate": 2.10702461660563e-06, "loss": 0.0002, "num_tokens": 81422725.0, "reward": -0.25, "reward_std": 0.5117493987083435, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5117493987083435, "sampling/importance_sampling_ratio/max": 1.2599159240722657, "sampling/importance_sampling_ratio/mean": 0.9993674278259277, "sampling/importance_sampling_ratio/min": 0.6497232854366303, "sampling/sampling_logp_difference/max": 0.4776890933513641, "sampling/sampling_logp_difference/mean": 0.02234691921621561, "step": 18690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 199.2625, "completions/mean_terminated_length": 199.2625, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0623516999185085, "epoch": 1.7192240507492875, "frac_reward_zero_std": 0.5, "grad_norm": 6.698129653930664, "kl": 0.009331289402325638, "learning_rate": 2.104522691766833e-06, "loss": 0.0005, "num_tokens": 81464466.0, "reward": 0.2, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2549283981323243, "sampling/importance_sampling_ratio/mean": 0.9992727994918823, "sampling/importance_sampling_ratio/min": 0.7635851025581359, "sampling/sampling_logp_difference/max": 0.28274967670440676, "sampling/sampling_logp_difference/mean": 0.021854187548160552, "step": 18700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.2, "completions/max_terminated_length": 293.2, "completions/mean_length": 210.175, "completions/mean_terminated_length": 210.175, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 1.1897834636271, "epoch": 1.720143421899421, "frac_reward_zero_std": 0.8, "grad_norm": 0.10032236576080322, "kl": 0.009440573782194405, "learning_rate": 2.1020211731149335e-06, "loss": 0.0005, "num_tokens": 81509512.0, "reward": -0.075, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.2448788642883302, "sampling/importance_sampling_ratio/mean": 1.0000928401947022, "sampling/importance_sampling_ratio/min": 0.7795328140258789, "sampling/sampling_logp_difference/max": 0.25947587490081786, "sampling/sampling_logp_difference/mean": 0.022956511192023753, "step": 18710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.2, "completions/max_terminated_length": 314.2, "completions/mean_length": 206.3375, "completions/mean_terminated_length": 206.3375, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0749994695186615, "epoch": 1.7210627930495541, "frac_reward_zero_std": 0.4, "grad_norm": 0.12599912285804749, "kl": 0.011594225961016492, "learning_rate": 2.0995200632191905e-06, "loss": 0.0008, "num_tokens": 81553611.0, "reward": 0.025, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2395375728607179, "sampling/importance_sampling_ratio/mean": 1.000106018781662, "sampling/importance_sampling_ratio/min": 0.7880165338516235, "sampling/sampling_logp_difference/max": 0.24820319414138795, "sampling/sampling_logp_difference/mean": 0.021062922291457654, "step": 18720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.8, "completions/max_terminated_length": 319.8, "completions/mean_length": 200.075, "completions/mean_terminated_length": 200.075, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0559182420372963, "epoch": 1.7219821641996873, "frac_reward_zero_std": 0.6, "grad_norm": 5.132392883300781, "kl": 0.00991884205432143, "learning_rate": 2.0970193646484456e-06, "loss": 0.0004, "num_tokens": 81597057.0, "reward": -0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.4210909843444823, "sampling/importance_sampling_ratio/mean": 1.0005007266998291, "sampling/importance_sampling_ratio/min": 0.7826883912086486, "sampling/sampling_logp_difference/max": 0.3592599451541901, "sampling/sampling_logp_difference/mean": 0.020877989009022713, "step": 18730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.4, "completions/max_terminated_length": 279.4, "completions/mean_length": 201.15, "completions/mean_terminated_length": 201.15, "completions/min_length": 119.9, "completions/min_terminated_length": 119.9, "entropy": 1.1318767547607422, "epoch": 1.7229015353498207, "frac_reward_zero_std": 0.5, "grad_norm": 0.09339828044176102, "kl": 0.008954423863906413, "learning_rate": 2.094519079971116e-06, "loss": -0.0002, "num_tokens": 81641117.0, "reward": -0.025, "reward_std": 0.5024828433990478, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5024828553199768, "sampling/importance_sampling_ratio/max": 1.2930519223213195, "sampling/importance_sampling_ratio/mean": 0.9999574780464172, "sampling/importance_sampling_ratio/min": 0.7600728750228882, "sampling/sampling_logp_difference/max": 0.32624592781066897, "sampling/sampling_logp_difference/mean": 0.022463308833539485, "step": 18740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.1, "completions/max_length": 321.0, "completions/max_terminated_length": 269.8, "completions/mean_length": 232.375, "completions/mean_terminated_length": 181.175, "completions/min_length": 169.5, "completions/min_terminated_length": 118.3, "entropy": 1.0812872912734748, "epoch": 1.7238209064999541, "frac_reward_zero_std": 0.4, "grad_norm": 6.088545322418213, "kl": 0.010176639704150147, "learning_rate": 2.0920192117551934e-06, "loss": 0.0004, "num_tokens": 81683467.0, "reward": -0.625, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.3227062225341797, "sampling/importance_sampling_ratio/mean": 1.0001849353313446, "sampling/importance_sampling_ratio/min": 0.7437509089708328, "sampling/sampling_logp_difference/max": 0.3462545871734619, "sampling/sampling_logp_difference/mean": 0.02164580291137099, "step": 18750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.1, "completions/max_terminated_length": 297.1, "completions/mean_length": 205.8625, "completions/mean_terminated_length": 205.8625, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0931560218334198, "epoch": 1.7247402776500873, "frac_reward_zero_std": 0.4, "grad_norm": 6.067193984985352, "kl": 0.010337213930324652, "learning_rate": 2.089519762568245e-06, "loss": 0.0004, "num_tokens": 81724040.0, "reward": 0.06875, "reward_std": 0.5182157576084137, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.5182157695293427, "sampling/importance_sampling_ratio/max": 1.2640668153762817, "sampling/importance_sampling_ratio/mean": 0.9995331227779388, "sampling/importance_sampling_ratio/min": 0.7631088376045227, "sampling/sampling_logp_difference/max": 0.2942716360092163, "sampling/sampling_logp_difference/mean": 0.02265171557664871, "step": 18760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 214.125, "completions/mean_terminated_length": 214.125, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0933828435838222, "epoch": 1.7256596488002205, "frac_reward_zero_std": 0.5, "grad_norm": 0.08947087824344635, "kl": 0.009074412018526346, "learning_rate": 2.0870207349774046e-06, "loss": 0.0007, "num_tokens": 81769714.0, "reward": 0.31875, "reward_std": 0.3879900202155113, "rewards/reward_fn/mean": 0.31875, "rewards/reward_fn/std": 0.38799004405736925, "sampling/importance_sampling_ratio/max": 1.281247365474701, "sampling/importance_sampling_ratio/mean": 0.9997156620025635, "sampling/importance_sampling_ratio/min": 0.7613817811012268, "sampling/sampling_logp_difference/max": 0.3118799328804016, "sampling/sampling_logp_difference/mean": 0.0214183509349823, "step": 18770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 204.9125, "completions/mean_terminated_length": 204.9125, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0659455873072148, "epoch": 1.726579019950354, "frac_reward_zero_std": 0.7, "grad_norm": 0.09879468381404877, "kl": 0.009034479581168852, "learning_rate": 2.0845221315493737e-06, "loss": 0.0004, "num_tokens": 81812723.0, "reward": 0.15, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.289888072013855, "sampling/importance_sampling_ratio/mean": 0.9998917877674103, "sampling/importance_sampling_ratio/min": 0.7892951548099518, "sampling/sampling_logp_difference/max": 0.2754847645759583, "sampling/sampling_logp_difference/mean": 0.021890506707131864, "step": 18780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 313.3, "completions/max_terminated_length": 296.2, "completions/mean_length": 204.4125, "completions/mean_terminated_length": 201.0875015258789, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0950109012424947, "epoch": 1.7274983911004873, "frac_reward_zero_std": 0.8, "grad_norm": 4.727851390838623, "kl": 0.00899333702691365, "learning_rate": 2.0820239548504177e-06, "loss": 0.0005, "num_tokens": 81854292.0, "reward": 0.11875, "reward_std": 0.18097035884857177, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.18097036480903625, "sampling/importance_sampling_ratio/max": 1.2229734539985657, "sampling/importance_sampling_ratio/mean": 1.0000188946723938, "sampling/importance_sampling_ratio/min": 0.7721309840679169, "sampling/sampling_logp_difference/max": 0.26233933568000795, "sampling/sampling_logp_difference/mean": 0.02212198469787836, "step": 18790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.8, "completions/max_terminated_length": 307.8, "completions/mean_length": 222.9, "completions/mean_terminated_length": 222.9, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0778833515942097, "epoch": 1.7284177622506207, "frac_reward_zero_std": 0.7, "grad_norm": 5.206390380859375, "kl": 0.011719794300734065, "learning_rate": 2.0795262074463652e-06, "loss": 0.0006, "num_tokens": 81898308.0, "reward": -0.15, "reward_std": 0.2777460336685181, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.2777460336685181, "sampling/importance_sampling_ratio/max": 1.2484010815620423, "sampling/importance_sampling_ratio/mean": 0.9996367990970612, "sampling/importance_sampling_ratio/min": 0.7133231297135353, "sampling/sampling_logp_difference/max": 0.39109493494033815, "sampling/sampling_logp_difference/mean": 0.021394644677639008, "step": 18800 }, { "epoch": 1.7284177622506207, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 307.4589877835951, "eval_completions/max_terminated_length": 303.9912739965096, "eval_completions/mean_length": 213.42670157068062, "eval_completions/mean_terminated_length": 212.69321879708122, "eval_completions/min_length": 138.36649214659687, "eval_completions/min_terminated_length": 138.36649214659687, "eval_entropy": 1.0984780119559736, "eval_frac_reward_zero_std": 0.4956369982547993, "eval_kl": 0.009275672119435892, "eval_loss": 0.0004611547337844968, "eval_num_tokens": 81898308.0, "eval_reward": -0.11125654450261781, "eval_reward_std": 0.44425571934819014, "eval_rewards/reward_fn/mean": -0.11125654450261781, "eval_rewards/reward_fn/std": 0.4442557304785514, "eval_runtime": 1781.9305, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.2688704325474578, "eval_sampling/importance_sampling_ratio/mean": 0.9999928707435701, "eval_sampling/importance_sampling_ratio/min": 0.7672791982144378, "eval_sampling/sampling_logp_difference/max": 0.2938902152874499, "eval_sampling/sampling_logp_difference/mean": 0.02131398636067569, "eval_steps_per_second": 0.04, "step": 18800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 313.1, "completions/max_terminated_length": 297.7, "completions/mean_length": 207.3625, "completions/mean_terminated_length": 204.13750152587892, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.0914483413100242, "epoch": 1.729337133400754, "frac_reward_zero_std": 0.6, "grad_norm": 0.2500927746295929, "kl": 0.010640346456784754, "learning_rate": 2.077028891902602e-06, "loss": 0.0005, "num_tokens": 81940953.0, "reward": 0.01875, "reward_std": 0.3583177328109741, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.3583177328109741, "sampling/importance_sampling_ratio/max": 1.283879315853119, "sampling/importance_sampling_ratio/mean": 1.0008033156394958, "sampling/importance_sampling_ratio/min": 0.772942841053009, "sampling/sampling_logp_difference/max": 0.2829241454601288, "sampling/sampling_logp_difference/mean": 0.022372497990727425, "step": 18810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.8, "completions/max_terminated_length": 304.8, "completions/mean_length": 211.2625, "completions/mean_terminated_length": 211.2625, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.1081389717757701, "epoch": 1.730256504550887, "frac_reward_zero_std": 0.5, "grad_norm": 0.08635537326335907, "kl": 0.010831702360883355, "learning_rate": 2.0745320107840743e-06, "loss": 0.0005, "num_tokens": 81985182.0, "reward": -0.325, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2719999074935913, "sampling/importance_sampling_ratio/mean": 1.0002923846244811, "sampling/importance_sampling_ratio/min": 0.7319398164749146, "sampling/sampling_logp_difference/max": 0.3236455678939819, "sampling/sampling_logp_difference/mean": 0.022106329351663588, "step": 18820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 295.5, "completions/max_terminated_length": 281.3, "completions/mean_length": 202.8625, "completions/mean_terminated_length": 199.46607208251953, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.0377251081168652, "epoch": 1.7311758757010205, "frac_reward_zero_std": 0.5, "grad_norm": 4.168562889099121, "kl": 0.008415885746944695, "learning_rate": 2.072035566655275e-06, "loss": 0.0003, "num_tokens": 82027475.0, "reward": 0.16875, "reward_std": 0.37123105525970457, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.3712310612201691, "sampling/importance_sampling_ratio/max": 1.271875512599945, "sampling/importance_sampling_ratio/mean": 1.000210964679718, "sampling/importance_sampling_ratio/min": 0.7835461676120759, "sampling/sampling_logp_difference/max": 0.2607477426528931, "sampling/sampling_logp_difference/mean": 0.02085668481886387, "step": 18830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 214.3375, "completions/mean_terminated_length": 214.3375, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.0273145683109761, "epoch": 1.732095246851154, "frac_reward_zero_std": 0.4, "grad_norm": 4.753859043121338, "kl": 0.009657797592808492, "learning_rate": 2.069539562080255e-06, "loss": 0.0009, "num_tokens": 82069702.0, "reward": 0.075, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.233411705493927, "sampling/importance_sampling_ratio/mean": 0.9996070444583893, "sampling/importance_sampling_ratio/min": 0.7197293728590012, "sampling/sampling_logp_difference/max": 0.34813770055770876, "sampling/sampling_logp_difference/mean": 0.020931472443044184, "step": 18840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.9, "completions/max_terminated_length": 320.9, "completions/mean_length": 201.4125, "completions/mean_terminated_length": 201.4125, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.0178004756569863, "epoch": 1.733014618001287, "frac_reward_zero_std": 0.7, "grad_norm": 0.08869250118732452, "kl": 0.00869817191560287, "learning_rate": 2.0670439996226098e-06, "loss": 0.0005, "num_tokens": 82113471.0, "reward": 0.45, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2526474356651307, "sampling/importance_sampling_ratio/mean": 1.0002776622772216, "sampling/importance_sampling_ratio/min": 0.784420657157898, "sampling/sampling_logp_difference/max": 0.26624908447265627, "sampling/sampling_logp_difference/mean": 0.021036745980381966, "step": 18850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 223.125, "completions/mean_terminated_length": 223.125, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.0226757667958737, "epoch": 1.7339339891514203, "frac_reward_zero_std": 0.3, "grad_norm": 4.533810615539551, "kl": 0.009535423587658442, "learning_rate": 2.064548881845484e-06, "loss": 0.0005, "num_tokens": 82155505.0, "reward": -0.1, "reward_std": 0.6917853593826294, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6917854070663452, "sampling/importance_sampling_ratio/max": 1.3278433799743652, "sampling/importance_sampling_ratio/mean": 1.0002246260643006, "sampling/importance_sampling_ratio/min": 0.7754889130592346, "sampling/sampling_logp_difference/max": 0.29390551447868346, "sampling/sampling_logp_difference/mean": 0.021586837805807592, "step": 18860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.4, "completions/max_terminated_length": 265.4, "completions/mean_length": 187.6625, "completions/mean_terminated_length": 187.6625, "completions/min_length": 127.5, "completions/min_terminated_length": 127.5, "entropy": 1.0287713006138801, "epoch": 1.7348533603015537, "frac_reward_zero_std": 0.5, "grad_norm": 6.766656398773193, "kl": 0.009529035276500508, "learning_rate": 2.062054211311561e-06, "loss": 0.0005, "num_tokens": 82194430.0, "reward": 0.29375, "reward_std": 0.3878512144088745, "rewards/reward_fn/mean": 0.29375, "rewards/reward_fn/std": 0.3878512144088745, "sampling/importance_sampling_ratio/max": 1.249627459049225, "sampling/importance_sampling_ratio/mean": 0.9999645054340363, "sampling/importance_sampling_ratio/min": 0.7447542071342468, "sampling/sampling_logp_difference/max": 0.32248747646808623, "sampling/sampling_logp_difference/mean": 0.021615232713520525, "step": 18870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 330.8, "completions/max_terminated_length": 330.8, "completions/mean_length": 230.375, "completions/mean_terminated_length": 230.375, "completions/min_length": 157.9, "completions/min_terminated_length": 157.9, "entropy": 1.115916434675455, "epoch": 1.735772731451687, "frac_reward_zero_std": 0.5, "grad_norm": 6.277275562286377, "kl": 0.009385678597027436, "learning_rate": 2.059559990583069e-06, "loss": 0.0007, "num_tokens": 82239484.0, "reward": -0.075, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.3017681241035461, "sampling/importance_sampling_ratio/mean": 0.9998137891292572, "sampling/importance_sampling_ratio/min": 0.7753528356552124, "sampling/sampling_logp_difference/max": 0.30694098472595216, "sampling/sampling_logp_difference/mean": 0.0218438521027565, "step": 18880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.8, "completions/max_terminated_length": 264.8, "completions/mean_length": 196.175, "completions/mean_terminated_length": 196.175, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0341955162584782, "epoch": 1.7366921026018205, "frac_reward_zero_std": 0.3, "grad_norm": 6.37503719329834, "kl": 0.008841252926504239, "learning_rate": 2.0570662222217717e-06, "loss": 0.0004, "num_tokens": 82280834.0, "reward": -0.05, "reward_std": 0.5496452271938324, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5496452391147614, "sampling/importance_sampling_ratio/max": 1.2756008505821228, "sampling/importance_sampling_ratio/mean": 1.0002501606941223, "sampling/importance_sampling_ratio/min": 0.7971882402896882, "sampling/sampling_logp_difference/max": 0.24924442768096924, "sampling/sampling_logp_difference/mean": 0.02073884680867195, "step": 18890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 201.7375, "completions/mean_terminated_length": 201.7375, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.1049373790621757, "epoch": 1.7376114737519537, "frac_reward_zero_std": 0.3, "grad_norm": 4.164557933807373, "kl": 0.010155504607246258, "learning_rate": 2.0545729087889703e-06, "loss": 0.0003, "num_tokens": 82321341.0, "reward": -0.25, "reward_std": 0.6220328629016876, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.6220328748226166, "sampling/importance_sampling_ratio/max": 1.2497952580451965, "sampling/importance_sampling_ratio/mean": 1.0007186830043793, "sampling/importance_sampling_ratio/min": 0.8092712938785553, "sampling/sampling_logp_difference/max": 0.23092751502990722, "sampling/sampling_logp_difference/mean": 0.02201874889433384, "step": 18900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 208.25, "completions/mean_terminated_length": 208.25, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0989752635359764, "epoch": 1.7385308449020869, "frac_reward_zero_std": 0.4, "grad_norm": 7.599300861358643, "kl": 0.008500971214380116, "learning_rate": 2.052080052845497e-06, "loss": 0.0005, "num_tokens": 82367337.0, "reward": 0.275, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.3310053944587708, "sampling/importance_sampling_ratio/mean": 1.000033789873123, "sampling/importance_sampling_ratio/min": 0.7870433509349823, "sampling/sampling_logp_difference/max": 0.3013129949569702, "sampling/sampling_logp_difference/mean": 0.02180729303508997, "step": 18910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 340.0, "completions/max_terminated_length": 337.3, "completions/mean_length": 221.475, "completions/mean_terminated_length": 218.5375, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0135517694056033, "epoch": 1.7394502160522203, "frac_reward_zero_std": 0.4, "grad_norm": 0.09419547021389008, "kl": 0.010455350420670584, "learning_rate": 2.0495876569517147e-06, "loss": 0.0006, "num_tokens": 82412751.0, "reward": -0.00625, "reward_std": 0.5544095754623413, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.5544095873832703, "sampling/importance_sampling_ratio/max": 1.279429817199707, "sampling/importance_sampling_ratio/mean": 0.999454116821289, "sampling/importance_sampling_ratio/min": 0.7643988251686096, "sampling/sampling_logp_difference/max": 0.3027299404144287, "sampling/sampling_logp_difference/mean": 0.02064895536750555, "step": 18920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.1, "completions/max_terminated_length": 309.1, "completions/mean_length": 215.4, "completions/mean_terminated_length": 215.4, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.036310261487961, "epoch": 1.7403695872023537, "frac_reward_zero_std": 0.5, "grad_norm": 5.644139289855957, "kl": 0.008408059642533772, "learning_rate": 2.0470957236675133e-06, "loss": 0.0005, "num_tokens": 82454503.0, "reward": -0.3, "reward_std": 0.4847656965255737, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.48476572036743165, "sampling/importance_sampling_ratio/max": 1.2666004180908204, "sampling/importance_sampling_ratio/mean": 1.0000992238521575, "sampling/importance_sampling_ratio/min": 0.7831731855869293, "sampling/sampling_logp_difference/max": 0.2750298261642456, "sampling/sampling_logp_difference/mean": 0.02054826058447361, "step": 18930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.1, "completions/max_terminated_length": 288.1, "completions/mean_length": 198.075, "completions/mean_terminated_length": 198.075, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 0.9923021696507931, "epoch": 1.7412889583524869, "frac_reward_zero_std": 0.6, "grad_norm": 5.96364688873291, "kl": 0.009016617565066554, "learning_rate": 2.0446042555523084e-06, "loss": 0.0004, "num_tokens": 82497365.0, "reward": 0.125, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2496931195259093, "sampling/importance_sampling_ratio/mean": 1.000329428911209, "sampling/importance_sampling_ratio/min": 0.7930532276630402, "sampling/sampling_logp_difference/max": 0.25195870399475095, "sampling/sampling_logp_difference/mean": 0.02023693025112152, "step": 18940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.4, "completions/max_terminated_length": 282.4, "completions/mean_length": 205.65, "completions/mean_terminated_length": 205.65, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0740924127399922, "epoch": 1.74220832950262, "frac_reward_zero_std": 0.6, "grad_norm": 5.75060510635376, "kl": 0.012308613007189707, "learning_rate": 2.042113255165038e-06, "loss": 0.0004, "num_tokens": 82540393.0, "reward": 0.0, "reward_std": 0.3593845307826996, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3593845427036285, "sampling/importance_sampling_ratio/max": 1.2702736258506775, "sampling/importance_sampling_ratio/mean": 1.0004222214221954, "sampling/importance_sampling_ratio/min": 0.7514060020446778, "sampling/sampling_logp_difference/max": 0.31293894052505494, "sampling/sampling_logp_difference/mean": 0.0214857691898942, "step": 18950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.3, "completions/max_terminated_length": 332.3, "completions/mean_length": 216.425, "completions/mean_terminated_length": 216.425, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.1643930472433568, "epoch": 1.7431277006527535, "frac_reward_zero_std": 0.5, "grad_norm": 0.0886620357632637, "kl": 0.010635931379511021, "learning_rate": 2.039622725064158e-06, "loss": 0.0009, "num_tokens": 82586947.0, "reward": -0.175, "reward_std": 0.4738378763198853, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4738378882408142, "sampling/importance_sampling_ratio/max": 1.3030524492263793, "sampling/importance_sampling_ratio/mean": 0.9995925903320313, "sampling/importance_sampling_ratio/min": 0.7440625190734863, "sampling/sampling_logp_difference/max": 0.3167165756225586, "sampling/sampling_logp_difference/mean": 0.022620122134685516, "step": 18960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 198.7375, "completions/mean_terminated_length": 198.7375, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0139654085040093, "epoch": 1.7440470718028869, "frac_reward_zero_std": 0.5, "grad_norm": 0.06901269406080246, "kl": 0.008148305918439291, "learning_rate": 2.037132667807643e-06, "loss": 0.0003, "num_tokens": 82626750.0, "reward": -0.475, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2748358368873596, "sampling/importance_sampling_ratio/mean": 1.0000792324543, "sampling/importance_sampling_ratio/min": 0.7664789319038391, "sampling/sampling_logp_difference/max": 0.278423935174942, "sampling/sampling_logp_difference/mean": 0.02107358444482088, "step": 18970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.3, "completions/max_terminated_length": 282.3, "completions/mean_length": 198.7875, "completions/mean_terminated_length": 198.7875, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.0095680877566338, "epoch": 1.7449664429530203, "frac_reward_zero_std": 0.4, "grad_norm": 9.85877799987793, "kl": 0.008983468802762217, "learning_rate": 2.034643085952982e-06, "loss": 0.0002, "num_tokens": 82667477.0, "reward": -0.025, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2627151131629943, "sampling/importance_sampling_ratio/mean": 0.9996314883232117, "sampling/importance_sampling_ratio/min": 0.7833971917629242, "sampling/sampling_logp_difference/max": 0.270917272567749, "sampling/sampling_logp_difference/mean": 0.021173138171434402, "step": 18980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.9, "completions/max_terminated_length": 320.9, "completions/mean_length": 220.6625, "completions/mean_terminated_length": 220.6625, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0582349359989167, "epoch": 1.7458858141031535, "frac_reward_zero_std": 0.4, "grad_norm": 4.467195987701416, "kl": 0.008788601867854596, "learning_rate": 2.0321539820571747e-06, "loss": 0.0005, "num_tokens": 82709098.0, "reward": 0.3, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2404918193817138, "sampling/importance_sampling_ratio/mean": 1.0003599226474762, "sampling/importance_sampling_ratio/min": 0.7819266796112061, "sampling/sampling_logp_difference/max": 0.25278412103652953, "sampling/sampling_logp_difference/mean": 0.021457366086542605, "step": 18990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 206.275, "completions/mean_terminated_length": 206.275, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.0926044307649136, "epoch": 1.7468051852532867, "frac_reward_zero_std": 0.3, "grad_norm": 6.382081985473633, "kl": 0.0121450899168849, "learning_rate": 2.0296653586767303e-06, "loss": 0.0011, "num_tokens": 82752272.0, "reward": 0.3, "reward_std": 0.5824443817138671, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.5824444055557251, "sampling/importance_sampling_ratio/max": 1.2831544876098633, "sampling/importance_sampling_ratio/mean": 1.0001841247081757, "sampling/importance_sampling_ratio/min": 0.7606160998344421, "sampling/sampling_logp_difference/max": 0.2963176488876343, "sampling/sampling_logp_difference/mean": 0.021387429535388948, "step": 19000 }, { "epoch": 1.7468051852532867, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 298.5759162303665, "eval_completions/max_terminated_length": 296.4973821989529, "eval_completions/mean_length": 209.21291448516578, "eval_completions/mean_terminated_length": 208.7006359266777, "eval_completions/min_length": 136.12041884816753, "eval_completions/min_terminated_length": 136.12041884816753, "eval_entropy": 1.0961008262779908, "eval_frac_reward_zero_std": 0.493891797556719, "eval_kl": 0.00949369358259658, "eval_loss": 0.00047756629646755755, "eval_num_tokens": 82752272.0, "eval_reward": -0.10329406631762653, "eval_reward_std": 0.44847228082894863, "eval_rewards/reward_fn/mean": -0.10329406631762653, "eval_rewards/reward_fn/std": 0.44847229268746436, "eval_runtime": 1736.6189, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2693792090574068, "eval_sampling/importance_sampling_ratio/mean": 0.9999298785161389, "eval_sampling/importance_sampling_ratio/min": 0.7750078870363885, "eval_sampling/sampling_logp_difference/max": 0.2862464564528973, "eval_sampling/sampling_logp_difference/mean": 0.02085953112691641, "eval_steps_per_second": 0.041, "step": 19000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.4, "completions/max_terminated_length": 326.4, "completions/mean_length": 231.275, "completions/mean_terminated_length": 231.275, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.1031394228339195, "epoch": 1.74772455640342, "frac_reward_zero_std": 0.3, "grad_norm": 0.09515127539634705, "kl": 0.009637376596219837, "learning_rate": 2.027177218367664e-06, "loss": 0.0006, "num_tokens": 82802422.0, "reward": -0.05, "reward_std": 0.6800980150699616, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.6800980269908905, "sampling/importance_sampling_ratio/max": 1.2194303512573241, "sampling/importance_sampling_ratio/mean": 1.00009628534317, "sampling/importance_sampling_ratio/min": 0.7632496118545532, "sampling/sampling_logp_difference/max": 0.27411718368530275, "sampling/sampling_logp_difference/mean": 0.02148034293204546, "step": 19010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.3, "completions/max_terminated_length": 274.3, "completions/mean_length": 190.3125, "completions/mean_terminated_length": 190.3125, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.003994908183813, "epoch": 1.7486439275535535, "frac_reward_zero_std": 0.5, "grad_norm": 5.723337650299072, "kl": 0.009419036426697858, "learning_rate": 2.0246895636854964e-06, "loss": 0.0004, "num_tokens": 82840935.0, "reward": -0.45, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2426066637039184, "sampling/importance_sampling_ratio/mean": 0.9997664034366608, "sampling/importance_sampling_ratio/min": 0.781211543083191, "sampling/sampling_logp_difference/max": 0.26344850063323977, "sampling/sampling_logp_difference/mean": 0.020236152596771716, "step": 19020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 212.8, "completions/mean_terminated_length": 212.8, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.1159263633191585, "epoch": 1.7495632987036867, "frac_reward_zero_std": 0.8, "grad_norm": 0.08199475705623627, "kl": 0.009813789933105, "learning_rate": 2.0222023971852465e-06, "loss": 0.0005, "num_tokens": 82884543.0, "reward": -0.275, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.3819616198539735, "sampling/importance_sampling_ratio/mean": 1.0002470195293427, "sampling/importance_sampling_ratio/min": 0.799774843454361, "sampling/sampling_logp_difference/max": 0.3240975499153137, "sampling/sampling_logp_difference/mean": 0.021972007863223553, "step": 19030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.2, "completions/max_terminated_length": 329.2, "completions/mean_length": 209.0875, "completions/mean_terminated_length": 209.0875, "completions/min_length": 127.3, "completions/min_terminated_length": 127.3, "entropy": 1.1378939114511013, "epoch": 1.7504826698538198, "frac_reward_zero_std": 0.5, "grad_norm": 8.132203102111816, "kl": 0.010827100177993997, "learning_rate": 2.019715721421437e-06, "loss": 0.0008, "num_tokens": 82931078.0, "reward": -0.3, "reward_std": 0.41916738748550414, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.41916738748550414, "sampling/importance_sampling_ratio/max": 1.2263607621192931, "sampling/importance_sampling_ratio/mean": 0.9995095133781433, "sampling/importance_sampling_ratio/min": 0.7568981885910034, "sampling/sampling_logp_difference/max": 0.28138374984264375, "sampling/sampling_logp_difference/mean": 0.022066918015480042, "step": 19040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 204.1, "completions/mean_terminated_length": 204.1, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0365931488573552, "epoch": 1.7514020410039532, "frac_reward_zero_std": 0.5, "grad_norm": 5.0648345947265625, "kl": 0.009374639665475115, "learning_rate": 2.0172295389480795e-06, "loss": 0.0003, "num_tokens": 82972454.0, "reward": -0.15, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2937631011009216, "sampling/importance_sampling_ratio/mean": 1.000127637386322, "sampling/importance_sampling_ratio/min": 0.7422723591327667, "sampling/sampling_logp_difference/max": 0.3292619585990906, "sampling/sampling_logp_difference/mean": 0.02112354338169098, "step": 19050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 218.25, "completions/mean_terminated_length": 218.25, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.0654959715902805, "epoch": 1.7523214121540867, "frac_reward_zero_std": 0.3, "grad_norm": 5.839908599853516, "kl": 0.009393475219258107, "learning_rate": 2.0147438523186845e-06, "loss": 0.0005, "num_tokens": 83013706.0, "reward": -0.125, "reward_std": 0.5824600756168365, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5824600756168365, "sampling/importance_sampling_ratio/max": 1.2577020168304442, "sampling/importance_sampling_ratio/mean": 0.9993900120258331, "sampling/importance_sampling_ratio/min": 0.7848693072795868, "sampling/sampling_logp_difference/max": 0.26383382081985474, "sampling/sampling_logp_difference/mean": 0.02026612013578415, "step": 19060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.8, "completions/max_terminated_length": 304.8, "completions/mean_length": 219.9, "completions/mean_terminated_length": 219.9, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.1327429980039596, "epoch": 1.75324078330422, "frac_reward_zero_std": 0.5, "grad_norm": 4.8145270347595215, "kl": 0.009555584244662896, "learning_rate": 2.0122586640862514e-06, "loss": 0.0006, "num_tokens": 83059018.0, "reward": -0.325, "reward_std": 0.45536120533943175, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.45536120533943175, "sampling/importance_sampling_ratio/max": 1.30152428150177, "sampling/importance_sampling_ratio/mean": 0.9993757843971253, "sampling/importance_sampling_ratio/min": 0.7888829171657562, "sampling/sampling_logp_difference/max": 0.2718285083770752, "sampling/sampling_logp_difference/mean": 0.021941224113106728, "step": 19070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 190.5875, "completions/mean_terminated_length": 190.5875, "completions/min_length": 118.6, "completions/min_terminated_length": 118.6, "entropy": 1.1105473078787327, "epoch": 1.7541601544543532, "frac_reward_zero_std": 0.5, "grad_norm": 0.10846243798732758, "kl": 0.01048773891525343, "learning_rate": 2.009773976803268e-06, "loss": 0.0006, "num_tokens": 83099945.0, "reward": 0.275, "reward_std": 0.4410387217998505, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.4410387217998505, "sampling/importance_sampling_ratio/max": 1.2367001056671143, "sampling/importance_sampling_ratio/mean": 0.9994556307792664, "sampling/importance_sampling_ratio/min": 0.7466389179229737, "sampling/sampling_logp_difference/max": 0.3075710594654083, "sampling/sampling_logp_difference/mean": 0.02169880736619234, "step": 19080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.4, "completions/max_terminated_length": 333.4, "completions/mean_length": 229.8375, "completions/mean_terminated_length": 229.8375, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.1101336538791657, "epoch": 1.7550795256044864, "frac_reward_zero_std": 0.6, "grad_norm": 0.09990604221820831, "kl": 0.00907126116217114, "learning_rate": 2.007289793021706e-06, "loss": 0.0004, "num_tokens": 83144644.0, "reward": -0.525, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.3023305773735045, "sampling/importance_sampling_ratio/mean": 0.9998796820640564, "sampling/importance_sampling_ratio/min": 0.7425808429718017, "sampling/sampling_logp_difference/max": 0.3221469044685364, "sampling/sampling_logp_difference/mean": 0.02115669995546341, "step": 19090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 283.8, "completions/max_terminated_length": 276.3, "completions/mean_length": 204.2, "completions/mean_terminated_length": 201.0232147216797, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.02086336389184, "epoch": 1.7559988967546198, "frac_reward_zero_std": 0.2, "grad_norm": 5.554154396057129, "kl": 0.01001971278747078, "learning_rate": 2.0048061152930227e-06, "loss": 0.0003, "num_tokens": 83185780.0, "reward": 0.24375, "reward_std": 0.6972034454345704, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.6972034692764282, "sampling/importance_sampling_ratio/max": 1.2408589243888855, "sampling/importance_sampling_ratio/mean": 1.0001751601696014, "sampling/importance_sampling_ratio/min": 0.7908605217933655, "sampling/sampling_logp_difference/max": 0.25444291830062865, "sampling/sampling_logp_difference/mean": 0.02098406944423914, "step": 19100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 199.25, "completions/mean_terminated_length": 199.25, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0266061305999756, "epoch": 1.7569182679047533, "frac_reward_zero_std": 0.7, "grad_norm": 0.17616645991802216, "kl": 0.012256631543277764, "learning_rate": 2.0023229461681524e-06, "loss": 0.0006, "num_tokens": 83228848.0, "reward": 0.575, "reward_std": 0.21213203072547912, "rewards/reward_fn/mean": 0.575, "rewards/reward_fn/std": 0.21213203072547912, "sampling/importance_sampling_ratio/max": 1.2324702858924865, "sampling/importance_sampling_ratio/mean": 1.0000720858573913, "sampling/importance_sampling_ratio/min": 0.7809971511363983, "sampling/sampling_logp_difference/max": 0.2543911933898926, "sampling/sampling_logp_difference/mean": 0.020775367878377436, "step": 19110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 297.2, "completions/max_terminated_length": 282.5, "completions/mean_length": 205.6875, "completions/mean_terminated_length": 202.825, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.10525618866086, "epoch": 1.7578376390548864, "frac_reward_zero_std": 0.4, "grad_norm": 5.313398361206055, "kl": 0.009989375242730603, "learning_rate": 1.999840288197511e-06, "loss": 0.0005, "num_tokens": 83271863.0, "reward": 0.11875, "reward_std": 0.4805720314383507, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.4805720552802086, "sampling/importance_sampling_ratio/max": 1.2580427765846252, "sampling/importance_sampling_ratio/mean": 1.0000350415706634, "sampling/importance_sampling_ratio/min": 0.7703831911087036, "sampling/sampling_logp_difference/max": 0.281741738319397, "sampling/sampling_logp_difference/mean": 0.021735955961048603, "step": 19120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.3, "completions/max_terminated_length": 273.3, "completions/mean_length": 195.3375, "completions/mean_terminated_length": 195.3375, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0647993385791779, "epoch": 1.7587570102050196, "frac_reward_zero_std": 0.8, "grad_norm": 0.09165676683187485, "kl": 0.008687788661336526, "learning_rate": 1.997358143930987e-06, "loss": 0.0004, "num_tokens": 83312970.0, "reward": -0.075, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.2358355283737184, "sampling/importance_sampling_ratio/mean": 1.0001867353916167, "sampling/importance_sampling_ratio/min": 0.7676097512245178, "sampling/sampling_logp_difference/max": 0.2824359893798828, "sampling/sampling_logp_difference/mean": 0.020770317502319814, "step": 19130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 201.6375, "completions/mean_terminated_length": 201.6375, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 1.1629041016101838, "epoch": 1.759676381355153, "frac_reward_zero_std": 0.5, "grad_norm": 0.0983375608921051, "kl": 0.008995145664084703, "learning_rate": 1.9948765159179404e-06, "loss": 0.0003, "num_tokens": 83359837.0, "reward": -0.725, "reward_std": 0.42594102025032043, "rewards/reward_fn/mean": -0.725, "rewards/reward_fn/std": 0.42594102025032043, "sampling/importance_sampling_ratio/max": 1.31246178150177, "sampling/importance_sampling_ratio/mean": 0.9996237576007843, "sampling/importance_sampling_ratio/min": 0.792204350233078, "sampling/sampling_logp_difference/max": 0.3009119063615799, "sampling/sampling_logp_difference/mean": 0.0220351941883564, "step": 19140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 229.7375, "completions/mean_terminated_length": 229.7375, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.057513266056776, "epoch": 1.7605957525052864, "frac_reward_zero_std": 0.5, "grad_norm": 6.235275745391846, "kl": 0.008745724201435223, "learning_rate": 1.9923954067072025e-06, "loss": 0.0001, "num_tokens": 83404688.0, "reward": -0.00625, "reward_std": 0.42418383806943893, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.42418386191129687, "sampling/importance_sampling_ratio/max": 1.3304612159729003, "sampling/importance_sampling_ratio/mean": 1.000092738866806, "sampling/importance_sampling_ratio/min": 0.7945016980171203, "sampling/sampling_logp_difference/max": 0.2980660438537598, "sampling/sampling_logp_difference/mean": 0.020918837003409862, "step": 19150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.9, "completions/max_terminated_length": 314.9, "completions/mean_length": 213.8625, "completions/mean_terminated_length": 213.8625, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.0570309557020665, "epoch": 1.7615151236554198, "frac_reward_zero_std": 0.4, "grad_norm": 0.06569840013980865, "kl": 0.008637670709867961, "learning_rate": 1.9899148188470706e-06, "loss": 0.0002, "num_tokens": 83448213.0, "reward": -0.225, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.2833052396774292, "sampling/importance_sampling_ratio/mean": 0.9998652458190918, "sampling/importance_sampling_ratio/min": 0.7692978620529175, "sampling/sampling_logp_difference/max": 0.29422727227211, "sampling/sampling_logp_difference/mean": 0.0209282286465168, "step": 19160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 333.7, "completions/max_terminated_length": 307.0, "completions/mean_length": 209.375, "completions/mean_terminated_length": 205.38750152587892, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0574724890291691, "epoch": 1.762434494805553, "frac_reward_zero_std": 0.4, "grad_norm": 6.326870918273926, "kl": 0.008764446500572376, "learning_rate": 1.98743475488531e-06, "loss": 0.0003, "num_tokens": 83490691.0, "reward": -0.13125, "reward_std": 0.4292962059378624, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.4292962059378624, "sampling/importance_sampling_ratio/max": 1.2545127272605896, "sampling/importance_sampling_ratio/mean": 1.0001750528812408, "sampling/importance_sampling_ratio/min": 0.7635991215705872, "sampling/sampling_logp_difference/max": 0.2962791919708252, "sampling/sampling_logp_difference/mean": 0.020384109392762185, "step": 19170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 313.3, "completions/max_terminated_length": 303.8, "completions/mean_length": 227.2375, "completions/mean_terminated_length": 224.26428833007813, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.047320181876421, "epoch": 1.7633538659556862, "frac_reward_zero_std": 0.4, "grad_norm": 0.07622069120407104, "kl": 0.010473327746149152, "learning_rate": 1.9849552173691417e-06, "loss": 0.0004, "num_tokens": 83533014.0, "reward": -0.10625, "reward_std": 0.538993501663208, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.5389935314655304, "sampling/importance_sampling_ratio/max": 1.245521330833435, "sampling/importance_sampling_ratio/mean": 0.9998294532299041, "sampling/importance_sampling_ratio/min": 0.6802623450756073, "sampling/sampling_logp_difference/max": 0.4109098851680756, "sampling/sampling_logp_difference/mean": 0.021388481371104717, "step": 19180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 208.7625, "completions/mean_terminated_length": 208.7625, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0325256675481795, "epoch": 1.7642732371058196, "frac_reward_zero_std": 0.5, "grad_norm": 6.002710819244385, "kl": 0.009672481045708992, "learning_rate": 1.982476208845251e-06, "loss": 0.0003, "num_tokens": 83575235.0, "reward": 0.24375, "reward_std": 0.4147059261798859, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.4147059261798859, "sampling/importance_sampling_ratio/max": 1.2978639960289002, "sampling/importance_sampling_ratio/mean": 0.9999656915664673, "sampling/importance_sampling_ratio/min": 0.7828871786594391, "sampling/sampling_logp_difference/max": 0.28519043922424314, "sampling/sampling_logp_difference/mean": 0.02082214392721653, "step": 19190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 208.4375, "completions/mean_terminated_length": 208.4375, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.1059572532773019, "epoch": 1.765192608255953, "frac_reward_zero_std": 0.3, "grad_norm": 6.278011322021484, "kl": 0.009876934843487107, "learning_rate": 1.979997731859779e-06, "loss": 0.0001, "num_tokens": 83621110.0, "reward": -0.35, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.2396753191947938, "sampling/importance_sampling_ratio/mean": 0.9996921598911286, "sampling/importance_sampling_ratio/min": 0.7476338922977448, "sampling/sampling_logp_difference/max": 0.297323477268219, "sampling/sampling_logp_difference/mean": 0.02172122560441494, "step": 19200 }, { "epoch": 1.765192608255953, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 295.3228621291448, "eval_completions/max_terminated_length": 293.2425828970332, "eval_completions/mean_length": 207.15750436300175, "eval_completions/mean_terminated_length": 206.59866631259885, "eval_completions/min_length": 135.09075043630017, "eval_completions/min_terminated_length": 135.09075043630017, "eval_entropy": 1.0895856033757094, "eval_frac_reward_zero_std": 0.5095986038394416, "eval_kl": 0.009214018378081694, "eval_loss": 0.0004435846640262753, "eval_num_tokens": 83621110.0, "eval_reward": -0.11147469458987784, "eval_reward_std": 0.43162742798032977, "eval_rewards/reward_fn/mean": -0.11147469458987784, "eval_rewards/reward_fn/std": 0.4316274392147131, "eval_runtime": 1719.636, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2676921017715028, "eval_sampling/importance_sampling_ratio/mean": 1.0000146241504277, "eval_sampling/importance_sampling_ratio/min": 0.7769861495931735, "eval_sampling/sampling_logp_difference/max": 0.2824239655418962, "eval_sampling/sampling_logp_difference/mean": 0.020719923322815546, "eval_steps_per_second": 0.042, "step": 19200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 203.4, "completions/mean_terminated_length": 203.4, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0494659289717674, "epoch": 1.7661119794060862, "frac_reward_zero_std": 0.3, "grad_norm": 3.643885850906372, "kl": 0.011062143056187778, "learning_rate": 1.977519788958319e-06, "loss": 0.0005, "num_tokens": 83661222.0, "reward": -0.625, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.239697277545929, "sampling/importance_sampling_ratio/mean": 1.0001967310905457, "sampling/importance_sampling_ratio/min": 0.7623812675476074, "sampling/sampling_logp_difference/max": 0.28430125713348386, "sampling/sampling_logp_difference/mean": 0.02066135164350271, "step": 19210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 291.2, "completions/max_terminated_length": 285.9, "completions/mean_length": 218.6625, "completions/mean_terminated_length": 216.02678833007812, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.0486921429634095, "epoch": 1.7670313505562194, "frac_reward_zero_std": 0.7, "grad_norm": 0.10594558715820312, "kl": 0.008463947789277882, "learning_rate": 1.9750423826859188e-06, "loss": 0.0003, "num_tokens": 83706563.0, "reward": -0.45625, "reward_std": 0.19189817756414412, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.1918981894850731, "sampling/importance_sampling_ratio/max": 1.2347222089767456, "sampling/importance_sampling_ratio/mean": 1.0004438519477845, "sampling/importance_sampling_ratio/min": 0.804423040151596, "sampling/sampling_logp_difference/max": 0.23446749448776244, "sampling/sampling_logp_difference/mean": 0.02048439923673868, "step": 19220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 346.4, "completions/max_terminated_length": 346.4, "completions/mean_length": 235.4875, "completions/mean_terminated_length": 235.4875, "completions/min_length": 161.2, "completions/min_terminated_length": 161.2, "entropy": 1.1072215564548968, "epoch": 1.7679507217063528, "frac_reward_zero_std": 0.4, "grad_norm": 0.0805465579032898, "kl": 0.00996072317357175, "learning_rate": 1.9725655155870717e-06, "loss": -0.0, "num_tokens": 83752410.0, "reward": 0.11875, "reward_std": 0.5166270434856415, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.5166270434856415, "sampling/importance_sampling_ratio/max": 1.2684766411781312, "sampling/importance_sampling_ratio/mean": 0.9999668061733246, "sampling/importance_sampling_ratio/min": 0.7221728324890136, "sampling/sampling_logp_difference/max": 0.4181505680084229, "sampling/sampling_logp_difference/mean": 0.02141439449042082, "step": 19230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 182.025, "completions/mean_terminated_length": 182.025, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.0236621282994747, "epoch": 1.7688700928564862, "frac_reward_zero_std": 0.5, "grad_norm": 5.125039100646973, "kl": 0.008395250482135453, "learning_rate": 1.970089190205718e-06, "loss": 0.0005, "num_tokens": 83793124.0, "reward": -0.3, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.260031259059906, "sampling/importance_sampling_ratio/mean": 0.9997851014137268, "sampling/importance_sampling_ratio/min": 0.7815550267696381, "sampling/sampling_logp_difference/max": 0.2744813561439514, "sampling/sampling_logp_difference/mean": 0.020646426267921923, "step": 19240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.4, "completions/max_terminated_length": 304.4, "completions/mean_length": 220.0875, "completions/mean_terminated_length": 220.0875, "completions/min_length": 153.8, "completions/min_terminated_length": 153.8, "entropy": 1.07141622453928, "epoch": 1.7697894640066196, "frac_reward_zero_std": 0.3, "grad_norm": 4.592382431030273, "kl": 0.009063113306183368, "learning_rate": 1.9676134090852444e-06, "loss": 0.001, "num_tokens": 83839347.0, "reward": 0.025, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.2641918540000916, "sampling/importance_sampling_ratio/mean": 0.9995726764202117, "sampling/importance_sampling_ratio/min": 0.76695876121521, "sampling/sampling_logp_difference/max": 0.29035997986793516, "sampling/sampling_logp_difference/mean": 0.02130478359758854, "step": 19250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.7, "completions/max_terminated_length": 328.7, "completions/mean_length": 221.575, "completions/mean_terminated_length": 221.575, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 1.12071106210351, "epoch": 1.7707088351567528, "frac_reward_zero_std": 0.3, "grad_norm": 0.10907705873250961, "kl": 0.009397917822934688, "learning_rate": 1.965138174768477e-06, "loss": 0.0007, "num_tokens": 83885121.0, "reward": -0.075, "reward_std": 0.6043157160282135, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6043157398700714, "sampling/importance_sampling_ratio/max": 1.2830836057662964, "sampling/importance_sampling_ratio/mean": 0.9997569859027863, "sampling/importance_sampling_ratio/min": 0.784736168384552, "sampling/sampling_logp_difference/max": 0.28047881126403806, "sampling/sampling_logp_difference/mean": 0.022147497162222862, "step": 19260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.8, "completions/max_terminated_length": 311.8, "completions/mean_length": 231.125, "completions/mean_terminated_length": 231.125, "completions/min_length": 157.8, "completions/min_terminated_length": 157.8, "entropy": 1.0382263109087944, "epoch": 1.771628206306886, "frac_reward_zero_std": 0.3, "grad_norm": 4.511131286621094, "kl": 0.009483992360765114, "learning_rate": 1.9626634897976766e-06, "loss": 0.0005, "num_tokens": 83931723.0, "reward": -0.15, "reward_std": 0.6405095338821412, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.640509557723999, "sampling/importance_sampling_ratio/max": 1.346435534954071, "sampling/importance_sampling_ratio/mean": 0.9996383428573609, "sampling/importance_sampling_ratio/min": 0.7324358463287354, "sampling/sampling_logp_difference/max": 0.34567212462425234, "sampling/sampling_logp_difference/mean": 0.021083102840930224, "step": 19270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.9, "completions/max_terminated_length": 270.9, "completions/mean_length": 200.1375, "completions/mean_terminated_length": 200.1375, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.096966777741909, "epoch": 1.7725475774570194, "frac_reward_zero_std": 0.4, "grad_norm": 0.10441561788320541, "kl": 0.009229630586924031, "learning_rate": 1.9601893567145454e-06, "loss": 0.0006, "num_tokens": 83974622.0, "reward": 0.075, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2612476110458375, "sampling/importance_sampling_ratio/mean": 0.9995319306850433, "sampling/importance_sampling_ratio/min": 0.7687383651733398, "sampling/sampling_logp_difference/max": 0.2869745373725891, "sampling/sampling_logp_difference/mean": 0.021453574672341345, "step": 19280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.3, "completions/max_terminated_length": 267.3, "completions/mean_length": 201.3125, "completions/mean_terminated_length": 201.3125, "completions/min_length": 119.0, "completions/min_terminated_length": 119.0, "entropy": 0.9785691536962986, "epoch": 1.7734669486071528, "frac_reward_zero_std": 0.6, "grad_norm": 0.18917255103588104, "kl": 0.010157083263038658, "learning_rate": 1.957715778060216e-06, "loss": 0.0006, "num_tokens": 84018367.0, "reward": 0.475, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2789737343788148, "sampling/importance_sampling_ratio/mean": 0.9998923659324646, "sampling/importance_sampling_ratio/min": 0.7938157677650451, "sampling/sampling_logp_difference/max": 0.26970637440681455, "sampling/sampling_logp_difference/mean": 0.02044976521283388, "step": 19290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.9, "completions/max_terminated_length": 320.9, "completions/mean_length": 211.1, "completions/mean_terminated_length": 211.1, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.14443234577775, "epoch": 1.774386319757286, "frac_reward_zero_std": 0.2, "grad_norm": 6.530838966369629, "kl": 0.011293440836016088, "learning_rate": 1.9552427563752517e-06, "loss": 0.0006, "num_tokens": 84063311.0, "reward": 0.1, "reward_std": 0.6859699070453644, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6859699189662933, "sampling/importance_sampling_ratio/max": 1.3101686596870423, "sampling/importance_sampling_ratio/mean": 1.0003834784030914, "sampling/importance_sampling_ratio/min": 0.7888779461383819, "sampling/sampling_logp_difference/max": 0.28609566688537597, "sampling/sampling_logp_difference/mean": 0.022276022285223008, "step": 19300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.6, "completions/max_terminated_length": 311.6, "completions/mean_length": 212.325, "completions/mean_terminated_length": 212.325, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.0812247559428214, "epoch": 1.7753056909074192, "frac_reward_zero_std": 0.4, "grad_norm": 0.07779921591281891, "kl": 0.009181450086180121, "learning_rate": 1.952770294199646e-06, "loss": 0.0007, "num_tokens": 84107961.0, "reward": 0.14375, "reward_std": 0.517137861251831, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.5171378672122955, "sampling/importance_sampling_ratio/max": 1.233493721485138, "sampling/importance_sampling_ratio/mean": 0.9997764885425567, "sampling/importance_sampling_ratio/min": 0.7987427651882172, "sampling/sampling_logp_difference/max": 0.23882296085357665, "sampling/sampling_logp_difference/mean": 0.02076419284567237, "step": 19310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.4, "completions/max_terminated_length": 282.4, "completions/mean_length": 186.6375, "completions/mean_terminated_length": 186.6375, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 0.9669328261166811, "epoch": 1.7762250620575526, "frac_reward_zero_std": 0.7, "grad_norm": 11.825000762939453, "kl": 0.008915910834912211, "learning_rate": 1.950298394072812e-06, "loss": 0.0005, "num_tokens": 84144596.0, "reward": 0.475, "reward_std": 0.2886738538742065, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.2886738657951355, "sampling/importance_sampling_ratio/max": 1.327384340763092, "sampling/importance_sampling_ratio/mean": 0.9996634006500245, "sampling/importance_sampling_ratio/min": 0.7576848089694976, "sampling/sampling_logp_difference/max": 0.3075959801673889, "sampling/sampling_logp_difference/mean": 0.020358146354556083, "step": 19320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 203.225, "completions/mean_terminated_length": 203.225, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.035882469266653, "epoch": 1.777144433207686, "frac_reward_zero_std": 0.4, "grad_norm": 6.760248184204102, "kl": 0.011758233269210905, "learning_rate": 1.9478270585335917e-06, "loss": 0.0009, "num_tokens": 84192710.0, "reward": 0.04375, "reward_std": 0.45870069712400435, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.4587007209658623, "sampling/importance_sampling_ratio/max": 1.233514630794525, "sampling/importance_sampling_ratio/mean": 0.9998098134994506, "sampling/importance_sampling_ratio/min": 0.7836328089237213, "sampling/sampling_logp_difference/max": 0.2505605161190033, "sampling/sampling_logp_difference/mean": 0.020759830437600612, "step": 19330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 218.0, "completions/mean_terminated_length": 218.0, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.151974394917488, "epoch": 1.7780638043578194, "frac_reward_zero_std": 0.4, "grad_norm": 0.1101730614900589, "kl": 0.010578313894802705, "learning_rate": 1.9453562901202445e-06, "loss": 0.0001, "num_tokens": 84237230.0, "reward": -0.4, "reward_std": 0.49665169715881347, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.49665169715881347, "sampling/importance_sampling_ratio/max": 1.3329776167869567, "sampling/importance_sampling_ratio/mean": 0.9997564613819122, "sampling/importance_sampling_ratio/min": 0.7603452444076538, "sampling/sampling_logp_difference/max": 0.33467774391174315, "sampling/sampling_logp_difference/mean": 0.022285629995167255, "step": 19340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.9, "completions/max_terminated_length": 307.9, "completions/mean_length": 199.5375, "completions/mean_terminated_length": 199.5375, "completions/min_length": 116.7, "completions/min_terminated_length": 116.7, "entropy": 1.0369049284607172, "epoch": 1.7789831755079526, "frac_reward_zero_std": 0.6, "grad_norm": 0.1392974555492401, "kl": 0.008957888395525516, "learning_rate": 1.942886091370448e-06, "loss": 0.0005, "num_tokens": 84279073.0, "reward": -0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.241047167778015, "sampling/importance_sampling_ratio/mean": 0.9997024178504944, "sampling/importance_sampling_ratio/min": 0.7983779430389404, "sampling/sampling_logp_difference/max": 0.23362441062927247, "sampling/sampling_logp_difference/mean": 0.020373407751321793, "step": 19350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.9, "completions/max_terminated_length": 305.9, "completions/mean_length": 210.8, "completions/mean_terminated_length": 210.8, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.0499577566981315, "epoch": 1.7799025466580858, "frac_reward_zero_std": 0.5, "grad_norm": 0.1260279268026352, "kl": 0.009254383132793009, "learning_rate": 1.940416464821293e-06, "loss": 0.0005, "num_tokens": 84324641.0, "reward": 0.19375, "reward_std": 0.48624867796897886, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.48624871373176576, "sampling/importance_sampling_ratio/max": 1.2261870980262757, "sampling/importance_sampling_ratio/mean": 1.0000686466693878, "sampling/importance_sampling_ratio/min": 0.7769893944263458, "sampling/sampling_logp_difference/max": 0.26505701541900634, "sampling/sampling_logp_difference/mean": 0.02113064844161272, "step": 19360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.2, "completions/max_terminated_length": 333.2, "completions/mean_length": 222.5, "completions/mean_terminated_length": 222.5, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.0876087725162507, "epoch": 1.7808219178082192, "frac_reward_zero_std": 0.5, "grad_norm": 0.15660460293293, "kl": 0.010969003132777289, "learning_rate": 1.9379474130092852e-06, "loss": 0.0004, "num_tokens": 84369177.0, "reward": -0.25, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.2516002774238586, "sampling/importance_sampling_ratio/mean": 1.0002316594123841, "sampling/importance_sampling_ratio/min": 0.7634538352489472, "sampling/sampling_logp_difference/max": 0.29138989448547364, "sampling/sampling_logp_difference/mean": 0.02105546686798334, "step": 19370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.1, "completions/max_terminated_length": 299.1, "completions/mean_length": 213.375, "completions/mean_terminated_length": 213.375, "completions/min_length": 118.3, "completions/min_terminated_length": 118.3, "entropy": 1.1231270544230938, "epoch": 1.7817412889583526, "frac_reward_zero_std": 0.6, "grad_norm": 0.08246929198503494, "kl": 0.010083266289439053, "learning_rate": 1.9354789384703375e-06, "loss": 0.0002, "num_tokens": 84413727.0, "reward": -0.05, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.257431972026825, "sampling/importance_sampling_ratio/mean": 0.9999575436115264, "sampling/importance_sampling_ratio/min": 0.8018836796283721, "sampling/sampling_logp_difference/max": 0.25239613056182864, "sampling/sampling_logp_difference/mean": 0.021712788566946983, "step": 19380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.9, "completions/max_terminated_length": 265.9, "completions/mean_length": 201.5375, "completions/mean_terminated_length": 201.5375, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.0548102483153343, "epoch": 1.7826606601084858, "frac_reward_zero_std": 0.6, "grad_norm": 0.13502158224582672, "kl": 0.009155839146114887, "learning_rate": 1.9330110437397735e-06, "loss": 0.0004, "num_tokens": 84458786.0, "reward": 0.025, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2627443075180054, "sampling/importance_sampling_ratio/mean": 1.0004811704158783, "sampling/importance_sampling_ratio/min": 0.8091148316860199, "sampling/sampling_logp_difference/max": 0.24590469598770143, "sampling/sampling_logp_difference/mean": 0.02118274848908186, "step": 19390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 297.8, "completions/max_terminated_length": 285.5, "completions/mean_length": 212.675, "completions/mean_terminated_length": 209.84285888671874, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 0.987805355526507, "epoch": 1.783580031258619, "frac_reward_zero_std": 0.5, "grad_norm": 5.587035655975342, "kl": 0.008745078535866924, "learning_rate": 1.930543731352316e-06, "loss": 0.0005, "num_tokens": 84504840.0, "reward": 0.26875, "reward_std": 0.43548367023468015, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.43548368811607363, "sampling/importance_sampling_ratio/max": 1.2666862964630128, "sampling/importance_sampling_ratio/mean": 1.0002551555633545, "sampling/importance_sampling_ratio/min": 0.7960111856460571, "sampling/sampling_logp_difference/max": 0.2555211901664734, "sampling/sampling_logp_difference/mean": 0.019246876472607255, "step": 19400 }, { "epoch": 1.783580031258619, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 301.6369982547993, "eval_completions/max_terminated_length": 300.7452006980803, "eval_completions/mean_length": 212.97753054101221, "eval_completions/mean_terminated_length": 212.6216136672734, "eval_completions/min_length": 140.02792321116928, "eval_completions/min_terminated_length": 140.02792321116928, "eval_entropy": 1.0783085276423117, "eval_frac_reward_zero_std": 0.493891797556719, "eval_kl": 0.00957055105852129, "eval_loss": 0.0004971835296601057, "eval_num_tokens": 84504840.0, "eval_reward": -0.11267452006980803, "eval_reward_std": 0.4493870729187188, "eval_rewards/reward_fn/mean": -0.11267452006980803, "eval_rewards/reward_fn/std": 0.4493870891461614, "eval_runtime": 1754.2331, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.289221216662808, "eval_sampling/importance_sampling_ratio/mean": 1.0000098752309723, "eval_sampling/importance_sampling_ratio/min": 0.7748116293814794, "eval_sampling/sampling_logp_difference/max": 0.2984050824900156, "eval_sampling/sampling_logp_difference/mean": 0.020960319894536627, "eval_steps_per_second": 0.041, "step": 19400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 344.7, "completions/max_terminated_length": 312.4, "completions/mean_length": 231.875, "completions/mean_terminated_length": 225.53214416503906, "completions/min_length": 155.5, "completions/min_terminated_length": 155.5, "entropy": 1.0506267052143812, "epoch": 1.7844994024087524, "frac_reward_zero_std": 0.5, "grad_norm": 0.0837261751294136, "kl": 0.009295406978344545, "learning_rate": 1.928077003842094e-06, "loss": 0.0003, "num_tokens": 84552606.0, "reward": -0.4125, "reward_std": 0.4481888100504875, "rewards/reward_fn/mean": -0.4125, "rewards/reward_fn/std": 0.44818882197141646, "sampling/importance_sampling_ratio/max": 1.2493242621421814, "sampling/importance_sampling_ratio/mean": 1.000121933221817, "sampling/importance_sampling_ratio/min": 0.7970013916492462, "sampling/sampling_logp_difference/max": 0.2482740879058838, "sampling/sampling_logp_difference/mean": 0.02045822758227587, "step": 19410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 197.4, "completions/mean_terminated_length": 197.4, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 0.9639423415064812, "epoch": 1.7854187735588858, "frac_reward_zero_std": 0.3, "grad_norm": 4.8623433113098145, "kl": 0.009026089613325895, "learning_rate": 1.925610863742635e-06, "loss": 0.0009, "num_tokens": 84594974.0, "reward": 0.175, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2691115498542787, "sampling/importance_sampling_ratio/mean": 1.000725305080414, "sampling/importance_sampling_ratio/min": 0.7738381087779999, "sampling/sampling_logp_difference/max": 0.2846218943595886, "sampling/sampling_logp_difference/mean": 0.020572179835289715, "step": 19420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.2, "completions/max_terminated_length": 320.2, "completions/mean_length": 222.0625, "completions/mean_terminated_length": 222.0625, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.0461970329284669, "epoch": 1.7863381447090192, "frac_reward_zero_std": 0.4, "grad_norm": 4.8122639656066895, "kl": 0.008625698264222593, "learning_rate": 1.923145313586861e-06, "loss": 0.0005, "num_tokens": 84643355.0, "reward": 0.05, "reward_std": 0.5479275226593018, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5479275465011597, "sampling/importance_sampling_ratio/max": 1.25738765001297, "sampling/importance_sampling_ratio/mean": 0.999793291091919, "sampling/importance_sampling_ratio/min": 0.7959460854530335, "sampling/sampling_logp_difference/max": 0.25146392583847044, "sampling/sampling_logp_difference/mean": 0.020489318296313286, "step": 19430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 352.6, "completions/max_terminated_length": 352.6, "completions/mean_length": 234.05, "completions/mean_terminated_length": 234.05, "completions/min_length": 152.3, "completions/min_terminated_length": 152.3, "entropy": 1.1048885785043239, "epoch": 1.7872575158591524, "frac_reward_zero_std": 0.2, "grad_norm": 5.086431980133057, "kl": 0.010031084564980119, "learning_rate": 1.9206803559070915e-06, "loss": 0.0002, "num_tokens": 84686543.0, "reward": -0.4, "reward_std": 0.6893488764762878, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6893489003181458, "sampling/importance_sampling_ratio/max": 1.2585394024848937, "sampling/importance_sampling_ratio/mean": 1.0002147376537323, "sampling/importance_sampling_ratio/min": 0.7625975191593171, "sampling/sampling_logp_difference/max": 0.2900933623313904, "sampling/sampling_logp_difference/mean": 0.02123612239956856, "step": 19440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 206.1875, "completions/mean_terminated_length": 206.1875, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.1105508767068386, "epoch": 1.7881768870092856, "frac_reward_zero_std": 0.5, "grad_norm": 5.729686737060547, "kl": 0.00923511050932575, "learning_rate": 1.918215993235034e-06, "loss": 0.0004, "num_tokens": 84728470.0, "reward": -0.425, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.3018525838851929, "sampling/importance_sampling_ratio/mean": 1.000130432844162, "sampling/importance_sampling_ratio/min": 0.7676836729049683, "sampling/sampling_logp_difference/max": 0.3095707356929779, "sampling/sampling_logp_difference/mean": 0.02193008828908205, "step": 19450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.2, "completions/max_terminated_length": 275.2, "completions/mean_length": 194.3125, "completions/mean_terminated_length": 194.3125, "completions/min_length": 118.9, "completions/min_terminated_length": 118.9, "entropy": 1.0155738055706025, "epoch": 1.789096258159419, "frac_reward_zero_std": 0.3, "grad_norm": 6.19144868850708, "kl": 0.01024327843915671, "learning_rate": 1.9157522281017865e-06, "loss": 0.0008, "num_tokens": 84771663.0, "reward": 0.15, "reward_std": 0.5749112248420716, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5749112248420716, "sampling/importance_sampling_ratio/max": 1.2488455295562744, "sampling/importance_sampling_ratio/mean": 1.0004096567630767, "sampling/importance_sampling_ratio/min": 0.7908173561096191, "sampling/sampling_logp_difference/max": 0.2521522045135498, "sampling/sampling_logp_difference/mean": 0.020658292062580585, "step": 19460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.3, "completions/max_terminated_length": 291.3, "completions/mean_length": 211.0625, "completions/mean_terminated_length": 211.0625, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 1.053350668400526, "epoch": 1.7900156293095524, "frac_reward_zero_std": 0.4, "grad_norm": 4.166743755340576, "kl": 0.009039385191863402, "learning_rate": 1.9132890630378346e-06, "loss": 0.0006, "num_tokens": 84813636.0, "reward": -0.375, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.2812275409698486, "sampling/importance_sampling_ratio/mean": 1.0003423273563385, "sampling/importance_sampling_ratio/min": 0.7863016486167907, "sampling/sampling_logp_difference/max": 0.2736312985420227, "sampling/sampling_logp_difference/mean": 0.021150571294128895, "step": 19470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.9, "completions/max_terminated_length": 287.9, "completions/mean_length": 212.075, "completions/mean_terminated_length": 212.075, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.0126426003873348, "epoch": 1.7909350004596856, "frac_reward_zero_std": 0.4, "grad_norm": 4.911064147949219, "kl": 0.009382009448017925, "learning_rate": 1.9108265005730457e-06, "loss": 0.0005, "num_tokens": 84858362.0, "reward": 0.4, "reward_std": 0.4680067300796509, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.4680067300796509, "sampling/importance_sampling_ratio/max": 1.2654994249343872, "sampling/importance_sampling_ratio/mean": 1.0004583299160004, "sampling/importance_sampling_ratio/min": 0.7954815208911896, "sampling/sampling_logp_difference/max": 0.25236759185791013, "sampling/sampling_logp_difference/mean": 0.020272131077945232, "step": 19480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 347.4, "completions/max_terminated_length": 313.0, "completions/mean_length": 234.7, "completions/mean_terminated_length": 228.21428680419922, "completions/min_length": 150.1, "completions/min_terminated_length": 150.1, "entropy": 1.2071880675852298, "epoch": 1.7918543716098188, "frac_reward_zero_std": 0.2, "grad_norm": 5.194241046905518, "kl": 0.011014699292718432, "learning_rate": 1.908364543236669e-06, "loss": 0.001, "num_tokens": 84905842.0, "reward": -0.3625, "reward_std": 0.7381100594997406, "rewards/reward_fn/mean": -0.3625, "rewards/reward_fn/std": 0.7381100833415986, "sampling/importance_sampling_ratio/max": 1.2960466504096986, "sampling/importance_sampling_ratio/mean": 1.0000091731548308, "sampling/importance_sampling_ratio/min": 0.7720128953456878, "sampling/sampling_logp_difference/max": 0.2839381814002991, "sampling/sampling_logp_difference/mean": 0.022550057061016558, "step": 19490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 196.875, "completions/mean_terminated_length": 196.875, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0246406309306622, "epoch": 1.7927737427599522, "frac_reward_zero_std": 0.4, "grad_norm": 6.137310981750488, "kl": 0.008645608325605281, "learning_rate": 1.9059031935573312e-06, "loss": -0.0001, "num_tokens": 84948200.0, "reward": -0.425, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2927849769592286, "sampling/importance_sampling_ratio/mean": 1.0003267645835876, "sampling/importance_sampling_ratio/min": 0.7833991467952728, "sampling/sampling_logp_difference/max": 0.28410524129867554, "sampling/sampling_logp_difference/mean": 0.020524751767516135, "step": 19500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 199.225, "completions/mean_terminated_length": 199.225, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.1087736688554286, "epoch": 1.7936931139100856, "frac_reward_zero_std": 0.8, "grad_norm": 1.0548746585845947, "kl": 0.010147134136059322, "learning_rate": 1.9034424540630367e-06, "loss": 0.0005, "num_tokens": 84990546.0, "reward": -0.475, "reward_std": 0.1776151716709137, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.1776151716709137, "sampling/importance_sampling_ratio/max": 1.2740545511245727, "sampling/importance_sampling_ratio/mean": 0.9994801223278046, "sampling/importance_sampling_ratio/min": 0.7226991325616836, "sampling/sampling_logp_difference/max": 0.36397876739501955, "sampling/sampling_logp_difference/mean": 0.021944558061659337, "step": 19510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.0, "completions/max_terminated_length": 280.0, "completions/mean_length": 194.4625, "completions/mean_terminated_length": 194.4625, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.04340160228312, "epoch": 1.794612485060219, "frac_reward_zero_std": 0.5, "grad_norm": 8.074248313903809, "kl": 0.009846351318992674, "learning_rate": 1.9009823272811612e-06, "loss": 0.0006, "num_tokens": 85032815.0, "reward": 0.025, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2594703793525697, "sampling/importance_sampling_ratio/mean": 0.9999652206897736, "sampling/importance_sampling_ratio/min": 0.8009726762771606, "sampling/sampling_logp_difference/max": 0.23587526679039, "sampling/sampling_logp_difference/mean": 0.02085103001445532, "step": 19520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.9, "completions/max_terminated_length": 286.9, "completions/mean_length": 194.375, "completions/mean_terminated_length": 194.375, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 0.9721465557813644, "epoch": 1.7955318562103522, "frac_reward_zero_std": 0.6, "grad_norm": 7.156385898590088, "kl": 0.009931440398213453, "learning_rate": 1.898522815738454e-06, "loss": 0.0005, "num_tokens": 85073269.0, "reward": -0.125, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.305891990661621, "sampling/importance_sampling_ratio/mean": 0.9994708359241485, "sampling/importance_sampling_ratio/min": 0.748533108830452, "sampling/sampling_logp_difference/max": 0.3589649796485901, "sampling/sampling_logp_difference/mean": 0.02041681855916977, "step": 19530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.0, "completions/max_terminated_length": 322.0, "completions/mean_length": 228.2125, "completions/mean_terminated_length": 228.2125, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.1157794438302517, "epoch": 1.7964512273604853, "frac_reward_zero_std": 0.5, "grad_norm": 6.602654457092285, "kl": 0.00921061944682151, "learning_rate": 1.896063921961028e-06, "loss": 0.0005, "num_tokens": 85119798.0, "reward": -0.2, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2810415983200074, "sampling/importance_sampling_ratio/mean": 0.9997957944869995, "sampling/importance_sampling_ratio/min": 0.795010107755661, "sampling/sampling_logp_difference/max": 0.2668586254119873, "sampling/sampling_logp_difference/mean": 0.02070627100765705, "step": 19540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 295.8, "completions/max_terminated_length": 290.9, "completions/mean_length": 211.0875, "completions/mean_terminated_length": 208.7375, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 0.9766888786107302, "epoch": 1.7973705985106188, "frac_reward_zero_std": 0.5, "grad_norm": 0.08448850363492966, "kl": 0.008802618645131589, "learning_rate": 1.8936056484743659e-06, "loss": 0.0006, "num_tokens": 85160661.0, "reward": -0.00625, "reward_std": 0.4587006986141205, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.4587007284164429, "sampling/importance_sampling_ratio/max": 1.2561034679412841, "sampling/importance_sampling_ratio/mean": 1.000113660097122, "sampling/importance_sampling_ratio/min": 0.7507474750280381, "sampling/sampling_logp_difference/max": 0.3419951558113098, "sampling/sampling_logp_difference/mean": 0.01980922035872936, "step": 19550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 194.3375, "completions/mean_terminated_length": 194.3375, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0803027898073196, "epoch": 1.7982899696607522, "frac_reward_zero_std": 0.4, "grad_norm": 0.08960393816232681, "kl": 0.00951119469536934, "learning_rate": 1.8911479978033116e-06, "loss": 0.0007, "num_tokens": 85202232.0, "reward": -0.15, "reward_std": 0.5294508516788483, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5294508635997772, "sampling/importance_sampling_ratio/max": 1.2261127710342408, "sampling/importance_sampling_ratio/mean": 0.9996846377849579, "sampling/importance_sampling_ratio/min": 0.7671169579029083, "sampling/sampling_logp_difference/max": 0.26872313022613525, "sampling/sampling_logp_difference/mean": 0.021234538033604623, "step": 19560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.4, "completions/max_terminated_length": 275.4, "completions/mean_length": 186.6375, "completions/mean_terminated_length": 186.6375, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.053387788683176, "epoch": 1.7992093408108853, "frac_reward_zero_std": 0.5, "grad_norm": 8.411700248718262, "kl": 0.008290148977539501, "learning_rate": 1.8886909724720691e-06, "loss": 0.0004, "num_tokens": 85242691.0, "reward": -0.05, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.277222228050232, "sampling/importance_sampling_ratio/mean": 0.9993468046188354, "sampling/importance_sampling_ratio/min": 0.8054819941520691, "sampling/sampling_logp_difference/max": 0.25311300754547117, "sampling/sampling_logp_difference/mean": 0.020711152255535124, "step": 19570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.4, "completions/max_terminated_length": 263.4, "completions/mean_length": 182.45, "completions/mean_terminated_length": 182.45, "completions/min_length": 115.7, "completions/min_terminated_length": 115.7, "entropy": 1.0007963351905347, "epoch": 1.8001287119610185, "frac_reward_zero_std": 0.5, "grad_norm": 5.672979831695557, "kl": 0.008849398963502609, "learning_rate": 1.8862345750042002e-06, "loss": 0.0003, "num_tokens": 85282087.0, "reward": -0.1, "reward_std": 0.3897472023963928, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3897472023963928, "sampling/importance_sampling_ratio/max": 1.2194852113723755, "sampling/importance_sampling_ratio/mean": 0.999730509519577, "sampling/importance_sampling_ratio/min": 0.7953074812889099, "sampling/sampling_logp_difference/max": 0.23935918807983397, "sampling/sampling_logp_difference/mean": 0.020051315799355506, "step": 19580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 214.55, "completions/mean_terminated_length": 214.55, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.0436974182724952, "epoch": 1.801048083111152, "frac_reward_zero_std": 0.3, "grad_norm": 7.490072727203369, "kl": 0.010873592473217286, "learning_rate": 1.883778807922622e-06, "loss": 0.0008, "num_tokens": 85324763.0, "reward": 0.0625, "reward_std": 0.535893426835537, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.5358934387564659, "sampling/importance_sampling_ratio/max": 1.3001521587371827, "sampling/importance_sampling_ratio/mean": 1.0004318714141847, "sampling/importance_sampling_ratio/min": 0.7467000305652618, "sampling/sampling_logp_difference/max": 0.32777385115623475, "sampling/sampling_logp_difference/mean": 0.0212148018181324, "step": 19590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.2, "completions/max_terminated_length": 298.7, "completions/mean_length": 223.1375, "completions/mean_terminated_length": 219.69107208251953, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0922662667930125, "epoch": 1.8019674542612854, "frac_reward_zero_std": 0.7, "grad_norm": 0.07150261104106903, "kl": 0.008831599279073999, "learning_rate": 1.8813236737496035e-06, "loss": 0.0007, "num_tokens": 85370310.0, "reward": -0.05625, "reward_std": 0.3217787265777588, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.3217787504196167, "sampling/importance_sampling_ratio/max": 1.3273424625396728, "sampling/importance_sampling_ratio/mean": 1.0000539004802704, "sampling/importance_sampling_ratio/min": 0.7834661066532135, "sampling/sampling_logp_difference/max": 0.2977943181991577, "sampling/sampling_logp_difference/mean": 0.021595523320138456, "step": 19600 }, { "epoch": 1.8019674542612854, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 294.6544502617801, "eval_completions/max_terminated_length": 293.27923211169286, "eval_completions/mean_length": 206.4415357766143, "eval_completions/mean_terminated_length": 206.00342814127603, "eval_completions/min_length": 134.08551483420592, "eval_completions/min_terminated_length": 134.08551483420592, "eval_entropy": 1.0818474337484616, "eval_frac_reward_zero_std": 0.49214659685863876, "eval_kl": 0.0094301931911114, "eval_loss": 0.0004788691003341228, "eval_num_tokens": 85370310.0, "eval_reward": -0.1181282722513089, "eval_reward_std": 0.4476533203953016, "eval_rewards/reward_fn/mean": -0.1181282722513089, "eval_rewards/reward_fn/std": 0.4476533341262145, "eval_runtime": 1718.9743, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.265480866814991, "eval_sampling/importance_sampling_ratio/mean": 0.9999999248960671, "eval_sampling/importance_sampling_ratio/min": 0.7744625950879035, "eval_sampling/sampling_logp_difference/max": 0.2848565935256385, "eval_sampling/sampling_logp_difference/mean": 0.020828915841188313, "eval_steps_per_second": 0.042, "step": 19600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.3, "completions/max_terminated_length": 323.3, "completions/mean_length": 215.7, "completions/mean_terminated_length": 215.7, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.0478144034743309, "epoch": 1.8028868254114185, "frac_reward_zero_std": 0.6, "grad_norm": 7.793282985687256, "kl": 0.010205858541303315, "learning_rate": 1.8788691750067655e-06, "loss": 0.0004, "num_tokens": 85416078.0, "reward": -0.05, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2364765048027038, "sampling/importance_sampling_ratio/mean": 0.9990589618682861, "sampling/importance_sampling_ratio/min": 0.7918550729751587, "sampling/sampling_logp_difference/max": 0.24551435708999633, "sampling/sampling_logp_difference/mean": 0.020964093133807184, "step": 19610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.4, "completions/max_terminated_length": 268.4, "completions/mean_length": 199.175, "completions/mean_terminated_length": 199.175, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.0943655677139759, "epoch": 1.803806196561552, "frac_reward_zero_std": 0.3, "grad_norm": 0.10264355689287186, "kl": 0.00936293219565414, "learning_rate": 1.8764153142150754e-06, "loss": 0.0008, "num_tokens": 85460076.0, "reward": -0.275, "reward_std": 0.6186381995677948, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6186382234096527, "sampling/importance_sampling_ratio/max": 1.2563108086586, "sampling/importance_sampling_ratio/mean": 1.0001594364643096, "sampling/importance_sampling_ratio/min": 0.7697018325328827, "sampling/sampling_logp_difference/max": 0.2793015480041504, "sampling/sampling_logp_difference/mean": 0.021254190243780613, "step": 19620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 287.6, "completions/max_terminated_length": 283.8, "completions/mean_length": 221.9, "completions/mean_terminated_length": 214.875, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0142985701560974, "epoch": 1.8047255677116851, "frac_reward_zero_std": 0.1, "grad_norm": 6.596829891204834, "kl": 0.011230822559446097, "learning_rate": 1.8739620938948433e-06, "loss": 0.0009, "num_tokens": 85502996.0, "reward": -0.025, "reward_std": 0.8432861804962158, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.8432862043380738, "sampling/importance_sampling_ratio/max": 1.2632776260375977, "sampling/importance_sampling_ratio/mean": 1.0004761159420013, "sampling/importance_sampling_ratio/min": 0.753109809756279, "sampling/sampling_logp_difference/max": 0.336643648147583, "sampling/sampling_logp_difference/mean": 0.01995506603270769, "step": 19630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.2, "completions/max_terminated_length": 312.2, "completions/mean_length": 229.025, "completions/mean_terminated_length": 225.88035888671874, "completions/min_length": 149.7, "completions/min_terminated_length": 149.7, "entropy": 1.1423318289220332, "epoch": 1.8056449388618185, "frac_reward_zero_std": 0.3, "grad_norm": 4.961842060089111, "kl": 0.010730771982343868, "learning_rate": 1.8715095165657238e-06, "loss": 0.0004, "num_tokens": 85548422.0, "reward": -0.10625, "reward_std": 0.6232402443885803, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.6232402503490448, "sampling/importance_sampling_ratio/max": 1.256884479522705, "sampling/importance_sampling_ratio/mean": 1.000254476070404, "sampling/importance_sampling_ratio/min": 0.7899461328983307, "sampling/sampling_logp_difference/max": 0.25473973751068113, "sampling/sampling_logp_difference/mean": 0.021580992452800273, "step": 19640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.9, "completions/max_terminated_length": 296.9, "completions/mean_length": 211.4, "completions/mean_terminated_length": 211.4, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.0526555337011814, "epoch": 1.806564310011952, "frac_reward_zero_std": 0.5, "grad_norm": 5.5711188316345215, "kl": 0.009272913791937754, "learning_rate": 1.8690575847467107e-06, "loss": 0.0009, "num_tokens": 85589262.0, "reward": 0.04375, "reward_std": 0.44022402763366697, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.44022404551506045, "sampling/importance_sampling_ratio/max": 1.287692129611969, "sampling/importance_sampling_ratio/mean": 1.0001347422599793, "sampling/importance_sampling_ratio/min": 0.7306984454393387, "sampling/sampling_logp_difference/max": 0.35074132680892944, "sampling/sampling_logp_difference/mean": 0.02133138757199049, "step": 19650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 198.0125, "completions/mean_terminated_length": 198.0125, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0077449146658182, "epoch": 1.8074836811620851, "frac_reward_zero_std": 0.5, "grad_norm": 8.31083869934082, "kl": 0.008652940834872424, "learning_rate": 1.8666063009561344e-06, "loss": 0.0004, "num_tokens": 85631863.0, "reward": 0.05, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.2880322098731996, "sampling/importance_sampling_ratio/mean": 0.9991119980812073, "sampling/importance_sampling_ratio/min": 0.7671273171901702, "sampling/sampling_logp_difference/max": 0.3079285740852356, "sampling/sampling_logp_difference/mean": 0.020755801163613797, "step": 19660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.9, "completions/max_terminated_length": 257.9, "completions/mean_length": 197.5875, "completions/mean_terminated_length": 197.5875, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0481701493263245, "epoch": 1.8084030523122183, "frac_reward_zero_std": 0.3, "grad_norm": 5.682338237762451, "kl": 0.009403378420392983, "learning_rate": 1.8641556677116596e-06, "loss": 0.0003, "num_tokens": 85674062.0, "reward": 0.175, "reward_std": 0.6295817136764527, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.6295817255973816, "sampling/importance_sampling_ratio/max": 1.2512914299964906, "sampling/importance_sampling_ratio/mean": 1.0000519812107087, "sampling/importance_sampling_ratio/min": 0.7697308301925659, "sampling/sampling_logp_difference/max": 0.275012868642807, "sampling/sampling_logp_difference/mean": 0.020740146189928053, "step": 19670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.3, "completions/max_terminated_length": 257.3, "completions/mean_length": 189.825, "completions/mean_terminated_length": 189.825, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 0.9616469230502844, "epoch": 1.8093224234623517, "frac_reward_zero_std": 0.2, "grad_norm": 4.474955081939697, "kl": 0.012574754073284566, "learning_rate": 1.8617056875302825e-06, "loss": 0.0008, "num_tokens": 85714064.0, "reward": 0.325, "reward_std": 0.6203559041023254, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.6203559160232544, "sampling/importance_sampling_ratio/max": 1.2574510335922242, "sampling/importance_sampling_ratio/mean": 1.0003540992736817, "sampling/importance_sampling_ratio/min": 0.7713484585285186, "sampling/sampling_logp_difference/max": 0.27875320315361024, "sampling/sampling_logp_difference/mean": 0.019904662668704987, "step": 19680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 212.6625, "completions/mean_terminated_length": 212.6625, "completions/min_length": 148.8, "completions/min_terminated_length": 148.8, "entropy": 1.01550507619977, "epoch": 1.8102417946124851, "frac_reward_zero_std": 0.7, "grad_norm": 0.09427832067012787, "kl": 0.011383780173491686, "learning_rate": 1.8592563629283306e-06, "loss": 0.0003, "num_tokens": 85754661.0, "reward": -0.5, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.3357011437416078, "sampling/importance_sampling_ratio/mean": 0.9998615741729736, "sampling/importance_sampling_ratio/min": 0.7666589856147766, "sampling/sampling_logp_difference/max": 0.32751452922821045, "sampling/sampling_logp_difference/mean": 0.020364928431808947, "step": 19690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.5, "completions/max_terminated_length": 319.5, "completions/mean_length": 217.375, "completions/mean_terminated_length": 217.375, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.0641882680356503, "epoch": 1.8111611657626183, "frac_reward_zero_std": 0.6, "grad_norm": 7.186395168304443, "kl": 0.01048609838471748, "learning_rate": 1.856807696421456e-06, "loss": 0.0005, "num_tokens": 85799523.0, "reward": -0.08125, "reward_std": 0.32362287640571596, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.3236228823661804, "sampling/importance_sampling_ratio/max": 1.2780785799026488, "sampling/importance_sampling_ratio/mean": 1.0001107096672057, "sampling/importance_sampling_ratio/min": 0.7715914666652679, "sampling/sampling_logp_difference/max": 0.2994180917739868, "sampling/sampling_logp_difference/mean": 0.021284938976168632, "step": 19700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.6, "completions/max_terminated_length": 324.6, "completions/mean_length": 220.95, "completions/mean_terminated_length": 220.95, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0420475795865058, "epoch": 1.8120805369127517, "frac_reward_zero_std": 0.5, "grad_norm": 0.09336770325899124, "kl": 0.01414004502003081, "learning_rate": 1.854359690524635e-06, "loss": 0.0007, "num_tokens": 85843647.0, "reward": -0.2875, "reward_std": 0.45316134095191957, "rewards/reward_fn/mean": -0.2875, "rewards/reward_fn/std": 0.45316136479377744, "sampling/importance_sampling_ratio/max": 1.2883472204208375, "sampling/importance_sampling_ratio/mean": 1.000252616405487, "sampling/importance_sampling_ratio/min": 0.7423620402812958, "sampling/sampling_logp_difference/max": 0.31803167462348936, "sampling/sampling_logp_difference/mean": 0.020891806483268736, "step": 19710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.3, "completions/max_terminated_length": 285.3, "completions/mean_length": 203.1125, "completions/mean_terminated_length": 203.1125, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0735899180173873, "epoch": 1.812999908062885, "frac_reward_zero_std": 0.4, "grad_norm": 6.137740612030029, "kl": 0.008787349335034379, "learning_rate": 1.8519123477521661e-06, "loss": 0.0005, "num_tokens": 85886688.0, "reward": 0.05, "reward_std": 0.5807266771793366, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5807267129421234, "sampling/importance_sampling_ratio/max": 1.2731894254684448, "sampling/importance_sampling_ratio/mean": 1.0002490639686585, "sampling/importance_sampling_ratio/min": 0.76953244805336, "sampling/sampling_logp_difference/max": 0.2885615050792694, "sampling/sampling_logp_difference/mean": 0.022121322713792323, "step": 19720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.3, "completions/max_terminated_length": 327.3, "completions/mean_length": 212.425, "completions/mean_terminated_length": 212.425, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 0.993070725351572, "epoch": 1.8139192792130183, "frac_reward_zero_std": 0.6, "grad_norm": 4.966591835021973, "kl": 0.008351793597103096, "learning_rate": 1.849465670617667e-06, "loss": 0.0004, "num_tokens": 85926402.0, "reward": 0.25, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.3050229787826537, "sampling/importance_sampling_ratio/mean": 1.0000426173210144, "sampling/importance_sampling_ratio/min": 0.7773508369922638, "sampling/sampling_logp_difference/max": 0.28097217082977294, "sampling/sampling_logp_difference/mean": 0.020654189586639404, "step": 19730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 215.3375, "completions/mean_terminated_length": 215.3375, "completions/min_length": 150.0, "completions/min_terminated_length": 150.0, "entropy": 1.1417989894747733, "epoch": 1.8148386503631517, "frac_reward_zero_std": 0.4, "grad_norm": 0.09284371882677078, "kl": 0.009538648679153994, "learning_rate": 1.8470196616340701e-06, "loss": 0.0002, "num_tokens": 85970805.0, "reward": -0.075, "reward_std": 0.48987806439399717, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.48987806439399717, "sampling/importance_sampling_ratio/max": 1.2535497307777406, "sampling/importance_sampling_ratio/mean": 0.999837589263916, "sampling/importance_sampling_ratio/min": 0.751339316368103, "sampling/sampling_logp_difference/max": 0.30762237310409546, "sampling/sampling_logp_difference/mean": 0.02206195369362831, "step": 19740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 218.025, "completions/mean_terminated_length": 218.025, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0140916220843792, "epoch": 1.815758021513285, "frac_reward_zero_std": 0.7, "grad_norm": 0.0858820378780365, "kl": 0.008584285163669847, "learning_rate": 1.8445743233136254e-06, "loss": 0.0002, "num_tokens": 86016951.0, "reward": 0.0, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.3067899823188782, "sampling/importance_sampling_ratio/mean": 0.9999642133712768, "sampling/importance_sampling_ratio/min": 0.7177504122257232, "sampling/sampling_logp_difference/max": 0.3448774695396423, "sampling/sampling_logp_difference/mean": 0.020408570021390914, "step": 19750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.9, "completions/max_terminated_length": 264.9, "completions/mean_length": 187.8625, "completions/mean_terminated_length": 187.8625, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 1.1498338773846626, "epoch": 1.816677392663418, "frac_reward_zero_std": 0.7, "grad_norm": 6.064332008361816, "kl": 0.009254298120504245, "learning_rate": 1.8421296581678874e-06, "loss": 0.0004, "num_tokens": 86060684.0, "reward": 0.375, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2721920967102052, "sampling/importance_sampling_ratio/mean": 0.9996862649917603, "sampling/importance_sampling_ratio/min": 0.7924518823623657, "sampling/sampling_logp_difference/max": 0.2569157361984253, "sampling/sampling_logp_difference/mean": 0.02224649488925934, "step": 19760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.9, "completions/max_terminated_length": 316.9, "completions/mean_length": 225.425, "completions/mean_terminated_length": 225.425, "completions/min_length": 163.1, "completions/min_terminated_length": 163.1, "entropy": 1.1023948721587657, "epoch": 1.8175967638135515, "frac_reward_zero_std": 0.7, "grad_norm": 0.06457577645778656, "kl": 0.009310008960892447, "learning_rate": 1.8396856687077257e-06, "loss": 0.0006, "num_tokens": 86106270.0, "reward": -0.15, "reward_std": 0.299601674079895, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.29960169792175295, "sampling/importance_sampling_ratio/max": 1.2637608766555786, "sampling/importance_sampling_ratio/mean": 0.999967634677887, "sampling/importance_sampling_ratio/min": 0.7702328383922576, "sampling/sampling_logp_difference/max": 0.2878342270851135, "sampling/sampling_logp_difference/mean": 0.021718919463455676, "step": 19770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.2, "completions/max_terminated_length": 290.2, "completions/mean_length": 216.8375, "completions/mean_terminated_length": 216.8375, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 0.9979631207883358, "epoch": 1.818516134963685, "frac_reward_zero_std": 0.4, "grad_norm": 6.523681163787842, "kl": 0.009522995803854429, "learning_rate": 1.8372423574433135e-06, "loss": 0.0004, "num_tokens": 86149233.0, "reward": 0.0, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.2511794328689576, "sampling/importance_sampling_ratio/mean": 1.0000752210617065, "sampling/importance_sampling_ratio/min": 0.7816982090473175, "sampling/sampling_logp_difference/max": 0.2592421293258667, "sampling/sampling_logp_difference/mean": 0.020116601884365082, "step": 19780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.5, "completions/max_terminated_length": 277.5, "completions/mean_length": 199.4, "completions/mean_terminated_length": 199.4, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.11259433478117, "epoch": 1.819435506113818, "frac_reward_zero_std": 0.4, "grad_norm": 5.165045738220215, "kl": 0.008934077358571813, "learning_rate": 1.8347997268841267e-06, "loss": 0.0007, "num_tokens": 86192105.0, "reward": -0.525, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2531922221183778, "sampling/importance_sampling_ratio/mean": 0.9998731970787048, "sampling/importance_sampling_ratio/min": 0.7454203307628632, "sampling/sampling_logp_difference/max": 0.3211917161941528, "sampling/sampling_logp_difference/mean": 0.02173961792141199, "step": 19790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 375.2, "completions/max_terminated_length": 367.8, "completions/mean_length": 257.5875, "completions/mean_terminated_length": 254.97321472167968, "completions/min_length": 156.2, "completions/min_terminated_length": 156.2, "entropy": 1.0772136747837067, "epoch": 1.8203548772639515, "frac_reward_zero_std": 0.3, "grad_norm": 9.912672996520996, "kl": 0.009610734088346362, "learning_rate": 1.8323577795389424e-06, "loss": 0.0005, "num_tokens": 86241592.0, "reward": -0.2125, "reward_std": 0.6033627092838287, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.6033627331256867, "sampling/importance_sampling_ratio/max": 1.2775170445442199, "sampling/importance_sampling_ratio/mean": 0.9998564004898072, "sampling/importance_sampling_ratio/min": 0.7310947775840759, "sampling/sampling_logp_difference/max": 0.32358919382095336, "sampling/sampling_logp_difference/mean": 0.021435271948575974, "step": 19800 }, { "epoch": 1.8203548772639515, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 297.2705061082024, "eval_completions/max_terminated_length": 294.7452006980803, "eval_completions/mean_length": 207.60514834205932, "eval_completions/mean_terminated_length": 207.01740231605515, "eval_completions/min_length": 137.565445026178, "eval_completions/min_terminated_length": 137.565445026178, "eval_entropy": 1.0782199054176271, "eval_frac_reward_zero_std": 0.47294938917975565, "eval_kl": 0.009473210688270823, "eval_loss": 0.00043691747123375535, "eval_num_tokens": 86241592.0, "eval_reward": -0.10634816753926701, "eval_reward_std": 0.4576006416510953, "eval_rewards/reward_fn/mean": -0.10634816753926701, "eval_rewards/reward_fn/std": 0.4576006571503834, "eval_runtime": 1730.4457, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2707584203849913, "eval_sampling/importance_sampling_ratio/mean": 0.9999570795586804, "eval_sampling/importance_sampling_ratio/min": 0.769922747505897, "eval_sampling/sampling_logp_difference/max": 0.2919234961635363, "eval_sampling/sampling_logp_difference/mean": 0.020989352583651144, "eval_steps_per_second": 0.042, "step": 19800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.8, "completions/max_terminated_length": 276.8, "completions/mean_length": 203.85, "completions/mean_terminated_length": 203.85, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.0859092749655246, "epoch": 1.8212742484140847, "frac_reward_zero_std": 0.4, "grad_norm": 6.117194652557373, "kl": 0.008800358651205898, "learning_rate": 1.8299165179158367e-06, "loss": 0.0004, "num_tokens": 86286740.0, "reward": 0.225, "reward_std": 0.5841213405132294, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5841213643550873, "sampling/importance_sampling_ratio/max": 1.2418041706085206, "sampling/importance_sampling_ratio/mean": 1.0001069068908692, "sampling/importance_sampling_ratio/min": 0.7836432158946991, "sampling/sampling_logp_difference/max": 0.25616902112960815, "sampling/sampling_logp_difference/mean": 0.021567078679800032, "step": 19810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 198.9, "completions/mean_terminated_length": 198.9, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.1287999287247659, "epoch": 1.822193619564218, "frac_reward_zero_std": 0.7, "grad_norm": 0.11367081105709076, "kl": 0.0103665987146087, "learning_rate": 1.8274759445221802e-06, "loss": 0.0004, "num_tokens": 86330508.0, "reward": -0.125, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2538563370704652, "sampling/importance_sampling_ratio/mean": 0.9995180666446686, "sampling/importance_sampling_ratio/min": 0.7793002247810363, "sampling/sampling_logp_difference/max": 0.2544142961502075, "sampling/sampling_logp_difference/mean": 0.0218738978728652, "step": 19820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 213.2375, "completions/mean_terminated_length": 213.2375, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0734468445181846, "epoch": 1.8231129907143515, "frac_reward_zero_std": 0.6, "grad_norm": 0.09529471397399902, "kl": 0.008921374322380871, "learning_rate": 1.8250360618646387e-06, "loss": 0.0005, "num_tokens": 86373943.0, "reward": -0.05, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2686028957366944, "sampling/importance_sampling_ratio/mean": 1.00007466673851, "sampling/importance_sampling_ratio/min": 0.779774671792984, "sampling/sampling_logp_difference/max": 0.2840897023677826, "sampling/sampling_logp_difference/mean": 0.021345170214772224, "step": 19830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.5, "completions/max_terminated_length": 284.5, "completions/mean_length": 213.175, "completions/mean_terminated_length": 213.175, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.0197639122605324, "epoch": 1.8240323618644847, "frac_reward_zero_std": 0.8, "grad_norm": 0.08186525106430054, "kl": 0.011056596011621877, "learning_rate": 1.8225968724491672e-06, "loss": 0.0007, "num_tokens": 86419989.0, "reward": 0.2, "reward_std": 0.20701966285705567, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.20701968669891357, "sampling/importance_sampling_ratio/max": 1.2705048441886901, "sampling/importance_sampling_ratio/mean": 1.0002313673496246, "sampling/importance_sampling_ratio/min": 0.7370791912078858, "sampling/sampling_logp_difference/max": 0.34419810175895693, "sampling/sampling_logp_difference/mean": 0.02043099943548441, "step": 19840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.3, "completions/max_terminated_length": 272.3, "completions/mean_length": 202.25, "completions/mean_terminated_length": 202.25, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0281270675361156, "epoch": 1.8249517330146179, "frac_reward_zero_std": 0.4, "grad_norm": 5.417271614074707, "kl": 0.009263064805418253, "learning_rate": 1.8201583787810073e-06, "loss": 0.0007, "num_tokens": 86461385.0, "reward": -0.125, "reward_std": 0.522677218914032, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.522677230834961, "sampling/importance_sampling_ratio/max": 1.2717837929725646, "sampling/importance_sampling_ratio/mean": 0.9997800886631012, "sampling/importance_sampling_ratio/min": 0.7574860572814941, "sampling/sampling_logp_difference/max": 0.29141005873680115, "sampling/sampling_logp_difference/mean": 0.020956055819988252, "step": 19850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.3, "completions/max_terminated_length": 262.3, "completions/mean_length": 190.625, "completions/mean_terminated_length": 190.625, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0717782594263554, "epoch": 1.8258711041647513, "frac_reward_zero_std": 0.6, "grad_norm": 0.09885425865650177, "kl": 0.009477449906989932, "learning_rate": 1.8177205833646888e-06, "loss": 0.0002, "num_tokens": 86499731.0, "reward": 0.025, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2262810230255128, "sampling/importance_sampling_ratio/mean": 0.9996652066707611, "sampling/importance_sampling_ratio/min": 0.7767757296562194, "sampling/sampling_logp_difference/max": 0.26413424015045167, "sampling/sampling_logp_difference/mean": 0.021057157218456267, "step": 19860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.2, "completions/max_terminated_length": 264.2, "completions/mean_length": 185.95, "completions/mean_terminated_length": 185.95, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0670934617519379, "epoch": 1.8267904753148847, "frac_reward_zero_std": 0.5, "grad_norm": 7.591878414154053, "kl": 0.011170013865921646, "learning_rate": 1.8152834887040227e-06, "loss": 0.0009, "num_tokens": 86540423.0, "reward": 0.325, "reward_std": 0.4191516935825348, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.4191517174243927, "sampling/importance_sampling_ratio/max": 1.2600827097892762, "sampling/importance_sampling_ratio/mean": 0.9996782422065735, "sampling/importance_sampling_ratio/min": 0.7899434030056, "sampling/sampling_logp_difference/max": 0.2543265998363495, "sampling/sampling_logp_difference/mean": 0.02126872856169939, "step": 19870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.1, "completions/max_terminated_length": 257.1, "completions/mean_length": 175.6, "completions/mean_terminated_length": 175.6, "completions/min_length": 111.3, "completions/min_terminated_length": 111.3, "entropy": 1.0027554951608182, "epoch": 1.8277098464650179, "frac_reward_zero_std": 0.5, "grad_norm": 6.005442142486572, "kl": 0.00900869475735817, "learning_rate": 1.8128470973021005e-06, "loss": 0.0004, "num_tokens": 86578383.0, "reward": 0.075, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2528497219085692, "sampling/importance_sampling_ratio/mean": 1.000886332988739, "sampling/importance_sampling_ratio/min": 0.7868674755096435, "sampling/sampling_logp_difference/max": 0.2611281394958496, "sampling/sampling_logp_difference/mean": 0.0200287614017725, "step": 19880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 241.15, "completions/mean_terminated_length": 241.15, "completions/min_length": 157.9, "completions/min_terminated_length": 157.9, "entropy": 1.1097195133566857, "epoch": 1.828629217615151, "frac_reward_zero_std": 0.5, "grad_norm": 0.08789180964231491, "kl": 0.009793851507129147, "learning_rate": 1.8104114116612912e-06, "loss": 0.0004, "num_tokens": 86630211.0, "reward": -0.425, "reward_std": 0.48816035985946654, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.4881603717803955, "sampling/importance_sampling_ratio/max": 1.2382986187934875, "sampling/importance_sampling_ratio/mean": 0.9994849562644958, "sampling/importance_sampling_ratio/min": 0.7745661795139313, "sampling/sampling_logp_difference/max": 0.2697045564651489, "sampling/sampling_logp_difference/mean": 0.0210368225350976, "step": 19890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.3, "completions/max_terminated_length": 318.4, "completions/mean_length": 221.6, "completions/mean_terminated_length": 218.38928680419923, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.0655255757272244, "epoch": 1.8295485887652845, "frac_reward_zero_std": 0.2, "grad_norm": 6.369986534118652, "kl": 0.009560704830801115, "learning_rate": 1.8079764342832387e-06, "loss": 0.0004, "num_tokens": 86675163.0, "reward": -0.18125, "reward_std": 0.733397263288498, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.7333972871303558, "sampling/importance_sampling_ratio/max": 1.3081321477890016, "sampling/importance_sampling_ratio/mean": 1.0004301726818086, "sampling/importance_sampling_ratio/min": 0.7854239344596863, "sampling/sampling_logp_difference/max": 0.2841362237930298, "sampling/sampling_logp_difference/mean": 0.021335683204233645, "step": 19900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 198.3, "completions/mean_terminated_length": 198.3, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.001520862430334, "epoch": 1.8304679599154179, "frac_reward_zero_std": 0.6, "grad_norm": 0.1042831763625145, "kl": 0.009252289470168762, "learning_rate": 1.8055421676688611e-06, "loss": 0.0006, "num_tokens": 86719539.0, "reward": -0.2, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.3169090390205382, "sampling/importance_sampling_ratio/mean": 1.0003787994384765, "sampling/importance_sampling_ratio/min": 0.776297652721405, "sampling/sampling_logp_difference/max": 0.29930944442749025, "sampling/sampling_logp_difference/mean": 0.02087666802108288, "step": 19910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.6, "completions/max_terminated_length": 286.6, "completions/mean_length": 210.55, "completions/mean_terminated_length": 210.55, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0333985418081284, "epoch": 1.8313873310655513, "frac_reward_zero_std": 0.4, "grad_norm": 0.08841577172279358, "kl": 0.009439031482907012, "learning_rate": 1.8031086143183451e-06, "loss": 0.0002, "num_tokens": 86759695.0, "reward": -0.2, "reward_std": 0.5731935203075409, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5731935322284698, "sampling/importance_sampling_ratio/max": 1.2658259272575378, "sampling/importance_sampling_ratio/mean": 0.9995207786560059, "sampling/importance_sampling_ratio/min": 0.7596469998359681, "sampling/sampling_logp_difference/max": 0.2959564089775085, "sampling/sampling_logp_difference/mean": 0.020856545120477677, "step": 19920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.4, "completions/max_terminated_length": 282.4, "completions/mean_length": 203.375, "completions/mean_terminated_length": 203.375, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.1276598922908305, "epoch": 1.8323067022156845, "frac_reward_zero_std": 0.5, "grad_norm": 5.752562046051025, "kl": 0.009213144218665547, "learning_rate": 1.8006757767311455e-06, "loss": 0.0004, "num_tokens": 86802893.0, "reward": 0.2, "reward_std": 0.4806115090847015, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.4806115210056305, "sampling/importance_sampling_ratio/max": 1.2630938053131104, "sampling/importance_sampling_ratio/mean": 1.0000311255455017, "sampling/importance_sampling_ratio/min": 0.7627857625484467, "sampling/sampling_logp_difference/max": 0.29539363980293276, "sampling/sampling_logp_difference/mean": 0.02187244612723589, "step": 19930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.8, "completions/max_terminated_length": 253.8, "completions/mean_length": 181.225, "completions/mean_terminated_length": 181.225, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0789805121719838, "epoch": 1.8332260733658177, "frac_reward_zero_std": 0.4, "grad_norm": 0.0988488644361496, "kl": 0.010397902838303708, "learning_rate": 1.7982436574059802e-06, "loss": 0.0006, "num_tokens": 86846183.0, "reward": 0.25, "reward_std": 0.5294508516788483, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.5294508635997772, "sampling/importance_sampling_ratio/max": 1.2987477779388428, "sampling/importance_sampling_ratio/mean": 1.0007956266403197, "sampling/importance_sampling_ratio/min": 0.8142839431762695, "sampling/sampling_logp_difference/max": 0.2646708548069, "sampling/sampling_logp_difference/mean": 0.020951716974377633, "step": 19940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 197.4625, "completions/mean_terminated_length": 197.4625, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.02247112095356, "epoch": 1.834145444515951, "frac_reward_zero_std": 0.7, "grad_norm": 0.12128690630197525, "kl": 0.01176636345917359, "learning_rate": 1.7958122588408316e-06, "loss": 0.0005, "num_tokens": 86886388.0, "reward": 0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2964435458183288, "sampling/importance_sampling_ratio/mean": 1.000288075208664, "sampling/importance_sampling_ratio/min": 0.752007007598877, "sampling/sampling_logp_difference/max": 0.31227121949195863, "sampling/sampling_logp_difference/mean": 0.02143199983984232, "step": 19950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 358.8, "completions/max_terminated_length": 358.5, "completions/mean_length": 257.4375, "completions/mean_terminated_length": 255.07321472167968, "completions/min_length": 160.4, "completions/min_terminated_length": 160.4, "entropy": 1.029621569812298, "epoch": 1.8350648156660845, "frac_reward_zero_std": 0.3, "grad_norm": 6.460198402404785, "kl": 0.010275021934648975, "learning_rate": 1.7933815835329398e-06, "loss": 0.0003, "num_tokens": 86933551.0, "reward": -0.1625, "reward_std": 0.6790185630321502, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.6790185809135437, "sampling/importance_sampling_ratio/max": 1.2376180171966553, "sampling/importance_sampling_ratio/mean": 1.0001295506954193, "sampling/importance_sampling_ratio/min": 0.777769124507904, "sampling/sampling_logp_difference/max": 0.25958376526832583, "sampling/sampling_logp_difference/mean": 0.020009639114141463, "step": 19960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.4, "completions/max_terminated_length": 317.4, "completions/mean_length": 217.6625, "completions/mean_terminated_length": 217.6625, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.1138310894370078, "epoch": 1.8359841868162177, "frac_reward_zero_std": 0.5, "grad_norm": 6.494725704193115, "kl": 0.009575955258333125, "learning_rate": 1.7909516339788064e-06, "loss": 0.0005, "num_tokens": 86977508.0, "reward": -0.2, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.2479289650917054, "sampling/importance_sampling_ratio/mean": 0.999570119380951, "sampling/importance_sampling_ratio/min": 0.7779994785785675, "sampling/sampling_logp_difference/max": 0.2659744143486023, "sampling/sampling_logp_difference/mean": 0.021975131519138814, "step": 19970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.4, "completions/max_terminated_length": 274.4, "completions/mean_length": 200.1375, "completions/mean_terminated_length": 200.1375, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.1100349314510822, "epoch": 1.8369035579663509, "frac_reward_zero_std": 0.5, "grad_norm": 0.0838751271367073, "kl": 0.00831122056406457, "learning_rate": 1.7885224126741799e-06, "loss": 0.0002, "num_tokens": 87019223.0, "reward": -0.25, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2452994465827942, "sampling/importance_sampling_ratio/mean": 0.9999521553516388, "sampling/importance_sampling_ratio/min": 0.7944052159786225, "sampling/sampling_logp_difference/max": 0.23595757484436036, "sampling/sampling_logp_difference/mean": 0.021435465104877947, "step": 19980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.0, "completions/max_terminated_length": 301.0, "completions/mean_length": 198.8875, "completions/mean_terminated_length": 198.8875, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0483787208795547, "epoch": 1.8378229291164843, "frac_reward_zero_std": 0.7, "grad_norm": 0.12192489206790924, "kl": 0.010376997795538046, "learning_rate": 1.7860939221140684e-06, "loss": 0.0005, "num_tokens": 87061030.0, "reward": -0.05, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2464868903160096, "sampling/importance_sampling_ratio/mean": 1.0000029802322388, "sampling/importance_sampling_ratio/min": 0.7905893325805664, "sampling/sampling_logp_difference/max": 0.24594535827636718, "sampling/sampling_logp_difference/mean": 0.020972936972975732, "step": 19990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.1, "completions/max_terminated_length": 293.1, "completions/mean_length": 208.125, "completions/mean_terminated_length": 208.125, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.1092155292630195, "epoch": 1.8387423002666177, "frac_reward_zero_std": 0.6, "grad_norm": 0.08822118490934372, "kl": 0.008987540454836563, "learning_rate": 1.783666164792725e-06, "loss": 0.0004, "num_tokens": 87103592.0, "reward": 0.175, "reward_std": 0.30471404194831847, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.30471404194831847, "sampling/importance_sampling_ratio/max": 1.259840977191925, "sampling/importance_sampling_ratio/mean": 0.9998475432395935, "sampling/importance_sampling_ratio/min": 0.790634286403656, "sampling/sampling_logp_difference/max": 0.24423525333404542, "sampling/sampling_logp_difference/mean": 0.022209726087749005, "step": 20000 }, { "epoch": 1.8387423002666177, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 300.9738219895288, "eval_completions/max_terminated_length": 300.2425828970332, "eval_completions/mean_length": 208.93979057591622, "eval_completions/mean_terminated_length": 208.74579288822193, "eval_completions/min_length": 134.7696335078534, "eval_completions/min_terminated_length": 134.7696335078534, "eval_entropy": 1.0826312527710646, "eval_frac_reward_zero_std": 0.5008726003490401, "eval_kl": 0.00923392935745405, "eval_loss": 0.000484466552734375, "eval_num_tokens": 87103592.0, "eval_reward": -0.10056719022687609, "eval_reward_std": 0.4379149943746197, "eval_rewards/reward_fn/mean": -0.10056719022687609, "eval_rewards/reward_fn/std": 0.43791500654520166, "eval_runtime": 1749.3141, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.272402344989943, "eval_sampling/importance_sampling_ratio/mean": 1.000040152622143, "eval_sampling/importance_sampling_ratio/min": 0.7695154669527192, "eval_sampling/sampling_logp_difference/max": 0.29817077309465323, "eval_sampling/sampling_logp_difference/mean": 0.021048632881118155, "eval_steps_per_second": 0.041, "step": 20000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.0, "completions/max_terminated_length": 256.0, "completions/mean_length": 183.575, "completions/mean_terminated_length": 183.575, "completions/min_length": 109.1, "completions/min_terminated_length": 109.1, "entropy": 1.0221968088299036, "epoch": 1.839661671416751, "frac_reward_zero_std": 0.6, "grad_norm": 7.3693671226501465, "kl": 0.009964083140948788, "learning_rate": 1.7812391432036523e-06, "loss": -0.0001, "num_tokens": 87141070.0, "reward": 0.075, "reward_std": 0.3518356800079346, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.35183569192886355, "sampling/importance_sampling_ratio/max": 1.2408892035484314, "sampling/importance_sampling_ratio/mean": 1.0002879798412323, "sampling/importance_sampling_ratio/min": 0.773862361907959, "sampling/sampling_logp_difference/max": 0.26613989770412444, "sampling/sampling_logp_difference/mean": 0.020911015383899213, "step": 20010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.7, "completions/max_terminated_length": 292.7, "completions/mean_length": 207.5125, "completions/mean_terminated_length": 207.5125, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.0230990171432495, "epoch": 1.8405810425668843, "frac_reward_zero_std": 0.2, "grad_norm": 6.100915431976318, "kl": 0.010938026226358488, "learning_rate": 1.778812859839595e-06, "loss": 0.0006, "num_tokens": 87186063.0, "reward": 0.01875, "reward_std": 0.6905405640602111, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.6905406057834625, "sampling/importance_sampling_ratio/max": 1.2899906992912293, "sampling/importance_sampling_ratio/mean": 0.9999219954013825, "sampling/importance_sampling_ratio/min": 0.7070562422275544, "sampling/sampling_logp_difference/max": 0.3986812949180603, "sampling/sampling_logp_difference/mean": 0.0208463491871953, "step": 20020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.3, "completions/max_terminated_length": 265.3, "completions/mean_length": 185.925, "completions/mean_terminated_length": 185.925, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.0744395691901445, "epoch": 1.8415004137170174, "frac_reward_zero_std": 0.5, "grad_norm": 8.506492614746094, "kl": 0.009171263538883067, "learning_rate": 1.7763873171925402e-06, "loss": 0.0006, "num_tokens": 87226529.0, "reward": -0.275, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.2726654648780822, "sampling/importance_sampling_ratio/mean": 1.0002079486846924, "sampling/importance_sampling_ratio/min": 0.7543794274330139, "sampling/sampling_logp_difference/max": 0.3141829252243042, "sampling/sampling_logp_difference/mean": 0.022049448266625405, "step": 20030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.9, "completions/max_terminated_length": 308.1, "completions/mean_length": 214.3, "completions/mean_terminated_length": 211.70357360839844, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0037650249898433, "epoch": 1.8424197848671509, "frac_reward_zero_std": 0.4, "grad_norm": 5.409912109375, "kl": 0.010864286753349005, "learning_rate": 1.773962517753715e-06, "loss": 0.0006, "num_tokens": 87267905.0, "reward": 0.09375, "reward_std": 0.6232083201408386, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.6232083559036254, "sampling/importance_sampling_ratio/max": 1.3305621027946473, "sampling/importance_sampling_ratio/mean": 1.0002251327037812, "sampling/importance_sampling_ratio/min": 0.7799508392810821, "sampling/sampling_logp_difference/max": 0.30615954995155337, "sampling/sampling_logp_difference/mean": 0.020314181596040724, "step": 20040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.2, "completions/max_terminated_length": 270.2, "completions/mean_length": 203.2, "completions/mean_terminated_length": 203.2, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0627994574606419, "epoch": 1.8433391560172843, "frac_reward_zero_std": 0.8, "grad_norm": 0.20447862148284912, "kl": 0.0091399236000143, "learning_rate": 1.7715384640135842e-06, "loss": 0.0006, "num_tokens": 87309593.0, "reward": -0.075, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2748384833335877, "sampling/importance_sampling_ratio/mean": 0.9998494803905487, "sampling/importance_sampling_ratio/min": 0.7510484993457794, "sampling/sampling_logp_difference/max": 0.3030635714530945, "sampling/sampling_logp_difference/mean": 0.02132609747350216, "step": 20050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.3, "completions/max_terminated_length": 261.3, "completions/mean_length": 187.4375, "completions/mean_terminated_length": 187.4375, "completions/min_length": 133.8, "completions/min_terminated_length": 133.8, "entropy": 1.0528856180608273, "epoch": 1.8442585271674174, "frac_reward_zero_std": 0.8, "grad_norm": 0.08907420933246613, "kl": 0.008855892438441515, "learning_rate": 1.7691151584618444e-06, "loss": 0.0003, "num_tokens": 87351484.0, "reward": 0.625, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.625, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2487425327301025, "sampling/importance_sampling_ratio/mean": 1.0001294314861298, "sampling/importance_sampling_ratio/min": 0.7785337209701538, "sampling/sampling_logp_difference/max": 0.26368495225906374, "sampling/sampling_logp_difference/mean": 0.02099918443709612, "step": 20060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 215.45, "completions/mean_terminated_length": 215.45, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.0600359365344048, "epoch": 1.8451778983175506, "frac_reward_zero_std": 0.5, "grad_norm": 0.08147872984409332, "kl": 0.008683448794181458, "learning_rate": 1.7666926035874238e-06, "loss": 0.0009, "num_tokens": 87395264.0, "reward": -0.175, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.2620323777198792, "sampling/importance_sampling_ratio/mean": 0.9999653577804566, "sampling/importance_sampling_ratio/min": 0.758871567249298, "sampling/sampling_logp_difference/max": 0.30626681447029114, "sampling/sampling_logp_difference/mean": 0.02101539820432663, "step": 20070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.4, "completions/max_terminated_length": 270.4, "completions/mean_length": 187.5, "completions/mean_terminated_length": 187.5, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.0612440563738346, "epoch": 1.846097269467684, "frac_reward_zero_std": 0.5, "grad_norm": 5.607916831970215, "kl": 0.008864289033226669, "learning_rate": 1.764270801878481e-06, "loss": 0.0003, "num_tokens": 87435616.0, "reward": -0.125, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.250066339969635, "sampling/importance_sampling_ratio/mean": 1.0000935792922974, "sampling/importance_sampling_ratio/min": 0.7582187533378602, "sampling/sampling_logp_difference/max": 0.2939316064119339, "sampling/sampling_logp_difference/mean": 0.021057703346014024, "step": 20080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 214.85, "completions/mean_terminated_length": 214.85, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.1345494121313096, "epoch": 1.8470166406178175, "frac_reward_zero_std": 0.5, "grad_norm": 7.745574951171875, "kl": 0.010626344196498393, "learning_rate": 1.7618497558224008e-06, "loss": 0.0005, "num_tokens": 87479604.0, "reward": 0.125, "reward_std": 0.4519508481025696, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4519508838653564, "sampling/importance_sampling_ratio/max": 1.3757115006446838, "sampling/importance_sampling_ratio/mean": 1.0004047632217408, "sampling/importance_sampling_ratio/min": 0.7143811240792275, "sampling/sampling_logp_difference/max": 0.46116451025009153, "sampling/sampling_logp_difference/mean": 0.022217263467609882, "step": 20090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.1, "completions/max_terminated_length": 310.5, "completions/mean_length": 226.45, "completions/mean_terminated_length": 223.66071472167968, "completions/min_length": 142.5, "completions/min_terminated_length": 142.5, "entropy": 1.0249069049954413, "epoch": 1.8479360117679509, "frac_reward_zero_std": 0.4, "grad_norm": 5.666406631469727, "kl": 0.008972450677538291, "learning_rate": 1.7594294679057915e-06, "loss": 0.0002, "num_tokens": 87527048.0, "reward": -0.05625, "reward_std": 0.5053922355175018, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.5053922533988953, "sampling/importance_sampling_ratio/max": 1.247917926311493, "sampling/importance_sampling_ratio/mean": 1.0002430975437164, "sampling/importance_sampling_ratio/min": 0.7952162206172944, "sampling/sampling_logp_difference/max": 0.23929100036621093, "sampling/sampling_logp_difference/mean": 0.019764078222215174, "step": 20100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 189.25, "completions/mean_terminated_length": 189.25, "completions/min_length": 116.3, "completions/min_terminated_length": 116.3, "entropy": 1.015947264432907, "epoch": 1.848855382918084, "frac_reward_zero_std": 0.3, "grad_norm": 0.08525297790765762, "kl": 0.008841281745117158, "learning_rate": 1.7570099406144827e-06, "loss": 0.0006, "num_tokens": 87569156.0, "reward": 0.0, "reward_std": 0.5496452271938324, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5496452391147614, "sampling/importance_sampling_ratio/max": 1.2889230489730834, "sampling/importance_sampling_ratio/mean": 1.0007056891918182, "sampling/importance_sampling_ratio/min": 0.7882094502449035, "sampling/sampling_logp_difference/max": 0.27510987520217894, "sampling/sampling_logp_difference/mean": 0.020902367308735847, "step": 20110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.1, "completions/max_terminated_length": 252.1, "completions/mean_length": 188.45, "completions/mean_terminated_length": 188.45, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.1382680527865887, "epoch": 1.8497747540682172, "frac_reward_zero_std": 0.4, "grad_norm": 7.346404075622559, "kl": 0.009250733756925911, "learning_rate": 1.7545911764335221e-06, "loss": 0.0004, "num_tokens": 87608736.0, "reward": 0.2, "reward_std": 0.5588710367679596, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5588710486888886, "sampling/importance_sampling_ratio/max": 1.2919618010520935, "sampling/importance_sampling_ratio/mean": 0.9999785661697388, "sampling/importance_sampling_ratio/min": 0.7719502627849579, "sampling/sampling_logp_difference/max": 0.29712305068969724, "sampling/sampling_logp_difference/mean": 0.022386788204312325, "step": 20120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 346.9, "completions/max_terminated_length": 346.9, "completions/mean_length": 217.825, "completions/mean_terminated_length": 217.825, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.1038315460085868, "epoch": 1.8506941252183506, "frac_reward_zero_std": 0.2, "grad_norm": 14.996846199035645, "kl": 0.011133792053442448, "learning_rate": 1.7521731778471756e-06, "loss": 0.0008, "num_tokens": 87653434.0, "reward": -0.18125, "reward_std": 0.7976230561733246, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.7976230919361115, "sampling/importance_sampling_ratio/max": 1.3317447900772095, "sampling/importance_sampling_ratio/mean": 0.9999539852142334, "sampling/importance_sampling_ratio/min": 0.7915409982204438, "sampling/sampling_logp_difference/max": 0.2917770266532898, "sampling/sampling_logp_difference/mean": 0.021410004794597627, "step": 20130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.8, "completions/max_terminated_length": 287.8, "completions/mean_length": 208.9625, "completions/mean_terminated_length": 208.9625, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.10024204403162, "epoch": 1.851613496368484, "frac_reward_zero_std": 0.4, "grad_norm": 8.315272331237793, "kl": 0.0095362629275769, "learning_rate": 1.749755947338921e-06, "loss": 0.0004, "num_tokens": 87700399.0, "reward": -0.225, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2948185086250306, "sampling/importance_sampling_ratio/mean": 0.9994288980960846, "sampling/importance_sampling_ratio/min": 0.7733211517333984, "sampling/sampling_logp_difference/max": 0.28738372325897216, "sampling/sampling_logp_difference/mean": 0.021974450908601284, "step": 20140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.9, "completions/max_terminated_length": 322.9, "completions/mean_length": 219.875, "completions/mean_terminated_length": 219.875, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.0570880852639675, "epoch": 1.8525328675186172, "frac_reward_zero_std": 0.3, "grad_norm": 0.09115266054868698, "kl": 0.008656574838096277, "learning_rate": 1.7473394873914484e-06, "loss": 0.0, "num_tokens": 87743429.0, "reward": 0.15, "reward_std": 0.6548320174217224, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6548320412635803, "sampling/importance_sampling_ratio/max": 1.2561904549598695, "sampling/importance_sampling_ratio/mean": 0.9997471690177917, "sampling/importance_sampling_ratio/min": 0.799795788526535, "sampling/sampling_logp_difference/max": 0.24281709194183348, "sampling/sampling_logp_difference/mean": 0.021518039889633654, "step": 20150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.1, "completions/max_terminated_length": 266.1, "completions/mean_length": 204.9, "completions/mean_terminated_length": 204.9, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.1374446295201779, "epoch": 1.8534522386687504, "frac_reward_zero_std": 0.3, "grad_norm": 4.701115131378174, "kl": 0.009545196819817647, "learning_rate": 1.7449238004866553e-06, "loss": 0.0006, "num_tokens": 87789277.0, "reward": -0.1, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2589873313903808, "sampling/importance_sampling_ratio/mean": 0.9996894896030426, "sampling/importance_sampling_ratio/min": 0.7680136620998382, "sampling/sampling_logp_difference/max": 0.2853203773498535, "sampling/sampling_logp_difference/mean": 0.022055899351835252, "step": 20160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 197.7375, "completions/mean_terminated_length": 197.7375, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.0181898899376391, "epoch": 1.8543716098188838, "frac_reward_zero_std": 0.3, "grad_norm": 6.083782196044922, "kl": 0.009566861207713374, "learning_rate": 1.7425088891056455e-06, "loss": -0.0001, "num_tokens": 87831208.0, "reward": -0.375, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.3410032391548157, "sampling/importance_sampling_ratio/mean": 1.00019513964653, "sampling/importance_sampling_ratio/min": 0.7753595352172852, "sampling/sampling_logp_difference/max": 0.3358783185482025, "sampling/sampling_logp_difference/mean": 0.02054791059345007, "step": 20170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.2, "completions/max_terminated_length": 280.2, "completions/mean_length": 201.175, "completions/mean_terminated_length": 201.175, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.057376790046692, "epoch": 1.8552909809690172, "frac_reward_zero_std": 0.5, "grad_norm": 7.134274959564209, "kl": 0.009176588745322078, "learning_rate": 1.7400947557287262e-06, "loss": 0.0006, "num_tokens": 87870638.0, "reward": 0.51875, "reward_std": 0.4346815586090088, "rewards/reward_fn/mean": 0.51875, "rewards/reward_fn/std": 0.43468157649040223, "sampling/importance_sampling_ratio/max": 1.25725599527359, "sampling/importance_sampling_ratio/mean": 0.9998452544212342, "sampling/importance_sampling_ratio/min": 0.788289749622345, "sampling/sampling_logp_difference/max": 0.2535882472991943, "sampling/sampling_logp_difference/mean": 0.021206096187233923, "step": 20180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 212.775, "completions/mean_terminated_length": 212.775, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0680035710334779, "epoch": 1.8562103521191506, "frac_reward_zero_std": 0.3, "grad_norm": 5.810356616973877, "kl": 0.008495552380918526, "learning_rate": 1.7376814028354089e-06, "loss": 0.0006, "num_tokens": 87913796.0, "reward": -0.025, "reward_std": 0.6152592301368713, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.6152592420578002, "sampling/importance_sampling_ratio/max": 1.2284470677375794, "sampling/importance_sampling_ratio/mean": 0.9998501062393188, "sampling/importance_sampling_ratio/min": 0.7767836153507233, "sampling/sampling_logp_difference/max": 0.2652864456176758, "sampling/sampling_logp_difference/mean": 0.020117553789168595, "step": 20190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 191.025, "completions/mean_terminated_length": 191.025, "completions/min_length": 113.5, "completions/min_terminated_length": 113.5, "entropy": 1.0570587053894998, "epoch": 1.8571297232692838, "frac_reward_zero_std": 0.5, "grad_norm": 6.014204502105713, "kl": 0.010126119520282372, "learning_rate": 1.7352688329043959e-06, "loss": 0.0003, "num_tokens": 87955662.0, "reward": 0.025, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.3165456771850585, "sampling/importance_sampling_ratio/mean": 1.0000326931476593, "sampling/importance_sampling_ratio/min": 0.7631303608417511, "sampling/sampling_logp_difference/max": 0.31991431713104246, "sampling/sampling_logp_difference/mean": 0.020746999979019166, "step": 20200 }, { "epoch": 1.8571297232692838, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 301.2233856893543, "eval_completions/max_terminated_length": 298.0331588132635, "eval_completions/mean_length": 208.98276614310646, "eval_completions/mean_terminated_length": 208.27458891277746, "eval_completions/min_length": 132.73821989528795, "eval_completions/min_terminated_length": 132.73821989528795, "eval_entropy": 1.0953978798360309, "eval_frac_reward_zero_std": 0.49214659685863876, "eval_kl": 0.009756244039634343, "eval_loss": 0.0004874148580711335, "eval_num_tokens": 87955662.0, "eval_reward": -0.10405759162303665, "eval_reward_std": 0.44957775288435387, "eval_rewards/reward_fn/mean": -0.10405759162303665, "eval_rewards/reward_fn/std": 0.44957776765548746, "eval_runtime": 1751.2082, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2623324662603008, "eval_sampling/importance_sampling_ratio/mean": 0.99995872362746, "eval_sampling/importance_sampling_ratio/min": 0.7682981387846965, "eval_sampling/sampling_logp_difference/max": 0.2915473711948328, "eval_sampling/sampling_logp_difference/mean": 0.020995629943231117, "eval_steps_per_second": 0.041, "step": 20200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.7, "completions/max_terminated_length": 327.7, "completions/mean_length": 223.325, "completions/mean_terminated_length": 223.325, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.0239561855793, "epoch": 1.858049094419417, "frac_reward_zero_std": 0.2, "grad_norm": 6.435481548309326, "kl": 0.009233083273284138, "learning_rate": 1.732857048413593e-06, "loss": 0.0004, "num_tokens": 88001280.0, "reward": 0.0, "reward_std": 0.6893488764762878, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.6893489003181458, "sampling/importance_sampling_ratio/max": 1.2612792372703552, "sampling/importance_sampling_ratio/mean": 1.0002092003822327, "sampling/importance_sampling_ratio/min": 0.781635582447052, "sampling/sampling_logp_difference/max": 0.26169748306274415, "sampling/sampling_logp_difference/mean": 0.02071175053715706, "step": 20210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.3, "completions/max_terminated_length": 300.3, "completions/mean_length": 211.575, "completions/mean_terminated_length": 211.575, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.1608866557478905, "epoch": 1.8589684655695504, "frac_reward_zero_std": 0.4, "grad_norm": 6.1961894035339355, "kl": 0.010919949025264941, "learning_rate": 1.7304460518400956e-06, "loss": 0.0003, "num_tokens": 88049686.0, "reward": -0.325, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.3043140172958374, "sampling/importance_sampling_ratio/mean": 0.9999923229217529, "sampling/importance_sampling_ratio/min": 0.7667559742927551, "sampling/sampling_logp_difference/max": 0.3225338578224182, "sampling/sampling_logp_difference/mean": 0.02199530079960823, "step": 20220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.8, "completions/max_terminated_length": 325.8, "completions/mean_length": 234.525, "completions/mean_terminated_length": 234.525, "completions/min_length": 153.3, "completions/min_terminated_length": 153.3, "entropy": 1.134981994330883, "epoch": 1.8598878367196838, "frac_reward_zero_std": 0.8, "grad_norm": 6.4393696784973145, "kl": 0.009440232755150647, "learning_rate": 1.7280358456601905e-06, "loss": 0.0002, "num_tokens": 88097944.0, "reward": -0.53125, "reward_std": 0.16787903904914855, "rewards/reward_fn/mean": -0.53125, "rewards/reward_fn/std": 0.16787904500961304, "sampling/importance_sampling_ratio/max": 1.3789947032928467, "sampling/importance_sampling_ratio/mean": 1.0002973437309266, "sampling/importance_sampling_ratio/min": 0.7589318156242371, "sampling/sampling_logp_difference/max": 0.3456646800041199, "sampling/sampling_logp_difference/mean": 0.021890461258590222, "step": 20230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 206.975, "completions/mean_terminated_length": 206.975, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.1140441939234733, "epoch": 1.860807207869817, "frac_reward_zero_std": 0.8, "grad_norm": 0.1329960972070694, "kl": 0.008711353276157751, "learning_rate": 1.7256264323493522e-06, "loss": 0.0004, "num_tokens": 88139502.0, "reward": -0.275, "reward_std": 0.1960918426513672, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.19609185457229614, "sampling/importance_sampling_ratio/max": 1.3089966416358947, "sampling/importance_sampling_ratio/mean": 0.9995012283325195, "sampling/importance_sampling_ratio/min": 0.7812541365623474, "sampling/sampling_logp_difference/max": 0.30868377685546877, "sampling/sampling_logp_difference/mean": 0.021693025156855582, "step": 20240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.1, "completions/max_terminated_length": 299.1, "completions/mean_length": 210.05, "completions/mean_terminated_length": 210.05, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.121271448582411, "epoch": 1.8617265790199502, "frac_reward_zero_std": 0.6, "grad_norm": 0.09850866347551346, "kl": 0.010216849588323385, "learning_rate": 1.723217814382241e-06, "loss": 0.0007, "num_tokens": 88183602.0, "reward": -0.275, "reward_std": 0.391424161195755, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.391424161195755, "sampling/importance_sampling_ratio/max": 1.273371982574463, "sampling/importance_sampling_ratio/mean": 1.0004603207111358, "sampling/importance_sampling_ratio/min": 0.7823164224624634, "sampling/sampling_logp_difference/max": 0.26414896845817565, "sampling/sampling_logp_difference/mean": 0.021961848996579648, "step": 20250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.1, "completions/max_terminated_length": 308.1, "completions/mean_length": 222.15, "completions/mean_terminated_length": 222.15, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.100114855915308, "epoch": 1.8626459501700836, "frac_reward_zero_std": 0.5, "grad_norm": 0.07596872746944427, "kl": 0.009223080714582466, "learning_rate": 1.720809994232701e-06, "loss": 0.0004, "num_tokens": 88234630.0, "reward": -0.45, "reward_std": 0.4082238733768463, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.40822388529777526, "sampling/importance_sampling_ratio/max": 1.3538416981697083, "sampling/importance_sampling_ratio/mean": 0.9999862551689148, "sampling/importance_sampling_ratio/min": 0.7725146353244782, "sampling/sampling_logp_difference/max": 0.34583571553230286, "sampling/sampling_logp_difference/mean": 0.021515064314007758, "step": 20260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 220.7875, "completions/mean_terminated_length": 220.7875, "completions/min_length": 162.1, "completions/min_terminated_length": 162.1, "entropy": 1.1031286373734475, "epoch": 1.863565321320217, "frac_reward_zero_std": 0.4, "grad_norm": 0.12049777805805206, "kl": 0.010494025523075833, "learning_rate": 1.7184029743737574e-06, "loss": 0.0008, "num_tokens": 88282317.0, "reward": 0.05, "reward_std": 0.4898623704910278, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.48986239433288575, "sampling/importance_sampling_ratio/max": 1.4065135955810546, "sampling/importance_sampling_ratio/mean": 1.0000886261463164, "sampling/importance_sampling_ratio/min": 0.760343199968338, "sampling/sampling_logp_difference/max": 0.38438678681850436, "sampling/sampling_logp_difference/mean": 0.021849595196545123, "step": 20270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.8, "completions/max_terminated_length": 334.8, "completions/mean_length": 235.0, "completions/mean_terminated_length": 235.0, "completions/min_length": 166.6, "completions/min_terminated_length": 166.6, "entropy": 1.0888990134000778, "epoch": 1.8644846924703504, "frac_reward_zero_std": 0.7, "grad_norm": 0.07005680352449417, "kl": 0.009404216788243502, "learning_rate": 1.715996757277612e-06, "loss": 0.0006, "num_tokens": 88328301.0, "reward": -0.3, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.269662868976593, "sampling/importance_sampling_ratio/mean": 1.0001089155673981, "sampling/importance_sampling_ratio/min": 0.7943276405334473, "sampling/sampling_logp_difference/max": 0.2508926272392273, "sampling/sampling_logp_difference/mean": 0.021378753520548344, "step": 20280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 188.775, "completions/mean_terminated_length": 188.775, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.0592200368642808, "epoch": 1.8654040636204836, "frac_reward_zero_std": 0.9, "grad_norm": 5.538166046142578, "kl": 0.009978621843038127, "learning_rate": 1.7135913454156417e-06, "loss": 0.0004, "num_tokens": 88368923.0, "reward": 0.1, "reward_std": 0.10690449476242066, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.10690449476242066, "sampling/importance_sampling_ratio/max": 1.2584319114685059, "sampling/importance_sampling_ratio/mean": 1.0000768065452577, "sampling/importance_sampling_ratio/min": 0.7967956125736236, "sampling/sampling_logp_difference/max": 0.249363511800766, "sampling/sampling_logp_difference/mean": 0.02143198810517788, "step": 20290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.4, "completions/max_terminated_length": 299.4, "completions/mean_length": 203.75, "completions/mean_terminated_length": 203.75, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.1008629567921162, "epoch": 1.8663234347706168, "frac_reward_zero_std": 0.7, "grad_norm": 6.7506489753723145, "kl": 0.008222161847515962, "learning_rate": 1.7111867412583983e-06, "loss": 0.0006, "num_tokens": 88413111.0, "reward": -0.4, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.261437678337097, "sampling/importance_sampling_ratio/mean": 0.9989443719387054, "sampling/importance_sampling_ratio/min": 0.7692138731479645, "sampling/sampling_logp_difference/max": 0.2733913898468018, "sampling/sampling_logp_difference/mean": 0.021696371398866175, "step": 20300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.0, "completions/max_terminated_length": 260.0, "completions/mean_length": 196.0, "completions/mean_terminated_length": 196.0, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 0.9646583691239357, "epoch": 1.8672428059207502, "frac_reward_zero_std": 0.3, "grad_norm": 4.372546672821045, "kl": 0.008418721970519983, "learning_rate": 1.7087829472756028e-06, "loss": 0.0008, "num_tokens": 88453575.0, "reward": -0.125, "reward_std": 0.5715165615081788, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5715165734291077, "sampling/importance_sampling_ratio/max": 1.2645981669425965, "sampling/importance_sampling_ratio/mean": 0.9996312856674194, "sampling/importance_sampling_ratio/min": 0.7719610095024109, "sampling/sampling_logp_difference/max": 0.29101386070251467, "sampling/sampling_logp_difference/mean": 0.019814768619835377, "step": 20310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.4, "completions/max_terminated_length": 303.4, "completions/mean_length": 202.7375, "completions/mean_terminated_length": 202.7375, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 1.1840962275862694, "epoch": 1.8681621770708836, "frac_reward_zero_std": 0.6, "grad_norm": 0.08387463539838791, "kl": 0.009122748250956648, "learning_rate": 1.7063799659361446e-06, "loss": 0.0002, "num_tokens": 88500322.0, "reward": -0.6, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2745587468147277, "sampling/importance_sampling_ratio/mean": 1.0004940330982208, "sampling/importance_sampling_ratio/min": 0.7406959116458893, "sampling/sampling_logp_difference/max": 0.31628262996673584, "sampling/sampling_logp_difference/mean": 0.022918219305574895, "step": 20320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.0, "completions/max_terminated_length": 296.0, "completions/mean_length": 215.7375, "completions/mean_terminated_length": 215.7375, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.1231898427009583, "epoch": 1.8690815482210168, "frac_reward_zero_std": 0.5, "grad_norm": 0.0756610557436943, "kl": 0.008293533971300348, "learning_rate": 1.7039777997080776e-06, "loss": 0.0005, "num_tokens": 88544365.0, "reward": -0.3125, "reward_std": 0.3639739915728569, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.3639739975333214, "sampling/importance_sampling_ratio/max": 1.2700671434402466, "sampling/importance_sampling_ratio/mean": 0.9999033749103546, "sampling/importance_sampling_ratio/min": 0.8065458357334137, "sampling/sampling_logp_difference/max": 0.25233614444732666, "sampling/sampling_logp_difference/mean": 0.021471801958978177, "step": 20330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.4, "completions/max_terminated_length": 305.4, "completions/mean_length": 209.65, "completions/mean_terminated_length": 209.65, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.112915751338005, "epoch": 1.87000091937115, "frac_reward_zero_std": 0.6, "grad_norm": 0.07976461946964264, "kl": 0.009138239367166534, "learning_rate": 1.7015764510586182e-06, "loss": 0.0005, "num_tokens": 88586649.0, "reward": -0.05, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.2543617963790894, "sampling/importance_sampling_ratio/mean": 0.999677836894989, "sampling/importance_sampling_ratio/min": 0.7933312594890595, "sampling/sampling_logp_difference/max": 0.24900709390640258, "sampling/sampling_logp_difference/mean": 0.021327178180217742, "step": 20340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 198.9, "completions/mean_terminated_length": 198.9, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.0081454358994961, "epoch": 1.8709202905212834, "frac_reward_zero_std": 0.4, "grad_norm": 7.0578227043151855, "kl": 0.00833753333427012, "learning_rate": 1.699175922454146e-06, "loss": 0.0004, "num_tokens": 88628449.0, "reward": -0.45, "reward_std": 0.5950491607189179, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.5950491964817047, "sampling/importance_sampling_ratio/max": 1.2667735934257507, "sampling/importance_sampling_ratio/mean": 0.9995591044425964, "sampling/importance_sampling_ratio/min": 0.7816056191921235, "sampling/sampling_logp_difference/max": 0.27672737240791323, "sampling/sampling_logp_difference/mean": 0.02080475352704525, "step": 20350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 368.7, "completions/max_terminated_length": 368.7, "completions/mean_length": 226.825, "completions/mean_terminated_length": 226.825, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.1102279648184776, "epoch": 1.8718396616714168, "frac_reward_zero_std": 0.6, "grad_norm": 7.182878017425537, "kl": 0.00908480930083897, "learning_rate": 1.6967762163601954e-06, "loss": 0.0008, "num_tokens": 88674491.0, "reward": -0.35, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2462942004203796, "sampling/importance_sampling_ratio/mean": 1.000103050470352, "sampling/importance_sampling_ratio/min": 0.7698765695095062, "sampling/sampling_logp_difference/max": 0.2754038095474243, "sampling/sampling_logp_difference/mean": 0.02082666605710983, "step": 20360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.4, "completions/max_terminated_length": 293.2, "completions/mean_length": 213.8, "completions/mean_terminated_length": 210.48035888671876, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.0526888694614172, "epoch": 1.8727590328215502, "frac_reward_zero_std": 0.3, "grad_norm": 4.697237968444824, "kl": 0.008641360685578548, "learning_rate": 1.6943773352414572e-06, "loss": -0.0001, "num_tokens": 88718563.0, "reward": -0.50625, "reward_std": 0.6770090699195862, "rewards/reward_fn/mean": -0.50625, "rewards/reward_fn/std": 0.6770090937614441, "sampling/importance_sampling_ratio/max": 1.2341258525848389, "sampling/importance_sampling_ratio/mean": 0.9998085498809814, "sampling/importance_sampling_ratio/min": 0.7598046660423279, "sampling/sampling_logp_difference/max": 0.28722788095474244, "sampling/sampling_logp_difference/mean": 0.020436633564531802, "step": 20370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 336.0, "completions/max_terminated_length": 334.4, "completions/mean_length": 234.5625, "completions/mean_terminated_length": 230.1666687011719, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.0392580408602954, "epoch": 1.8736784039716834, "frac_reward_zero_std": 0.5, "grad_norm": 0.14491429924964905, "kl": 0.010366550483740865, "learning_rate": 1.6919792815617736e-06, "loss": 0.0005, "num_tokens": 88763992.0, "reward": -0.2625, "reward_std": 0.3980934679508209, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.3980934679508209, "sampling/importance_sampling_ratio/max": 1.2139011859893798, "sampling/importance_sampling_ratio/mean": 1.0002495884895324, "sampling/importance_sampling_ratio/min": 0.7760718524456024, "sampling/sampling_logp_difference/max": 0.25753200650215147, "sampling/sampling_logp_difference/mean": 0.02106929300352931, "step": 20380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.2, "completions/max_terminated_length": 285.2, "completions/mean_length": 210.5875, "completions/mean_terminated_length": 210.5875, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0929462686181068, "epoch": 1.8745977751218166, "frac_reward_zero_std": 0.7, "grad_norm": 0.09339522570371628, "kl": 0.008058328519109637, "learning_rate": 1.6895820577841388e-06, "loss": 0.0004, "num_tokens": 88807463.0, "reward": 0.25, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2596887707710267, "sampling/importance_sampling_ratio/mean": 1.0000726878643036, "sampling/importance_sampling_ratio/min": 0.7719083726406097, "sampling/sampling_logp_difference/max": 0.27993819713592527, "sampling/sampling_logp_difference/mean": 0.02130321953445673, "step": 20390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.5, "completions/max_terminated_length": 270.5, "completions/mean_length": 190.8875, "completions/mean_terminated_length": 190.8875, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 1.1053206704556942, "epoch": 1.87551714627195, "frac_reward_zero_std": 0.4, "grad_norm": 5.124027252197266, "kl": 0.008667991688707844, "learning_rate": 1.6871856663706932e-06, "loss": 0.0001, "num_tokens": 88849966.0, "reward": 0.075, "reward_std": 0.544532859325409, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5445328950881958, "sampling/importance_sampling_ratio/max": 1.3104388475418092, "sampling/importance_sampling_ratio/mean": 0.9998076677322387, "sampling/importance_sampling_ratio/min": 0.7941719055175781, "sampling/sampling_logp_difference/max": 0.28517951965332033, "sampling/sampling_logp_difference/mean": 0.022116474248468874, "step": 20400 }, { "epoch": 1.87551714627195, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 298.1326352530541, "eval_completions/max_terminated_length": 295.87085514834206, "eval_completions/mean_length": 207.93411867364748, "eval_completions/mean_terminated_length": 207.49682135190847, "eval_completions/min_length": 134.0628272251309, "eval_completions/min_terminated_length": 134.0628272251309, "eval_entropy": 1.0868816074068426, "eval_frac_reward_zero_std": 0.5095986038394416, "eval_kl": 0.009022774705484536, "eval_loss": 0.00046327628660947084, "eval_num_tokens": 88849966.0, "eval_reward": -0.10427574171029669, "eval_reward_std": 0.4259197399098212, "eval_rewards/reward_fn/mean": -0.10427574171029669, "eval_rewards/reward_fn/std": 0.4259197536407341, "eval_runtime": 1736.2375, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2665896072437626, "eval_sampling/importance_sampling_ratio/mean": 0.9999986207714048, "eval_sampling/importance_sampling_ratio/min": 0.7711777566407155, "eval_sampling/sampling_logp_difference/max": 0.28651309078380494, "eval_sampling/sampling_logp_difference/mean": 0.020948488900785774, "eval_steps_per_second": 0.041, "step": 20400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.8, "completions/max_terminated_length": 282.8, "completions/mean_length": 209.8875, "completions/mean_terminated_length": 209.8875, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.002967816963792, "epoch": 1.8764365174220834, "frac_reward_zero_std": 0.5, "grad_norm": 4.790226459503174, "kl": 0.009555617795558646, "learning_rate": 1.6847901097827247e-06, "loss": 0.0003, "num_tokens": 88891373.0, "reward": 0.1, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.239779770374298, "sampling/importance_sampling_ratio/mean": 0.9999265670776367, "sampling/importance_sampling_ratio/min": 0.7806073784828186, "sampling/sampling_logp_difference/max": 0.26726412773132324, "sampling/sampling_logp_difference/mean": 0.019577317871153355, "step": 20410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 207.375, "completions/mean_terminated_length": 207.375, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.1007586590945722, "epoch": 1.8773558885722166, "frac_reward_zero_std": 0.5, "grad_norm": 4.745883941650391, "kl": 0.008485015723272227, "learning_rate": 1.682395390480659e-06, "loss": 0.0003, "num_tokens": 88935563.0, "reward": -0.4, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.242416751384735, "sampling/importance_sampling_ratio/mean": 0.9998814225196838, "sampling/importance_sampling_ratio/min": 0.7810950100421905, "sampling/sampling_logp_difference/max": 0.26420505046844484, "sampling/sampling_logp_difference/mean": 0.02185949366539717, "step": 20420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.5, "completions/max_terminated_length": 332.5, "completions/mean_length": 224.475, "completions/mean_terminated_length": 224.475, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.1179509252309798, "epoch": 1.8782752597223498, "frac_reward_zero_std": 0.4, "grad_norm": 5.904630661010742, "kl": 0.00954589513130486, "learning_rate": 1.680001510924067e-06, "loss": 0.0004, "num_tokens": 88982561.0, "reward": 0.125, "reward_std": 0.5260561883449555, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5260562121868133, "sampling/importance_sampling_ratio/max": 1.25164852142334, "sampling/importance_sampling_ratio/mean": 0.9993228077888489, "sampling/importance_sampling_ratio/min": 0.7452822506427765, "sampling/sampling_logp_difference/max": 0.307225775718689, "sampling/sampling_logp_difference/mean": 0.02180945351719856, "step": 20430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.2, "completions/max_terminated_length": 328.2, "completions/mean_length": 222.925, "completions/mean_terminated_length": 222.925, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.1112506099045276, "epoch": 1.8791946308724832, "frac_reward_zero_std": 0.4, "grad_norm": 5.3852410316467285, "kl": 0.009366459288867191, "learning_rate": 1.6776084735716538e-06, "loss": 0.0002, "num_tokens": 89025723.0, "reward": -0.425, "reward_std": 0.5622657001018524, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5622657001018524, "sampling/importance_sampling_ratio/max": 1.2681916117668153, "sampling/importance_sampling_ratio/mean": 0.9999255895614624, "sampling/importance_sampling_ratio/min": 0.7307508915662766, "sampling/sampling_logp_difference/max": 0.3414097398519516, "sampling/sampling_logp_difference/mean": 0.02152478639036417, "step": 20440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.1, "completions/max_terminated_length": 247.1, "completions/mean_length": 181.3375, "completions/mean_terminated_length": 181.3375, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.008895481005311, "epoch": 1.8801140020226166, "frac_reward_zero_std": 0.2, "grad_norm": 0.089714914560318, "kl": 0.012520854669855908, "learning_rate": 1.6752162808812614e-06, "loss": 0.0006, "num_tokens": 89065878.0, "reward": 0.0, "reward_std": 0.7002923905849456, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.7002924025058747, "sampling/importance_sampling_ratio/max": 1.2361097931861877, "sampling/importance_sampling_ratio/mean": 1.0000997185707092, "sampling/importance_sampling_ratio/min": 0.7885187208652497, "sampling/sampling_logp_difference/max": 0.24684042930603028, "sampling/sampling_logp_difference/mean": 0.021063679922372104, "step": 20450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.5, "completions/max_terminated_length": 304.4, "completions/mean_length": 223.8625, "completions/mean_terminated_length": 220.94285888671874, "completions/min_length": 141.1, "completions/min_terminated_length": 141.1, "entropy": 1.0838728077709674, "epoch": 1.88103337317275, "frac_reward_zero_std": 0.6, "grad_norm": 0.10485468059778214, "kl": 0.009439431579085066, "learning_rate": 1.6728249353098633e-06, "loss": 0.0004, "num_tokens": 89110971.0, "reward": -0.10625, "reward_std": 0.2516810342669487, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.2516810342669487, "sampling/importance_sampling_ratio/max": 1.2875593066215516, "sampling/importance_sampling_ratio/mean": 1.0001111805438996, "sampling/importance_sampling_ratio/min": 0.7702657461166382, "sampling/sampling_logp_difference/max": 0.2781583309173584, "sampling/sampling_logp_difference/mean": 0.022053110413253307, "step": 20460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.3, "completions/max_terminated_length": 332.3, "completions/mean_length": 231.7, "completions/mean_terminated_length": 231.7, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.0656858846545219, "epoch": 1.8819527443228832, "frac_reward_zero_std": 0.4, "grad_norm": 5.266294479370117, "kl": 0.011895505295251496, "learning_rate": 1.6704344393135625e-06, "loss": 0.0003, "num_tokens": 89158627.0, "reward": -0.25, "reward_std": 0.5336050391197205, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5336050629615784, "sampling/importance_sampling_ratio/max": 1.265998935699463, "sampling/importance_sampling_ratio/mean": 1.0000207781791688, "sampling/importance_sampling_ratio/min": 0.782886016368866, "sampling/sampling_logp_difference/max": 0.27137476205825806, "sampling/sampling_logp_difference/mean": 0.02074230760335922, "step": 20470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.3, "completions/max_terminated_length": 311.0, "completions/mean_length": 227.2375, "completions/mean_terminated_length": 223.0250015258789, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.0387359969317913, "epoch": 1.8828721154730164, "frac_reward_zero_std": 0.5, "grad_norm": 0.08646265417337418, "kl": 0.010149005256243982, "learning_rate": 1.6680447953475898e-06, "loss": 0.0006, "num_tokens": 89204782.0, "reward": 0.04375, "reward_std": 0.44560937881469725, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.44560940861701964, "sampling/importance_sampling_ratio/max": 1.2429385423660277, "sampling/importance_sampling_ratio/mean": 0.9996057331562043, "sampling/importance_sampling_ratio/min": 0.7706996977329255, "sampling/sampling_logp_difference/max": 0.267304515838623, "sampling/sampling_logp_difference/mean": 0.020917312242090702, "step": 20480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 202.2625, "completions/mean_terminated_length": 202.2625, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.025340597331524, "epoch": 1.8837914866231498, "frac_reward_zero_std": 0.6, "grad_norm": 0.13428686559200287, "kl": 0.008716269780416041, "learning_rate": 1.6656560058663018e-06, "loss": 0.0003, "num_tokens": 89254691.0, "reward": -0.325, "reward_std": 0.37031235098838805, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.370312374830246, "sampling/importance_sampling_ratio/max": 1.3193211913108827, "sampling/importance_sampling_ratio/mean": 0.9996734678745269, "sampling/importance_sampling_ratio/min": 0.7322959154844284, "sampling/sampling_logp_difference/max": 0.38047208189964293, "sampling/sampling_logp_difference/mean": 0.0214841291308403, "step": 20490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.1, "completions/max_terminated_length": 269.1, "completions/mean_length": 194.0625, "completions/mean_terminated_length": 194.0625, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.079616180062294, "epoch": 1.8847108577732832, "frac_reward_zero_std": 0.6, "grad_norm": 0.09561283141374588, "kl": 0.008057070188806392, "learning_rate": 1.6632680733231777e-06, "loss": 0.0007, "num_tokens": 89295792.0, "reward": 0.1, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2522526621818542, "sampling/importance_sampling_ratio/mean": 1.0008070170879364, "sampling/importance_sampling_ratio/min": 0.7706151723861694, "sampling/sampling_logp_difference/max": 0.28009009957313535, "sampling/sampling_logp_difference/mean": 0.021658307686448098, "step": 20500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.6, "completions/max_terminated_length": 283.6, "completions/mean_length": 199.725, "completions/mean_terminated_length": 199.725, "completions/min_length": 114.8, "completions/min_terminated_length": 114.8, "entropy": 1.0647620499134063, "epoch": 1.8856302289234164, "frac_reward_zero_std": 0.4, "grad_norm": 5.137349605560303, "kl": 0.009535978912026621, "learning_rate": 1.6608810001708137e-06, "loss": 0.0004, "num_tokens": 89343418.0, "reward": 0.175, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2626010298728942, "sampling/importance_sampling_ratio/mean": 0.9999798655509948, "sampling/importance_sampling_ratio/min": 0.7640774488449097, "sampling/sampling_logp_difference/max": 0.29070059061050413, "sampling/sampling_logp_difference/mean": 0.021621918864548208, "step": 20510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.2, "completions/max_terminated_length": 272.2, "completions/mean_length": 202.55, "completions/mean_terminated_length": 202.55, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.058648630231619, "epoch": 1.8865496000735495, "frac_reward_zero_std": 0.5, "grad_norm": 6.5929436683654785, "kl": 0.008533105871174484, "learning_rate": 1.658494788860926e-06, "loss": 0.0004, "num_tokens": 89385182.0, "reward": -0.125, "reward_std": 0.46289436221122743, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4628943860530853, "sampling/importance_sampling_ratio/max": 1.266815173625946, "sampling/importance_sampling_ratio/mean": 1.0000665009021759, "sampling/importance_sampling_ratio/min": 0.7538548707962036, "sampling/sampling_logp_difference/max": 0.2988824605941772, "sampling/sampling_logp_difference/mean": 0.02126534152776003, "step": 20520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.9, "completions/max_terminated_length": 335.9, "completions/mean_length": 232.2625, "completions/mean_terminated_length": 232.2625, "completions/min_length": 155.7, "completions/min_terminated_length": 155.7, "entropy": 1.0736057758331299, "epoch": 1.887468971223683, "frac_reward_zero_std": 0.8, "grad_norm": 0.09055932611227036, "kl": 0.00906398192164488, "learning_rate": 1.6561094418443455e-06, "loss": 0.0005, "num_tokens": 89433627.0, "reward": -0.425, "reward_std": 0.2104143261909485, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.21041433811187743, "sampling/importance_sampling_ratio/max": 1.2925012111663818, "sampling/importance_sampling_ratio/mean": 0.999838387966156, "sampling/importance_sampling_ratio/min": 0.7781914949417115, "sampling/sampling_logp_difference/max": 0.29183986186981203, "sampling/sampling_logp_difference/mean": 0.020989887602627277, "step": 20530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 219.8125, "completions/mean_terminated_length": 219.8125, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0712954603135585, "epoch": 1.8883883423738164, "frac_reward_zero_std": 0.5, "grad_norm": 0.08086920529603958, "kl": 0.008623416238697246, "learning_rate": 1.6537249615710143e-06, "loss": 0.0008, "num_tokens": 89481044.0, "reward": -0.05, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.3378339052200316, "sampling/importance_sampling_ratio/mean": 1.0000746905803681, "sampling/importance_sampling_ratio/min": 0.8004160821437836, "sampling/sampling_logp_difference/max": 0.2963345289230347, "sampling/sampling_logp_difference/mean": 0.021191131696105002, "step": 20540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.6, "completions/max_terminated_length": 306.6, "completions/mean_length": 217.1125, "completions/mean_terminated_length": 217.1125, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.0574621506035329, "epoch": 1.8893077135239498, "frac_reward_zero_std": 0.6, "grad_norm": 0.0683608129620552, "kl": 0.009701388262328692, "learning_rate": 1.651341350489985e-06, "loss": 0.0004, "num_tokens": 89524021.0, "reward": 0.1, "reward_std": 0.39897301197052004, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.39897301197052004, "sampling/importance_sampling_ratio/max": 1.2952882766723632, "sampling/importance_sampling_ratio/mean": 1.0000448822975159, "sampling/importance_sampling_ratio/min": 0.7596870243549347, "sampling/sampling_logp_difference/max": 0.30913671255111697, "sampling/sampling_logp_difference/mean": 0.02102250773459673, "step": 20550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 378.6, "completions/max_terminated_length": 375.7, "completions/mean_length": 251.6125, "completions/mean_terminated_length": 249.2107147216797, "completions/min_length": 165.6, "completions/min_terminated_length": 165.6, "entropy": 1.047645890340209, "epoch": 1.890227084674083, "frac_reward_zero_std": 0.4, "grad_norm": 0.07235308736562729, "kl": 0.00909764927928336, "learning_rate": 1.648958611049416e-06, "loss": 0.0003, "num_tokens": 89573702.0, "reward": -0.28125, "reward_std": 0.5729019403457641, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.5729019403457641, "sampling/importance_sampling_ratio/max": 1.3208552837371825, "sampling/importance_sampling_ratio/mean": 0.9994669139385224, "sampling/importance_sampling_ratio/min": 0.760936039686203, "sampling/sampling_logp_difference/max": 0.32800585627555845, "sampling/sampling_logp_difference/mean": 0.0204030254855752, "step": 20560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 219.0125, "completions/mean_terminated_length": 219.0125, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0173594988882542, "epoch": 1.8911464558242161, "frac_reward_zero_std": 0.5, "grad_norm": 9.491772651672363, "kl": 0.00818471212987788, "learning_rate": 1.6465767456965737e-06, "loss": 0.0005, "num_tokens": 89619495.0, "reward": 0.125, "reward_std": 0.45536120533943175, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.45536120533943175, "sampling/importance_sampling_ratio/max": 1.231529462337494, "sampling/importance_sampling_ratio/mean": 0.9999907553195954, "sampling/importance_sampling_ratio/min": 0.7872282922267914, "sampling/sampling_logp_difference/max": 0.24628231525421143, "sampling/sampling_logp_difference/mean": 0.020902708172798157, "step": 20570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 222.65, "completions/mean_terminated_length": 222.65, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.122019686549902, "epoch": 1.8920658269743496, "frac_reward_zero_std": 0.3, "grad_norm": 5.576419353485107, "kl": 0.010953779614646919, "learning_rate": 1.6441957568778234e-06, "loss": 0.0007, "num_tokens": 89664323.0, "reward": 0.1, "reward_std": 0.6800980150699616, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.6800980269908905, "sampling/importance_sampling_ratio/max": 1.3120846271514892, "sampling/importance_sampling_ratio/mean": 1.000499850511551, "sampling/importance_sampling_ratio/min": 0.7657740473747253, "sampling/sampling_logp_difference/max": 0.30882011651992797, "sampling/sampling_logp_difference/mean": 0.021881532855331898, "step": 20580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 216.8, "completions/mean_terminated_length": 216.8, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0924887023866177, "epoch": 1.892985198124483, "frac_reward_zero_std": 0.3, "grad_norm": 0.2766161262989044, "kl": 0.010245079890592024, "learning_rate": 1.6418156470386325e-06, "loss": 0.0004, "num_tokens": 89708379.0, "reward": -0.2, "reward_std": 0.662365174293518, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6623652219772339, "sampling/importance_sampling_ratio/max": 1.2900989174842834, "sampling/importance_sampling_ratio/mean": 0.9998782336711883, "sampling/importance_sampling_ratio/min": 0.7753895103931427, "sampling/sampling_logp_difference/max": 0.2834870517253876, "sampling/sampling_logp_difference/mean": 0.021802190132439135, "step": 20590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.1, "completions/max_terminated_length": 296.1, "completions/mean_length": 210.225, "completions/mean_terminated_length": 210.225, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.1211033314466476, "epoch": 1.8939045692746161, "frac_reward_zero_std": 0.7, "grad_norm": 0.09695301204919815, "kl": 0.008869532524840906, "learning_rate": 1.6394364186235633e-06, "loss": 0.0005, "num_tokens": 89748413.0, "reward": 0.425, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.249368417263031, "sampling/importance_sampling_ratio/mean": 1.0003124237060548, "sampling/importance_sampling_ratio/min": 0.7848419725894928, "sampling/sampling_logp_difference/max": 0.26166420578956606, "sampling/sampling_logp_difference/mean": 0.02173789609223604, "step": 20600 }, { "epoch": 1.8939045692746161, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 295.8499127399651, "eval_completions/max_terminated_length": 293.7294938917976, "eval_completions/mean_length": 207.75807155322863, "eval_completions/mean_terminated_length": 207.22581668341138, "eval_completions/min_length": 135.19895287958116, "eval_completions/min_terminated_length": 135.19895287958116, "eval_entropy": 1.0803008564791754, "eval_frac_reward_zero_std": 0.4781849912739965, "eval_kl": 0.008945873195718304, "eval_loss": 0.00046641132212243974, "eval_num_tokens": 89748413.0, "eval_reward": -0.08911431064572425, "eval_reward_std": 0.4539029687695062, "eval_rewards/reward_fn/mean": -0.08911431064572425, "eval_rewards/reward_fn/std": 0.45390298083606606, "eval_runtime": 1724.5296, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.2715526509243364, "eval_sampling/importance_sampling_ratio/mean": 1.000000862238919, "eval_sampling/importance_sampling_ratio/min": 0.7717333920755104, "eval_sampling/sampling_logp_difference/max": 0.2902655666515257, "eval_sampling/sampling_logp_difference/mean": 0.02080892724916339, "eval_steps_per_second": 0.042, "step": 20600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.9, "completions/max_terminated_length": 298.9, "completions/mean_length": 223.7625, "completions/mean_terminated_length": 223.7625, "completions/min_length": 155.6, "completions/min_terminated_length": 155.6, "entropy": 1.1169323071837425, "epoch": 1.8948239404247493, "frac_reward_zero_std": 0.5, "grad_norm": 0.07357217371463776, "kl": 0.00860890077019576, "learning_rate": 1.6370580740762754e-06, "loss": 0.0002, "num_tokens": 89795034.0, "reward": -0.275, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2604155540466309, "sampling/importance_sampling_ratio/mean": 0.9997382164001465, "sampling/importance_sampling_ratio/min": 0.7938172042369842, "sampling/sampling_logp_difference/max": 0.24993234276771545, "sampling/sampling_logp_difference/mean": 0.021366157196462156, "step": 20610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.2, "completions/max_terminated_length": 306.2, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.0284844756126403, "epoch": 1.8957433115748827, "frac_reward_zero_std": 0.5, "grad_norm": 0.10347241908311844, "kl": 0.009888591119670309, "learning_rate": 1.634680615839518e-06, "loss": 0.0006, "num_tokens": 89839496.0, "reward": -0.00625, "reward_std": 0.35858552902936935, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.35858552902936935, "sampling/importance_sampling_ratio/max": 1.3001866698265077, "sampling/importance_sampling_ratio/mean": 0.9997266173362732, "sampling/importance_sampling_ratio/min": 0.7585337877273559, "sampling/sampling_logp_difference/max": 0.32268968820571897, "sampling/sampling_logp_difference/mean": 0.020949642173945904, "step": 20620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.2, "completions/max_terminated_length": 293.2, "completions/mean_length": 206.2375, "completions/mean_terminated_length": 206.2375, "completions/min_length": 134.0, "completions/min_terminated_length": 134.0, "entropy": 1.0412440024316312, "epoch": 1.8966626827250161, "frac_reward_zero_std": 0.6, "grad_norm": 7.718129634857178, "kl": 0.009153928828891367, "learning_rate": 1.6323040463551333e-06, "loss": 0.0008, "num_tokens": 89881659.0, "reward": -0.45, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2837945342063903, "sampling/importance_sampling_ratio/mean": 1.000190305709839, "sampling/importance_sampling_ratio/min": 0.7411916553974152, "sampling/sampling_logp_difference/max": 0.3379689335823059, "sampling/sampling_logp_difference/mean": 0.02095847446471453, "step": 20630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.8, "completions/max_terminated_length": 281.8, "completions/mean_length": 215.3875, "completions/mean_terminated_length": 215.3875, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.0652388952672482, "epoch": 1.8975820538751496, "frac_reward_zero_std": 0.5, "grad_norm": 5.816048622131348, "kl": 0.010221025062492117, "learning_rate": 1.6299283680640455e-06, "loss": 0.0006, "num_tokens": 89925538.0, "reward": 0.05, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.3377911686897277, "sampling/importance_sampling_ratio/mean": 1.0004114508628845, "sampling/importance_sampling_ratio/min": 0.711362898349762, "sampling/sampling_logp_difference/max": 0.38377115726470945, "sampling/sampling_logp_difference/mean": 0.021466848999261857, "step": 20640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.5, "completions/max_terminated_length": 291.5, "completions/mean_length": 194.1625, "completions/mean_terminated_length": 194.1625, "completions/min_length": 120.9, "completions/min_terminated_length": 120.9, "entropy": 1.1129962369799613, "epoch": 1.8985014250252827, "frac_reward_zero_std": 0.6, "grad_norm": 0.07730504870414734, "kl": 0.008254003274487332, "learning_rate": 1.6275535834062694e-06, "loss": 0.0002, "num_tokens": 89970311.0, "reward": 0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2679516911506652, "sampling/importance_sampling_ratio/mean": 1.0005253911018372, "sampling/importance_sampling_ratio/min": 0.7960574924945831, "sampling/sampling_logp_difference/max": 0.2574121952056885, "sampling/sampling_logp_difference/mean": 0.021693380177021028, "step": 20650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 358.8, "completions/max_terminated_length": 349.2, "completions/mean_length": 231.0125, "completions/mean_terminated_length": 227.64464416503907, "completions/min_length": 150.2, "completions/min_terminated_length": 150.2, "entropy": 1.0870658136904239, "epoch": 1.899420796175416, "frac_reward_zero_std": 0.6, "grad_norm": 0.11329922825098038, "kl": 0.008758896007202565, "learning_rate": 1.6251796948208981e-06, "loss": 0.0007, "num_tokens": 90014424.0, "reward": -0.13125, "reward_std": 0.31010250449180604, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.3101025104522705, "sampling/importance_sampling_ratio/max": 1.2530495882034303, "sampling/importance_sampling_ratio/mean": 0.9996947228908539, "sampling/importance_sampling_ratio/min": 0.7664021968841552, "sampling/sampling_logp_difference/max": 0.26954805850982666, "sampling/sampling_logp_difference/mean": 0.022042001225054265, "step": 20660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 346.1, "completions/max_terminated_length": 343.0, "completions/mean_length": 229.7, "completions/mean_terminated_length": 226.85357360839845, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 1.078832568973303, "epoch": 1.9003401673255493, "frac_reward_zero_std": 0.2, "grad_norm": 0.10684377700090408, "kl": 0.009827220090664923, "learning_rate": 1.6228067047461053e-06, "loss": 0.0007, "num_tokens": 90057464.0, "reward": -0.70625, "reward_std": 0.6377657771110534, "rewards/reward_fn/mean": -0.70625, "rewards/reward_fn/std": 0.6377657890319824, "sampling/importance_sampling_ratio/max": 1.3208779096603394, "sampling/importance_sampling_ratio/mean": 1.0001461267471314, "sampling/importance_sampling_ratio/min": 0.7721107363700866, "sampling/sampling_logp_difference/max": 0.3278244137763977, "sampling/sampling_logp_difference/mean": 0.021039515733718872, "step": 20670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.8, "completions/max_terminated_length": 297.8, "completions/mean_length": 214.65, "completions/mean_terminated_length": 214.65, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.116967061907053, "epoch": 1.9012595384756827, "frac_reward_zero_std": 0.5, "grad_norm": 5.2092204093933105, "kl": 0.009495368582429364, "learning_rate": 1.6204346156191437e-06, "loss": 0.0007, "num_tokens": 90100644.0, "reward": -0.025, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.228716742992401, "sampling/importance_sampling_ratio/mean": 0.9995957016944885, "sampling/importance_sampling_ratio/min": 0.757811987400055, "sampling/sampling_logp_difference/max": 0.29529629945755004, "sampling/sampling_logp_difference/mean": 0.02229639533907175, "step": 20680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.3, "completions/max_terminated_length": 255.3, "completions/mean_length": 184.45, "completions/mean_terminated_length": 184.45, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 0.9976156331598759, "epoch": 1.902178909625816, "frac_reward_zero_std": 0.7, "grad_norm": 4.533091068267822, "kl": 0.007740517542697489, "learning_rate": 1.6180634298763367e-06, "loss": 0.0005, "num_tokens": 90139752.0, "reward": 0.2, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.3000281929969788, "sampling/importance_sampling_ratio/mean": 1.000694352388382, "sampling/importance_sampling_ratio/min": 0.7987823128700257, "sampling/sampling_logp_difference/max": 0.28071749210357666, "sampling/sampling_logp_difference/mean": 0.02042848728597164, "step": 20690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.2, "completions/max_terminated_length": 249.2, "completions/mean_length": 181.6125, "completions/mean_terminated_length": 181.6125, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 1.0053163815289736, "epoch": 1.9030982807759491, "frac_reward_zero_std": 0.7, "grad_norm": 0.0976424515247345, "kl": 0.010241929840412922, "learning_rate": 1.6156931499530826e-06, "loss": 0.0005, "num_tokens": 90177521.0, "reward": -0.25, "reward_std": 0.23400336503982544, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.23400336503982544, "sampling/importance_sampling_ratio/max": 1.2693826675415039, "sampling/importance_sampling_ratio/mean": 1.000139331817627, "sampling/importance_sampling_ratio/min": 0.754881602525711, "sampling/sampling_logp_difference/max": 0.2901972770690918, "sampling/sampling_logp_difference/mean": 0.02146184640005231, "step": 20700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 204.7875, "completions/mean_terminated_length": 204.7875, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 1.0618236020207406, "epoch": 1.9040176519260825, "frac_reward_zero_std": 0.7, "grad_norm": 0.07605550438165665, "kl": 0.009222830308135598, "learning_rate": 1.6133237782838502e-06, "loss": 0.0004, "num_tokens": 90222664.0, "reward": -0.20625, "reward_std": 0.27328457236289977, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.27328457236289977, "sampling/importance_sampling_ratio/max": 1.2846637606620788, "sampling/importance_sampling_ratio/mean": 0.9999203741550445, "sampling/importance_sampling_ratio/min": 0.7779305458068848, "sampling/sampling_logp_difference/max": 0.2900016665458679, "sampling/sampling_logp_difference/mean": 0.02106107324361801, "step": 20710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.4, "completions/max_terminated_length": 292.4, "completions/mean_length": 195.0, "completions/mean_terminated_length": 195.0, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.0444334670901299, "epoch": 1.904937023076216, "frac_reward_zero_std": 0.3, "grad_norm": 7.7402215003967285, "kl": 0.009380972178769298, "learning_rate": 1.6109553173021732e-06, "loss": 0.0002, "num_tokens": 90260504.0, "reward": -0.4, "reward_std": 0.6152435362339019, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.6152435719966889, "sampling/importance_sampling_ratio/max": 1.2800673246383667, "sampling/importance_sampling_ratio/mean": 0.999821001291275, "sampling/importance_sampling_ratio/min": 0.7845445275306702, "sampling/sampling_logp_difference/max": 0.28153235912323, "sampling/sampling_logp_difference/mean": 0.021178281866014004, "step": 20720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.1, "completions/max_terminated_length": 306.1, "completions/mean_length": 201.125, "completions/mean_terminated_length": 201.125, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 0.9921017803251744, "epoch": 1.9058563942263493, "frac_reward_zero_std": 0.5, "grad_norm": 7.117558002471924, "kl": 0.008038637129357085, "learning_rate": 1.608587769440651e-06, "loss": 0.001, "num_tokens": 90300978.0, "reward": -0.15, "reward_std": 0.46628902554512025, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4662890374660492, "sampling/importance_sampling_ratio/max": 1.2452963948249818, "sampling/importance_sampling_ratio/mean": 0.9998938977718353, "sampling/importance_sampling_ratio/min": 0.7909192919731141, "sampling/sampling_logp_difference/max": 0.23896432518959046, "sampling/sampling_logp_difference/mean": 0.020624322816729546, "step": 20730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 211.5, "completions/mean_terminated_length": 211.5, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.1609898410737514, "epoch": 1.9067757653764825, "frac_reward_zero_std": 0.6, "grad_norm": 6.6849446296691895, "kl": 0.009911271772580221, "learning_rate": 1.6062211371309446e-06, "loss": 0.0002, "num_tokens": 90346074.0, "reward": -0.18125, "reward_std": 0.3206740066409111, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.3206740185618401, "sampling/importance_sampling_ratio/max": 1.2359867572784424, "sampling/importance_sampling_ratio/mean": 0.9996381103992462, "sampling/importance_sampling_ratio/min": 0.7705992579460144, "sampling/sampling_logp_difference/max": 0.26670327186584475, "sampling/sampling_logp_difference/mean": 0.02273707389831543, "step": 20740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 212.7625, "completions/mean_terminated_length": 212.7625, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.1294696800410748, "epoch": 1.9076951365266157, "frac_reward_zero_std": 0.5, "grad_norm": 0.07705871015787125, "kl": 0.008505071816034614, "learning_rate": 1.6038554228037753e-06, "loss": 0.0004, "num_tokens": 90393239.0, "reward": -0.35, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.298323380947113, "sampling/importance_sampling_ratio/mean": 1.0004585921764373, "sampling/importance_sampling_ratio/min": 0.7787659943103791, "sampling/sampling_logp_difference/max": 0.27687835693359375, "sampling/sampling_logp_difference/mean": 0.021791683882474898, "step": 20750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.4, "completions/max_terminated_length": 259.4, "completions/mean_length": 190.35, "completions/mean_terminated_length": 190.35, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.0538315575569868, "epoch": 1.9086145076767491, "frac_reward_zero_std": 0.4, "grad_norm": 0.22674134373664856, "kl": 0.010081803548382595, "learning_rate": 1.601490628888922e-06, "loss": 0.0006, "num_tokens": 90433523.0, "reward": 0.175, "reward_std": 0.5185230314731598, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5185230314731598, "sampling/importance_sampling_ratio/max": 1.2703307628631593, "sampling/importance_sampling_ratio/mean": 0.9997845768928528, "sampling/importance_sampling_ratio/min": 0.7401171594858169, "sampling/sampling_logp_difference/max": 0.3709564983844757, "sampling/sampling_logp_difference/mean": 0.02094353027641773, "step": 20760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.8, "completions/max_terminated_length": 244.8, "completions/mean_length": 182.1375, "completions/mean_terminated_length": 182.1375, "completions/min_length": 118.3, "completions/min_terminated_length": 118.3, "entropy": 1.0666744761168956, "epoch": 1.9095338788268825, "frac_reward_zero_std": 0.6, "grad_norm": 0.07294011861085892, "kl": 0.009126391424797476, "learning_rate": 1.599126757815218e-06, "loss": 0.0002, "num_tokens": 90474742.0, "reward": 0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2695963501930236, "sampling/importance_sampling_ratio/mean": 0.9996902525424958, "sampling/importance_sampling_ratio/min": 0.7778341412544251, "sampling/sampling_logp_difference/max": 0.2907004952430725, "sampling/sampling_logp_difference/mean": 0.021617999672889708, "step": 20770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 206.575, "completions/mean_terminated_length": 206.575, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0261106442660093, "epoch": 1.9104532499770157, "frac_reward_zero_std": 0.4, "grad_norm": 6.560444355010986, "kl": 0.011044055971433408, "learning_rate": 1.5967638120105463e-06, "loss": 0.0007, "num_tokens": 90518572.0, "reward": -0.125, "reward_std": 0.5369997024536133, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5369997143745422, "sampling/importance_sampling_ratio/max": 1.2682689428329468, "sampling/importance_sampling_ratio/mean": 1.0001710951328278, "sampling/importance_sampling_ratio/min": 0.7707109808921814, "sampling/sampling_logp_difference/max": 0.2793963611125946, "sampling/sampling_logp_difference/mean": 0.020978967659175396, "step": 20780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 360.6, "completions/max_terminated_length": 360.6, "completions/mean_length": 246.425, "completions/mean_terminated_length": 246.425, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.002763357013464, "epoch": 1.911372621127149, "frac_reward_zero_std": 0.3, "grad_norm": 6.223293304443359, "kl": 0.00988422311784234, "learning_rate": 1.5944017939018436e-06, "loss": 0.0005, "num_tokens": 90563974.0, "reward": -0.125, "reward_std": 0.6842365086078643, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.6842365562915802, "sampling/importance_sampling_ratio/max": 1.28971426486969, "sampling/importance_sampling_ratio/mean": 1.0000754237174987, "sampling/importance_sampling_ratio/min": 0.750287663936615, "sampling/sampling_logp_difference/max": 0.30340352058410647, "sampling/sampling_logp_difference/mean": 0.02083461042493582, "step": 20790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.5, "completions/max_terminated_length": 263.5, "completions/mean_length": 188.5, "completions/mean_terminated_length": 188.5, "completions/min_length": 113.7, "completions/min_terminated_length": 113.7, "entropy": 0.9890485003590583, "epoch": 1.9122919922772823, "frac_reward_zero_std": 0.4, "grad_norm": 0.15311093628406525, "kl": 0.00838704569323454, "learning_rate": 1.5920407059150905e-06, "loss": 0.0, "num_tokens": 90606846.0, "reward": 0.175, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2607645750045777, "sampling/importance_sampling_ratio/mean": 0.9997847020626068, "sampling/importance_sampling_ratio/min": 0.7682810842990875, "sampling/sampling_logp_difference/max": 0.27852051258087157, "sampling/sampling_logp_difference/mean": 0.020656120544299484, "step": 20800 }, { "epoch": 1.9122919922772823, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 298.413612565445, "eval_completions/max_terminated_length": 297.9528795811518, "eval_completions/mean_length": 208.57002617801047, "eval_completions/mean_terminated_length": 208.24848344147102, "eval_completions/min_length": 133.3891797556719, "eval_completions/min_terminated_length": 133.3891797556719, "eval_entropy": 1.0812599926822888, "eval_frac_reward_zero_std": 0.4816753926701571, "eval_kl": 0.008824569273928592, "eval_loss": 0.00042886610026471317, "eval_num_tokens": 90606846.0, "eval_reward": -0.12532722513089006, "eval_reward_std": 0.44910967792099893, "eval_rewards/reward_fn/mean": -0.12532722513089006, "eval_rewards/reward_fn/std": 0.449109693420287, "eval_runtime": 1736.6775, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2737912008275536, "eval_sampling/importance_sampling_ratio/mean": 0.9999803756128015, "eval_sampling/importance_sampling_ratio/min": 0.7720173245741226, "eval_sampling/sampling_logp_difference/max": 0.2928673782325748, "eval_sampling/sampling_logp_difference/mean": 0.020824138180760707, "eval_steps_per_second": 0.041, "step": 20800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 199.9375, "completions/mean_terminated_length": 199.9375, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0386441469192504, "epoch": 1.9132113634274157, "frac_reward_zero_std": 0.5, "grad_norm": 0.08736582100391388, "kl": 0.009151078434661031, "learning_rate": 1.5896805504753155e-06, "loss": -0.0002, "num_tokens": 90650145.0, "reward": 0.25, "reward_std": 0.4225463569164276, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.4225463688373566, "sampling/importance_sampling_ratio/max": 1.275178861618042, "sampling/importance_sampling_ratio/mean": 0.9996629774570465, "sampling/importance_sampling_ratio/min": 0.7696052849292755, "sampling/sampling_logp_difference/max": 0.2898597002029419, "sampling/sampling_logp_difference/mean": 0.021087740175426008, "step": 20810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.8, "completions/max_terminated_length": 302.8, "completions/mean_length": 215.0, "completions/mean_terminated_length": 215.0, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.0485033117234708, "epoch": 1.9141307345775491, "frac_reward_zero_std": 0.4, "grad_norm": 0.09621494263410568, "kl": 0.009128739213338122, "learning_rate": 1.5873213300065852e-06, "loss": 0.0005, "num_tokens": 90692369.0, "reward": 0.05, "reward_std": 0.5403943657875061, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5403943657875061, "sampling/importance_sampling_ratio/max": 1.287535560131073, "sampling/importance_sampling_ratio/mean": 1.0000273764133454, "sampling/importance_sampling_ratio/min": 0.7693682372570038, "sampling/sampling_logp_difference/max": 0.29941980838775634, "sampling/sampling_logp_difference/mean": 0.02086138427257538, "step": 20820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.8, "completions/max_terminated_length": 267.8, "completions/mean_length": 195.8, "completions/mean_terminated_length": 195.8, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.014969675242901, "epoch": 1.9150501057276823, "frac_reward_zero_std": 0.5, "grad_norm": 0.12070383131504059, "kl": 0.010009225294925272, "learning_rate": 1.5849630469320088e-06, "loss": 0.0002, "num_tokens": 90731553.0, "reward": -0.025, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.303611397743225, "sampling/importance_sampling_ratio/mean": 1.000010085105896, "sampling/importance_sampling_ratio/min": 0.7446542501449585, "sampling/sampling_logp_difference/max": 0.33997013568878176, "sampling/sampling_logp_difference/mean": 0.021102079935371876, "step": 20830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.2, "completions/max_terminated_length": 306.2, "completions/mean_length": 216.275, "completions/mean_terminated_length": 216.275, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.0812695987522603, "epoch": 1.9159694768778155, "frac_reward_zero_std": 0.4, "grad_norm": 6.678839206695557, "kl": 0.010101751281763428, "learning_rate": 1.5826057036737327e-06, "loss": 0.0005, "num_tokens": 90780687.0, "reward": -0.025, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.3897802829742432, "sampling/importance_sampling_ratio/mean": 1.0003200769424438, "sampling/importance_sampling_ratio/min": 0.7744419574737549, "sampling/sampling_logp_difference/max": 0.3596459627151489, "sampling/sampling_logp_difference/mean": 0.021682369895279406, "step": 20840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.4, "completions/max_terminated_length": 287.4, "completions/mean_length": 196.0375, "completions/mean_terminated_length": 196.0375, "completions/min_length": 113.9, "completions/min_terminated_length": 113.9, "entropy": 1.06513981372118, "epoch": 1.916888848027949, "frac_reward_zero_std": 0.6, "grad_norm": 0.12041807919740677, "kl": 0.0085411431879038, "learning_rate": 1.5802493026529381e-06, "loss": 0.0002, "num_tokens": 90823290.0, "reward": 0.5, "reward_std": 0.40650616884231566, "rewards/reward_fn/mean": 0.5, "rewards/reward_fn/std": 0.4065061926841736, "sampling/importance_sampling_ratio/max": 1.2474699378013612, "sampling/importance_sampling_ratio/mean": 0.9997369408607483, "sampling/importance_sampling_ratio/min": 0.7878424167633057, "sampling/sampling_logp_difference/max": 0.26035332679748535, "sampling/sampling_logp_difference/mean": 0.021137782745063304, "step": 20850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 335.2, "completions/max_terminated_length": 331.3, "completions/mean_length": 228.9375, "completions/mean_terminated_length": 226.3, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.007054552808404, "epoch": 1.9178082191780823, "frac_reward_zero_std": 0.4, "grad_norm": 6.985903739929199, "kl": 0.008131466156919486, "learning_rate": 1.5778938462898355e-06, "loss": 0.0007, "num_tokens": 90869045.0, "reward": -0.15625, "reward_std": 0.42590154260396956, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.42590155452489853, "sampling/importance_sampling_ratio/max": 1.3062888026237487, "sampling/importance_sampling_ratio/mean": 1.0001601159572602, "sampling/importance_sampling_ratio/min": 0.7659714162349701, "sampling/sampling_logp_difference/max": 0.300845730304718, "sampling/sampling_logp_difference/mean": 0.019769678730517624, "step": 20860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.2, "completions/max_terminated_length": 315.2, "completions/mean_length": 221.9625, "completions/mean_terminated_length": 221.9625, "completions/min_length": 157.2, "completions/min_terminated_length": 157.2, "entropy": 1.1042153760790825, "epoch": 1.9187275903282155, "frac_reward_zero_std": 0.6, "grad_norm": 4.5418219566345215, "kl": 0.009006917872466148, "learning_rate": 1.575539337003669e-06, "loss": 0.0002, "num_tokens": 90916418.0, "reward": 0.125, "reward_std": 0.38463483452796937, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.38463485836982725, "sampling/importance_sampling_ratio/max": 1.260552430152893, "sampling/importance_sampling_ratio/mean": 1.0004830539226532, "sampling/importance_sampling_ratio/min": 0.7558118283748627, "sampling/sampling_logp_difference/max": 0.2990962266921997, "sampling/sampling_logp_difference/mean": 0.02109306547790766, "step": 20870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 196.8875, "completions/mean_terminated_length": 196.8875, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 0.9869284577667713, "epoch": 1.9196469614783487, "frac_reward_zero_std": 0.4, "grad_norm": 5.016903400421143, "kl": 0.007353348238393664, "learning_rate": 1.573185777212708e-06, "loss": 0.0003, "num_tokens": 90957225.0, "reward": 0.2, "reward_std": 0.5008058845996857, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5008058965206146, "sampling/importance_sampling_ratio/max": 1.25970561504364, "sampling/importance_sampling_ratio/mean": 1.0001304745674133, "sampling/importance_sampling_ratio/min": 0.7957006454467773, "sampling/sampling_logp_difference/max": 0.2422496795654297, "sampling/sampling_logp_difference/mean": 0.02073375191539526, "step": 20880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.4, "completions/max_terminated_length": 281.4, "completions/mean_length": 192.625, "completions/mean_terminated_length": 192.625, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.0715948216617108, "epoch": 1.920566332628482, "frac_reward_zero_std": 0.5, "grad_norm": 0.09212639182806015, "kl": 0.00959518289892003, "learning_rate": 1.5708331693342462e-06, "loss": 0.0009, "num_tokens": 90996547.0, "reward": -0.55, "reward_std": 0.4410230278968811, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.441023051738739, "sampling/importance_sampling_ratio/max": 1.2594697594642639, "sampling/importance_sampling_ratio/mean": 0.9993269085884094, "sampling/importance_sampling_ratio/min": 0.7715602517127991, "sampling/sampling_logp_difference/max": 0.27404859066009524, "sampling/sampling_logp_difference/mean": 0.021885924413800238, "step": 20890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.9, "completions/max_terminated_length": 312.9, "completions/mean_length": 212.6375, "completions/mean_terminated_length": 212.6375, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 1.109100379049778, "epoch": 1.9214857037786155, "frac_reward_zero_std": 0.3, "grad_norm": 5.687324047088623, "kl": 0.009059504041215406, "learning_rate": 1.5684815157846033e-06, "loss": 0.0004, "num_tokens": 91037542.0, "reward": 0.2, "reward_std": 0.6077103793621064, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.6077103912830353, "sampling/importance_sampling_ratio/max": 1.2420278310775756, "sampling/importance_sampling_ratio/mean": 1.0001430988311768, "sampling/importance_sampling_ratio/min": 0.7955015599727631, "sampling/sampling_logp_difference/max": 0.24253302812576294, "sampling/sampling_logp_difference/mean": 0.021783828176558016, "step": 20900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.6, "completions/max_terminated_length": 306.6, "completions/mean_length": 220.1375, "completions/mean_terminated_length": 220.1375, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.0696241192519664, "epoch": 1.9224050749287487, "frac_reward_zero_std": 0.4, "grad_norm": 7.077912330627441, "kl": 0.008662843389902263, "learning_rate": 1.5661308189791124e-06, "loss": 0.0004, "num_tokens": 91088849.0, "reward": 0.125, "reward_std": 0.5916544973850251, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.5916545450687408, "sampling/importance_sampling_ratio/max": 1.267828381061554, "sampling/importance_sampling_ratio/mean": 1.0000913739204407, "sampling/importance_sampling_ratio/min": 0.7779852032661438, "sampling/sampling_logp_difference/max": 0.2796412706375122, "sampling/sampling_logp_difference/mean": 0.02110248673707247, "step": 20910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 206.1875, "completions/mean_terminated_length": 206.1875, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.0656345672905445, "epoch": 1.923324446078882, "frac_reward_zero_std": 0.6, "grad_norm": 5.685739994049072, "kl": 0.010656511213164776, "learning_rate": 1.5637810813321297e-06, "loss": 0.0004, "num_tokens": 91132456.0, "reward": 0.15, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.237544047832489, "sampling/importance_sampling_ratio/mean": 1.0000223100185395, "sampling/importance_sampling_ratio/min": 0.7802249908447265, "sampling/sampling_logp_difference/max": 0.2617331385612488, "sampling/sampling_logp_difference/mean": 0.021349883079528807, "step": 20920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.5, "completions/max_terminated_length": 322.5, "completions/mean_length": 212.0125, "completions/mean_terminated_length": 212.0125, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.1589645557105541, "epoch": 1.9242438172290153, "frac_reward_zero_std": 0.3, "grad_norm": 7.894114971160889, "kl": 0.00838344609073829, "learning_rate": 1.5614323052570234e-06, "loss": 0.0004, "num_tokens": 91177073.0, "reward": 0.15, "reward_std": 0.593387895822525, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.593387907743454, "sampling/importance_sampling_ratio/max": 1.2488513827323913, "sampling/importance_sampling_ratio/mean": 0.9996794044971467, "sampling/importance_sampling_ratio/min": 0.7556593537330627, "sampling/sampling_logp_difference/max": 0.29667742252349855, "sampling/sampling_logp_difference/mean": 0.023254479095339776, "step": 20930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.4, "completions/max_terminated_length": 323.4, "completions/mean_length": 215.25, "completions/mean_terminated_length": 215.25, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0713353432714938, "epoch": 1.9251631883791487, "frac_reward_zero_std": 0.4, "grad_norm": 0.10647201538085938, "kl": 0.008832168823573738, "learning_rate": 1.5590844931661758e-06, "loss": 0.0006, "num_tokens": 91222173.0, "reward": 0.2, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.2958504557609558, "sampling/importance_sampling_ratio/mean": 1.0001378893852233, "sampling/importance_sampling_ratio/min": 0.7947922348976135, "sampling/sampling_logp_difference/max": 0.26201424598693845, "sampling/sampling_logp_difference/mean": 0.021204158663749695, "step": 20940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.6, "completions/max_terminated_length": 327.6, "completions/mean_length": 217.8875, "completions/mean_terminated_length": 217.8875, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.1517195463180543, "epoch": 1.926082559529282, "frac_reward_zero_std": 0.4, "grad_norm": 0.11078234016895294, "kl": 0.009684943145839498, "learning_rate": 1.5567376474709767e-06, "loss": 0.0003, "num_tokens": 91267324.0, "reward": -0.325, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.310058343410492, "sampling/importance_sampling_ratio/mean": 0.999994820356369, "sampling/importance_sampling_ratio/min": 0.7965555548667907, "sampling/sampling_logp_difference/max": 0.27946956753730773, "sampling/sampling_logp_difference/mean": 0.02260650638490915, "step": 20950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.9, "completions/max_terminated_length": 276.9, "completions/mean_length": 189.8625, "completions/mean_terminated_length": 189.8625, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.0621579363942146, "epoch": 1.9270019306794153, "frac_reward_zero_std": 0.3, "grad_norm": 5.602141857147217, "kl": 0.01178153424989432, "learning_rate": 1.5543917705818251e-06, "loss": 0.0006, "num_tokens": 91307601.0, "reward": -0.20625, "reward_std": 0.7225796580314636, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.7225796699523925, "sampling/importance_sampling_ratio/max": 1.391189181804657, "sampling/importance_sampling_ratio/mean": 1.0006951332092284, "sampling/importance_sampling_ratio/min": 0.805867338180542, "sampling/sampling_logp_difference/max": 0.3330625057220459, "sampling/sampling_logp_difference/mean": 0.021782043017446996, "step": 20960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 330.3, "completions/max_terminated_length": 312.9, "completions/mean_length": 223.425, "completions/mean_terminated_length": 217.53928833007814, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 0.9826403006911277, "epoch": 1.9279213018295485, "frac_reward_zero_std": 0.3, "grad_norm": 0.09900005161762238, "kl": 0.01050959361891728, "learning_rate": 1.5520468649081249e-06, "loss": 0.0007, "num_tokens": 91350835.0, "reward": -0.24375, "reward_std": 0.5167284354567527, "rewards/reward_fn/mean": -0.24375, "rewards/reward_fn/std": 0.5167284473776818, "sampling/importance_sampling_ratio/max": 1.2361551761627196, "sampling/importance_sampling_ratio/mean": 0.9996934413909913, "sampling/importance_sampling_ratio/min": 0.7997185409069061, "sampling/sampling_logp_difference/max": 0.2390514850616455, "sampling/sampling_logp_difference/mean": 0.020128354802727698, "step": 20970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.9, "completions/max_terminated_length": 275.9, "completions/mean_length": 201.6125, "completions/mean_terminated_length": 201.6125, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.1130043104290963, "epoch": 1.9288406729796819, "frac_reward_zero_std": 0.5, "grad_norm": 0.07863682508468628, "kl": 0.009103975864127279, "learning_rate": 1.5497029328582824e-06, "loss": 0.0005, "num_tokens": 91392404.0, "reward": 0.025, "reward_std": 0.5100160002708435, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5100160360336303, "sampling/importance_sampling_ratio/max": 1.2365198731422424, "sampling/importance_sampling_ratio/mean": 0.999304223060608, "sampling/importance_sampling_ratio/min": 0.7774069428443908, "sampling/sampling_logp_difference/max": 0.27521119713783265, "sampling/sampling_logp_difference/mean": 0.022107995860278607, "step": 20980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 206.4625, "completions/mean_terminated_length": 206.4625, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0578427501022816, "epoch": 1.9297600441298153, "frac_reward_zero_std": 0.5, "grad_norm": 0.0716865286231041, "kl": 0.008488464271067642, "learning_rate": 1.547359976839704e-06, "loss": 0.0007, "num_tokens": 91435809.0, "reward": -0.625, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.625, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3382083654403687, "sampling/importance_sampling_ratio/mean": 0.9998134315013886, "sampling/importance_sampling_ratio/min": 0.7706056535243988, "sampling/sampling_logp_difference/max": 0.36863718926906586, "sampling/sampling_logp_difference/mean": 0.02108570300042629, "step": 20990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 198.75, "completions/mean_terminated_length": 198.75, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.076958927512169, "epoch": 1.9306794152799485, "frac_reward_zero_std": 0.4, "grad_norm": 0.08285083621740341, "kl": 0.009278633573558182, "learning_rate": 1.545017999258792e-06, "loss": 0.0005, "num_tokens": 91476885.0, "reward": -0.275, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.265854251384735, "sampling/importance_sampling_ratio/mean": 1.0005371809005736, "sampling/importance_sampling_ratio/min": 0.7646299660205841, "sampling/sampling_logp_difference/max": 0.2887595772743225, "sampling/sampling_logp_difference/mean": 0.021819275617599488, "step": 21000 }, { "epoch": 1.9306794152799485, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 298.7452006980803, "eval_completions/max_terminated_length": 296.67015706806285, "eval_completions/mean_length": 208.77006980802793, "eval_completions/mean_terminated_length": 208.19587811375163, "eval_completions/min_length": 135.94415357766144, "eval_completions/min_terminated_length": 135.94415357766144, "eval_entropy": 1.0722955101759646, "eval_frac_reward_zero_std": 0.49214659685863876, "eval_kl": 0.008711016274107714, "eval_loss": 0.00046684089465998113, "eval_num_tokens": 91476885.0, "eval_reward": -0.10569371727748691, "eval_reward_std": 0.4503750744394399, "eval_rewards/reward_fn/mean": -0.10569371727748691, "eval_rewards/reward_fn/std": 0.45037508744219834, "eval_runtime": 1738.5675, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2773025374970095, "eval_sampling/importance_sampling_ratio/mean": 0.9999366034387918, "eval_sampling/importance_sampling_ratio/min": 0.7775212467191367, "eval_sampling/sampling_logp_difference/max": 0.28861570259455405, "eval_sampling/sampling_logp_difference/mean": 0.020770569045883303, "eval_steps_per_second": 0.041, "step": 21000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.5, "completions/max_terminated_length": 284.5, "completions/mean_length": 206.0375, "completions/mean_terminated_length": 206.0375, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.0567348010838031, "epoch": 1.9315987864300819, "frac_reward_zero_std": 0.5, "grad_norm": 6.054027080535889, "kl": 0.008597612689482049, "learning_rate": 1.5426770025209467e-06, "loss": 0.0006, "num_tokens": 91519384.0, "reward": -0.5, "reward_std": 0.3754247188568115, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.3754247188568115, "sampling/importance_sampling_ratio/max": 1.3117432475090027, "sampling/importance_sampling_ratio/mean": 1.0000665128231048, "sampling/importance_sampling_ratio/min": 0.7575277924537659, "sampling/sampling_logp_difference/max": 0.3191849112510681, "sampling/sampling_logp_difference/mean": 0.021131667122244836, "step": 21010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 208.1125, "completions/mean_terminated_length": 208.1125, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.102159559726715, "epoch": 1.932518157580215, "frac_reward_zero_std": 0.5, "grad_norm": 8.355255126953125, "kl": 0.009316037234384567, "learning_rate": 1.5403369890305581e-06, "loss": 0.0005, "num_tokens": 91561753.0, "reward": -0.275, "reward_std": 0.3863525390625, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.38635255098342897, "sampling/importance_sampling_ratio/max": 1.3511318683624267, "sampling/importance_sampling_ratio/mean": 0.9998055398464203, "sampling/importance_sampling_ratio/min": 0.7389084815979003, "sampling/sampling_logp_difference/max": 0.38599817752838134, "sampling/sampling_logp_difference/mean": 0.022242302633821966, "step": 21020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 354.8, "completions/max_terminated_length": 339.0, "completions/mean_length": 270.4375, "completions/mean_terminated_length": 260.7946441650391, "completions/min_length": 191.7, "completions/min_terminated_length": 191.7, "entropy": 1.1274611964821815, "epoch": 1.9334375287303485, "frac_reward_zero_std": 0.4, "grad_norm": 0.11697395145893097, "kl": 0.010512289917096495, "learning_rate": 1.537997961191009e-06, "loss": 0.0007, "num_tokens": 91613444.0, "reward": -0.30625, "reward_std": 0.3819169893860817, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.3819170013070107, "sampling/importance_sampling_ratio/max": 1.2645586490631104, "sampling/importance_sampling_ratio/mean": 1.000438505411148, "sampling/importance_sampling_ratio/min": 0.75790234208107, "sampling/sampling_logp_difference/max": 0.29433180689811705, "sampling/sampling_logp_difference/mean": 0.02181201558560133, "step": 21030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.0, "completions/max_terminated_length": 318.0, "completions/mean_length": 224.3, "completions/mean_terminated_length": 224.3, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.0710702769458293, "epoch": 1.9343568998804819, "frac_reward_zero_std": 0.5, "grad_norm": 0.0885797068476677, "kl": 0.009401999088004232, "learning_rate": 1.535659921404667e-06, "loss": 0.0006, "num_tokens": 91658276.0, "reward": -0.3, "reward_std": 0.47382218241691587, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.47382221817970277, "sampling/importance_sampling_ratio/max": 1.258183479309082, "sampling/importance_sampling_ratio/mean": 1.0001964688301086, "sampling/importance_sampling_ratio/min": 0.7862384080886841, "sampling/sampling_logp_difference/max": 0.2602337121963501, "sampling/sampling_logp_difference/mean": 0.021514797396957875, "step": 21040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.8, "completions/max_terminated_length": 303.8, "completions/mean_length": 216.5, "completions/mean_terminated_length": 216.5, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.1048254244029523, "epoch": 1.935276271030615, "frac_reward_zero_std": 0.4, "grad_norm": 5.566101551055908, "kl": 0.008465851203072815, "learning_rate": 1.5333228720728863e-06, "loss": 0.0003, "num_tokens": 91702020.0, "reward": -0.03125, "reward_std": 0.5542153477668762, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5542153716087341, "sampling/importance_sampling_ratio/max": 1.2630359888076783, "sampling/importance_sampling_ratio/mean": 1.0002264857292176, "sampling/importance_sampling_ratio/min": 0.7657571971416474, "sampling/sampling_logp_difference/max": 0.2764450490474701, "sampling/sampling_logp_difference/mean": 0.021964126080274583, "step": 21050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 346.4, "completions/max_terminated_length": 327.0, "completions/mean_length": 224.0875, "completions/mean_terminated_length": 220.76785888671876, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.0879449471831322, "epoch": 1.9361956421807482, "frac_reward_zero_std": 0.3, "grad_norm": 0.08885195106267929, "kl": 0.008485780344926752, "learning_rate": 1.5309868155960054e-06, "loss": 0.0001, "num_tokens": 91747427.0, "reward": 0.14375, "reward_std": 0.5622262224555016, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.5622262343764305, "sampling/importance_sampling_ratio/max": 1.3426278948783874, "sampling/importance_sampling_ratio/mean": 1.000609540939331, "sampling/importance_sampling_ratio/min": 0.7896876096725464, "sampling/sampling_logp_difference/max": 0.315903913974762, "sampling/sampling_logp_difference/mean": 0.021744335256516933, "step": 21060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 320.7, "completions/max_terminated_length": 304.0, "completions/mean_length": 218.3625, "completions/mean_terminated_length": 214.6642868041992, "completions/min_length": 137.8, "completions/min_terminated_length": 137.8, "entropy": 0.979505094513297, "epoch": 1.9371150133308817, "frac_reward_zero_std": 0.6, "grad_norm": 9.920928955078125, "kl": 0.009568222556845285, "learning_rate": 1.528651754373341e-06, "loss": 0.0005, "num_tokens": 91789528.0, "reward": 0.11875, "reward_std": 0.32362287640571596, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.3236228823661804, "sampling/importance_sampling_ratio/max": 1.261058270931244, "sampling/importance_sampling_ratio/mean": 0.9998659908771514, "sampling/importance_sampling_ratio/min": 0.7265777140855789, "sampling/sampling_logp_difference/max": 0.34095377922058107, "sampling/sampling_logp_difference/mean": 0.020031581819057464, "step": 21070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 218.625, "completions/mean_terminated_length": 218.625, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.1006341978907586, "epoch": 1.938034384481015, "frac_reward_zero_std": 0.5, "grad_norm": 6.238985538482666, "kl": 0.008592248789500444, "learning_rate": 1.5263176908031879e-06, "loss": 0.0007, "num_tokens": 91836866.0, "reward": -0.275, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2268172264099122, "sampling/importance_sampling_ratio/mean": 1.000015789270401, "sampling/importance_sampling_ratio/min": 0.7349351048469543, "sampling/sampling_logp_difference/max": 0.31764001250267027, "sampling/sampling_logp_difference/mean": 0.02057417631149292, "step": 21080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.4, "completions/max_terminated_length": 299.4, "completions/mean_length": 205.8125, "completions/mean_terminated_length": 205.8125, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.1063862763345242, "epoch": 1.9389537556311482, "frac_reward_zero_std": 0.6, "grad_norm": 0.09101583063602448, "kl": 0.008232143451459705, "learning_rate": 1.5239846272828162e-06, "loss": 0.0003, "num_tokens": 91883499.0, "reward": -0.3, "reward_std": 0.3921836853027344, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.3921837091445923, "sampling/importance_sampling_ratio/max": 1.2282373666763307, "sampling/importance_sampling_ratio/mean": 0.999763548374176, "sampling/importance_sampling_ratio/min": 0.7710076034069061, "sampling/sampling_logp_difference/max": 0.2657676815986633, "sampling/sampling_logp_difference/mean": 0.02207399643957615, "step": 21090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 191.4125, "completions/mean_terminated_length": 191.4125, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.0878178879618645, "epoch": 1.9398731267812817, "frac_reward_zero_std": 0.5, "grad_norm": 6.5920209884643555, "kl": 0.009336776449345052, "learning_rate": 1.5216525662084697e-06, "loss": 0.0003, "num_tokens": 91926492.0, "reward": 0.15, "reward_std": 0.5066213369369507, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5066213846206665, "sampling/importance_sampling_ratio/max": 1.2525041937828063, "sampling/importance_sampling_ratio/mean": 0.9999818563461303, "sampling/importance_sampling_ratio/min": 0.7731407344341278, "sampling/sampling_logp_difference/max": 0.2690204083919525, "sampling/sampling_logp_difference/mean": 0.02185372281819582, "step": 21100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 218.625, "completions/mean_terminated_length": 218.625, "completions/min_length": 156.0, "completions/min_terminated_length": 156.0, "entropy": 1.1055614829063416, "epoch": 1.9407924979314148, "frac_reward_zero_std": 0.6, "grad_norm": 6.904996395111084, "kl": 0.009946778195444494, "learning_rate": 1.5193215099753612e-06, "loss": 0.0006, "num_tokens": 91974414.0, "reward": -0.225, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.263353168964386, "sampling/importance_sampling_ratio/mean": 1.000439041852951, "sampling/importance_sampling_ratio/min": 0.710037286579609, "sampling/sampling_logp_difference/max": 0.41080487370491026, "sampling/sampling_logp_difference/mean": 0.021371754072606563, "step": 21110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 201.1125, "completions/mean_terminated_length": 201.1125, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.044187419116497, "epoch": 1.9417118690815482, "frac_reward_zero_std": 0.4, "grad_norm": 6.158493518829346, "kl": 0.007630195646197535, "learning_rate": 1.5169914609776744e-06, "loss": 0.0004, "num_tokens": 92015815.0, "reward": -0.4, "reward_std": 0.5916701912879944, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.5916702151298523, "sampling/importance_sampling_ratio/max": 1.2221010208129883, "sampling/importance_sampling_ratio/mean": 1.0001423895359038, "sampling/importance_sampling_ratio/min": 0.7494834661483765, "sampling/sampling_logp_difference/max": 0.29730710983276365, "sampling/sampling_logp_difference/mean": 0.02081954777240753, "step": 21120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 200.675, "completions/mean_terminated_length": 200.675, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0482952520251274, "epoch": 1.9426312402316817, "frac_reward_zero_std": 0.6, "grad_norm": 0.1404152810573578, "kl": 0.008349995297612623, "learning_rate": 1.5146624216085536e-06, "loss": 0.0002, "num_tokens": 92056653.0, "reward": 0.15, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.2440240144729615, "sampling/importance_sampling_ratio/mean": 0.9997642517089844, "sampling/importance_sampling_ratio/min": 0.7690218567848206, "sampling/sampling_logp_difference/max": 0.2702393591403961, "sampling/sampling_logp_difference/mean": 0.021110146678984164, "step": 21130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.0, "completions/max_terminated_length": 254.0, "completions/mean_length": 194.525, "completions/mean_terminated_length": 194.525, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.1358453884720803, "epoch": 1.9435506113818148, "frac_reward_zero_std": 0.7, "grad_norm": 0.1596246063709259, "kl": 0.00879970779060386, "learning_rate": 1.512334394260111e-06, "loss": 0.0007, "num_tokens": 92097095.0, "reward": -0.25, "reward_std": 0.2777460336685181, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.2777460336685181, "sampling/importance_sampling_ratio/max": 1.2328149199485778, "sampling/importance_sampling_ratio/mean": 1.0000723481178284, "sampling/importance_sampling_ratio/min": 0.7816563785076142, "sampling/sampling_logp_difference/max": 0.25410628914833067, "sampling/sampling_logp_difference/mean": 0.022333303838968276, "step": 21140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 349.0, "completions/max_terminated_length": 349.0, "completions/mean_length": 214.475, "completions/mean_terminated_length": 214.475, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.009906916320324, "epoch": 1.944469982531948, "frac_reward_zero_std": 0.5, "grad_norm": 0.08448950201272964, "kl": 0.009044785171863624, "learning_rate": 1.5100073813234162e-06, "loss": 0.0003, "num_tokens": 92141197.0, "reward": -0.475, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.320759117603302, "sampling/importance_sampling_ratio/mean": 1.000392997264862, "sampling/importance_sampling_ratio/min": 0.7707600831985474, "sampling/sampling_logp_difference/max": 0.33127000331878664, "sampling/sampling_logp_difference/mean": 0.020381515100598335, "step": 21150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.4, "completions/max_terminated_length": 331.4, "completions/mean_length": 209.4625, "completions/mean_terminated_length": 209.4625, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.127914746850729, "epoch": 1.9453893536820814, "frac_reward_zero_std": 0.4, "grad_norm": 0.08546194434165955, "kl": 0.010413249995326623, "learning_rate": 1.5076813851884992e-06, "loss": 0.0008, "num_tokens": 92187010.0, "reward": -0.025, "reward_std": 0.5882755279541015, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5882755637168884, "sampling/importance_sampling_ratio/max": 1.343780755996704, "sampling/importance_sampling_ratio/mean": 0.9997976243495941, "sampling/importance_sampling_ratio/min": 0.7642972528934479, "sampling/sampling_logp_difference/max": 0.3555413246154785, "sampling/sampling_logp_difference/mean": 0.02234354168176651, "step": 21160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.6, "completions/max_terminated_length": 318.6, "completions/mean_length": 206.6875, "completions/mean_terminated_length": 206.6875, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.0565400175750255, "epoch": 1.9463087248322148, "frac_reward_zero_std": 0.6, "grad_norm": 0.10049253702163696, "kl": 0.008585221105022356, "learning_rate": 1.5053564082443428e-06, "loss": 0.0002, "num_tokens": 92229641.0, "reward": 0.2, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.233456814289093, "sampling/importance_sampling_ratio/mean": 0.9999394237995147, "sampling/importance_sampling_ratio/min": 0.7826355576515198, "sampling/sampling_logp_difference/max": 0.25227153301239014, "sampling/sampling_logp_difference/mean": 0.021091587282717227, "step": 21170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.5, "completions/max_terminated_length": 271.5, "completions/mean_length": 195.05, "completions/mean_terminated_length": 195.05, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.163472831249237, "epoch": 1.947228095982348, "frac_reward_zero_std": 0.2, "grad_norm": 8.824557304382324, "kl": 0.009165668478817678, "learning_rate": 1.5030324528788859e-06, "loss": 0.0007, "num_tokens": 92270429.0, "reward": 0.075, "reward_std": 0.7836078464984894, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.7836078703403473, "sampling/importance_sampling_ratio/max": 1.2705222964286804, "sampling/importance_sampling_ratio/mean": 0.9999978601932525, "sampling/importance_sampling_ratio/min": 0.7728952288627624, "sampling/sampling_logp_difference/max": 0.29518054723739623, "sampling/sampling_logp_difference/mean": 0.02272309735417366, "step": 21180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.4, "completions/max_terminated_length": 302.4, "completions/mean_length": 213.0875, "completions/mean_terminated_length": 213.0875, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0121307611465453, "epoch": 1.9481474671324814, "frac_reward_zero_std": 0.6, "grad_norm": 0.09223631769418716, "kl": 0.009167515786248259, "learning_rate": 1.500709521479016e-06, "loss": 0.0002, "num_tokens": 92313348.0, "reward": 0.35, "reward_std": 0.2828427076339722, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.2828427076339722, "sampling/importance_sampling_ratio/max": 1.318452203273773, "sampling/importance_sampling_ratio/mean": 0.9998744070529938, "sampling/importance_sampling_ratio/min": 0.7884115397930145, "sampling/sampling_logp_difference/max": 0.29897003769874575, "sampling/sampling_logp_difference/mean": 0.02029999103397131, "step": 21190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.2, "completions/max_terminated_length": 311.2, "completions/mean_length": 216.7875, "completions/mean_terminated_length": 216.7875, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0803259558975697, "epoch": 1.9490668382826146, "frac_reward_zero_std": 0.3, "grad_norm": 5.049854278564453, "kl": 0.008984354717540555, "learning_rate": 1.498387616430571e-06, "loss": 0.0001, "num_tokens": 92358211.0, "reward": -0.35625, "reward_std": 0.5328060373663902, "rewards/reward_fn/mean": -0.35625, "rewards/reward_fn/std": 0.5328060492873192, "sampling/importance_sampling_ratio/max": 1.236818528175354, "sampling/importance_sampling_ratio/mean": 1.0001780152320863, "sampling/importance_sampling_ratio/min": 0.7996718168258667, "sampling/sampling_logp_difference/max": 0.2281087338924408, "sampling/sampling_logp_difference/mean": 0.020809098333120345, "step": 21200 }, { "epoch": 1.9490668382826146, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 302.565445026178, "eval_completions/max_terminated_length": 300.586387434555, "eval_completions/mean_length": 209.95724258289704, "eval_completions/mean_terminated_length": 209.44766493659162, "eval_completions/min_length": 134.3787085514834, "eval_completions/min_terminated_length": 134.3787085514834, "eval_entropy": 1.0766597538926423, "eval_frac_reward_zero_std": 0.5043630017452007, "eval_kl": 0.008910247761866705, "eval_loss": 0.0004435390292201191, "eval_num_tokens": 92358211.0, "eval_reward": -0.10154886561954625, "eval_reward_std": 0.4391147714062719, "eval_rewards/reward_fn/mean": -0.10154886561954625, "eval_rewards/reward_fn/std": 0.4391147849291407, "eval_runtime": 1756.253, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.273380959637294, "eval_sampling/importance_sampling_ratio/mean": 1.0000079245051372, "eval_sampling/importance_sampling_ratio/min": 0.7756824545419237, "eval_sampling/sampling_logp_difference/max": 0.28736800605505963, "eval_sampling/sampling_logp_difference/mean": 0.02083412052358112, "eval_steps_per_second": 0.041, "step": 21200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 336.8, "completions/max_terminated_length": 320.0, "completions/mean_length": 227.0375, "completions/mean_terminated_length": 223.8375, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.074483861029148, "epoch": 1.949986209432748, "frac_reward_zero_std": 0.6, "grad_norm": 0.09189213067293167, "kl": 0.00876375517109409, "learning_rate": 1.4960667401183333e-06, "loss": 0.0002, "num_tokens": 92407638.0, "reward": 0.04375, "reward_std": 0.2516810342669487, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.2516810342669487, "sampling/importance_sampling_ratio/max": 1.212996482849121, "sampling/importance_sampling_ratio/mean": 0.9999383866786957, "sampling/importance_sampling_ratio/min": 0.7652329504489899, "sampling/sampling_logp_difference/max": 0.27235034108161926, "sampling/sampling_logp_difference/mean": 0.021674555726349354, "step": 21210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.9, "completions/max_terminated_length": 335.9, "completions/mean_length": 229.225, "completions/mean_terminated_length": 229.225, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.022176595032215, "epoch": 1.9509055805828814, "frac_reward_zero_std": 0.3, "grad_norm": 5.418438911437988, "kl": 0.011371588506153784, "learning_rate": 1.4937468949260284e-06, "loss": 0.0006, "num_tokens": 92453336.0, "reward": -0.15, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.281804871559143, "sampling/importance_sampling_ratio/mean": 0.9997515499591827, "sampling/importance_sampling_ratio/min": 0.7382362872362137, "sampling/sampling_logp_difference/max": 0.37533590793609617, "sampling/sampling_logp_difference/mean": 0.021079852432012557, "step": 21220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.2, "completions/max_terminated_length": 309.2, "completions/mean_length": 220.45, "completions/mean_terminated_length": 220.45, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.1019725501537323, "epoch": 1.9518249517330146, "frac_reward_zero_std": 0.1, "grad_norm": 5.1964311599731445, "kl": 0.00813859902555123, "learning_rate": 1.4914280832363232e-06, "loss": 0.0004, "num_tokens": 92501300.0, "reward": -0.45, "reward_std": 0.792874401807785, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.792874413728714, "sampling/importance_sampling_ratio/max": 1.2630964398384095, "sampling/importance_sampling_ratio/mean": 0.999256718158722, "sampling/importance_sampling_ratio/min": 0.7389826595783233, "sampling/sampling_logp_difference/max": 0.3316605806350708, "sampling/sampling_logp_difference/mean": 0.021152886003255843, "step": 21230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.8, "completions/max_terminated_length": 274.8, "completions/mean_length": 190.8625, "completions/mean_terminated_length": 190.8625, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.043604888021946, "epoch": 1.9527443228831478, "frac_reward_zero_std": 0.3, "grad_norm": 6.958163261413574, "kl": 0.009762546414276586, "learning_rate": 1.489110307430824e-06, "loss": 0.0004, "num_tokens": 92543953.0, "reward": -0.225, "reward_std": 0.6371148705482483, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.6371149063110352, "sampling/importance_sampling_ratio/max": 1.2514428734779357, "sampling/importance_sampling_ratio/mean": 1.0003077030181884, "sampling/importance_sampling_ratio/min": 0.785511177778244, "sampling/sampling_logp_difference/max": 0.27343244552612306, "sampling/sampling_logp_difference/mean": 0.0209883701056242, "step": 21240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.9, "completions/max_terminated_length": 294.9, "completions/mean_length": 221.85, "completions/mean_terminated_length": 221.85, "completions/min_length": 151.5, "completions/min_terminated_length": 151.5, "entropy": 1.1418745942413806, "epoch": 1.9536636940332812, "frac_reward_zero_std": 0.6, "grad_norm": 5.574055194854736, "kl": 0.008972679759608582, "learning_rate": 1.4867935698900715e-06, "loss": 0.0007, "num_tokens": 92593789.0, "reward": -0.15, "reward_std": 0.34844101667404176, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.34844104051589964, "sampling/importance_sampling_ratio/max": 1.295318341255188, "sampling/importance_sampling_ratio/mean": 0.9995025753974914, "sampling/importance_sampling_ratio/min": 0.7855361998081207, "sampling/sampling_logp_difference/max": 0.2881822526454926, "sampling/sampling_logp_difference/mean": 0.02145673055201769, "step": 21250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.2, "completions/max_terminated_length": 295.2, "completions/mean_length": 216.45, "completions/mean_terminated_length": 216.45, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.0210408620536326, "epoch": 1.9545830651834146, "frac_reward_zero_std": 0.5, "grad_norm": 0.07465477287769318, "kl": 0.009480227291351184, "learning_rate": 1.484477872993541e-06, "loss": 0.0009, "num_tokens": 92641449.0, "reward": 0.05, "reward_std": 0.4553455114364624, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.45534553527832033, "sampling/importance_sampling_ratio/max": 1.2854811787605285, "sampling/importance_sampling_ratio/mean": 1.0001559257507324, "sampling/importance_sampling_ratio/min": 0.7483643740415573, "sampling/sampling_logp_difference/max": 0.3191485583782196, "sampling/sampling_logp_difference/mean": 0.020444356091320514, "step": 21260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.0, "completions/max_terminated_length": 279.0, "completions/mean_length": 189.6125, "completions/mean_terminated_length": 189.6125, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 0.9812168262898922, "epoch": 1.9555024363335478, "frac_reward_zero_std": 0.4, "grad_norm": 7.07713508605957, "kl": 0.008948355508618989, "learning_rate": 1.4821632191196386e-06, "loss": 0.0008, "num_tokens": 92682426.0, "reward": 0.125, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.2315496444702148, "sampling/importance_sampling_ratio/mean": 0.9997580409049988, "sampling/importance_sampling_ratio/min": 0.7945205628871918, "sampling/sampling_logp_difference/max": 0.23045759201049804, "sampling/sampling_logp_difference/mean": 0.02027287147939205, "step": 21270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.5, "completions/max_terminated_length": 270.5, "completions/mean_length": 192.8, "completions/mean_terminated_length": 192.8, "completions/min_length": 120.3, "completions/min_terminated_length": 120.3, "entropy": 1.0233675740659236, "epoch": 1.956421807483681, "frac_reward_zero_std": 0.7, "grad_norm": 0.14776553213596344, "kl": 0.009213358676061035, "learning_rate": 1.4798496106457002e-06, "loss": 0.0007, "num_tokens": 92723234.0, "reward": 0.1375, "reward_std": 0.12793734967708587, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.12793734967708587, "sampling/importance_sampling_ratio/max": 1.2452876687049865, "sampling/importance_sampling_ratio/mean": 0.9996596992015838, "sampling/importance_sampling_ratio/min": 0.7788904488086701, "sampling/sampling_logp_difference/max": 0.2561988651752472, "sampling/sampling_logp_difference/mean": 0.02115345522761345, "step": 21280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.4, "completions/max_terminated_length": 300.4, "completions/mean_length": 200.175, "completions/mean_terminated_length": 200.175, "completions/min_length": 114.8, "completions/min_terminated_length": 114.8, "entropy": 1.0263498779386282, "epoch": 1.9573411786338144, "frac_reward_zero_std": 0.3, "grad_norm": 0.15245181322097778, "kl": 0.008808057248825208, "learning_rate": 1.4775370499479869e-06, "loss": 0.0004, "num_tokens": 92766640.0, "reward": -0.08125, "reward_std": 0.5294113740324974, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.5294113978743553, "sampling/importance_sampling_ratio/max": 1.2682355523109436, "sampling/importance_sampling_ratio/mean": 1.0001741230487824, "sampling/importance_sampling_ratio/min": 0.7704642593860627, "sampling/sampling_logp_difference/max": 0.2691575288772583, "sampling/sampling_logp_difference/mean": 0.020953622087836264, "step": 21290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.3, "completions/max_terminated_length": 291.3, "completions/mean_length": 218.4, "completions/mean_terminated_length": 218.4, "completions/min_length": 152.9, "completions/min_terminated_length": 152.9, "entropy": 1.0478652529418468, "epoch": 1.9582605497839478, "frac_reward_zero_std": 0.7, "grad_norm": 0.0849028155207634, "kl": 0.009350107013597153, "learning_rate": 1.475225539401684e-06, "loss": 0.0005, "num_tokens": 92809696.0, "reward": -0.025, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.2505181789398194, "sampling/importance_sampling_ratio/mean": 1.0002147376537323, "sampling/importance_sampling_ratio/min": 0.74228076338768, "sampling/sampling_logp_difference/max": 0.3137127995491028, "sampling/sampling_logp_difference/mean": 0.021092795208096503, "step": 21300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 212.8, "completions/mean_terminated_length": 212.8, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0566436015069485, "epoch": 1.9591799209340812, "frac_reward_zero_std": 0.4, "grad_norm": 4.968390464782715, "kl": 0.0105677169340197, "learning_rate": 1.4729150813808973e-06, "loss": 0.0005, "num_tokens": 92852584.0, "reward": 0.225, "reward_std": 0.5042005479335785, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.5042005479335785, "sampling/importance_sampling_ratio/max": 1.2570233941078186, "sampling/importance_sampling_ratio/mean": 0.9997462213039399, "sampling/importance_sampling_ratio/min": 0.7676755368709565, "sampling/sampling_logp_difference/max": 0.2717937231063843, "sampling/sampling_logp_difference/mean": 0.021032998338341713, "step": 21310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.9, "completions/max_terminated_length": 294.9, "completions/mean_length": 200.5125, "completions/mean_terminated_length": 200.5125, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.0079824052751065, "epoch": 1.9600992920842144, "frac_reward_zero_std": 0.5, "grad_norm": 0.08657649159431458, "kl": 0.008819766191300005, "learning_rate": 1.4706056782586532e-06, "loss": 0.0004, "num_tokens": 92894897.0, "reward": 0.075, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.3401150703430176, "sampling/importance_sampling_ratio/mean": 1.0002123057842254, "sampling/importance_sampling_ratio/min": 0.7924579322338104, "sampling/sampling_logp_difference/max": 0.29418652057647704, "sampling/sampling_logp_difference/mean": 0.020875578746199608, "step": 21320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.8, "completions/max_terminated_length": 327.8, "completions/mean_length": 222.25, "completions/mean_terminated_length": 222.25, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 0.9679672505706549, "epoch": 1.9610186632343476, "frac_reward_zero_std": 0.3, "grad_norm": 0.07200785726308823, "kl": 0.008221825349028222, "learning_rate": 1.4682973324068929e-06, "loss": 0.0005, "num_tokens": 92940397.0, "reward": -0.2, "reward_std": 0.5824443817138671, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5824444055557251, "sampling/importance_sampling_ratio/max": 1.2636836647987366, "sampling/importance_sampling_ratio/mean": 0.9992217242717742, "sampling/importance_sampling_ratio/min": 0.7567607879638671, "sampling/sampling_logp_difference/max": 0.29147958755493164, "sampling/sampling_logp_difference/mean": 0.01987753752619028, "step": 21330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.9, "completions/max_terminated_length": 331.9, "completions/mean_length": 213.2625, "completions/mean_terminated_length": 213.2625, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.1498613037168979, "epoch": 1.961938034384481, "frac_reward_zero_std": 0.4, "grad_norm": 6.206754207611084, "kl": 0.008678248804062604, "learning_rate": 1.465990046196475e-06, "loss": 0.0007, "num_tokens": 92988538.0, "reward": 0.025, "reward_std": 0.5554763734340668, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5554763972759247, "sampling/importance_sampling_ratio/max": 1.294390070438385, "sampling/importance_sampling_ratio/mean": 0.9999190509319306, "sampling/importance_sampling_ratio/min": 0.7681761801242828, "sampling/sampling_logp_difference/max": 0.2966695189476013, "sampling/sampling_logp_difference/mean": 0.022026864252984522, "step": 21340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.3, "completions/max_terminated_length": 289.2, "completions/mean_length": 209.975, "completions/mean_terminated_length": 207.5, "completions/min_length": 147.9, "completions/min_terminated_length": 147.9, "entropy": 1.095412090793252, "epoch": 1.9628574055346144, "frac_reward_zero_std": 0.5, "grad_norm": 5.999139785766602, "kl": 0.008238008999614976, "learning_rate": 1.463683821997164e-06, "loss": 0.0003, "num_tokens": 93032224.0, "reward": -0.28125, "reward_std": 0.4490040421485901, "rewards/reward_fn/mean": -0.28125, "rewards/reward_fn/std": 0.4490040600299835, "sampling/importance_sampling_ratio/max": 1.2479411959648132, "sampling/importance_sampling_ratio/mean": 1.0005657017230987, "sampling/importance_sampling_ratio/min": 0.7864063501358032, "sampling/sampling_logp_difference/max": 0.2544170022010803, "sampling/sampling_logp_difference/mean": 0.021140670403838158, "step": 21350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 307.8, "completions/max_terminated_length": 306.3, "completions/mean_length": 226.0375, "completions/mean_terminated_length": 222.40833435058593, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.0121444009244442, "epoch": 1.9637767766847476, "frac_reward_zero_std": 0.3, "grad_norm": 0.09325094521045685, "kl": 0.008132225010194815, "learning_rate": 1.4613786621776393e-06, "loss": 0.0004, "num_tokens": 93075155.0, "reward": -0.3375, "reward_std": 0.6006778359413147, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.6006778597831726, "sampling/importance_sampling_ratio/max": 1.2753342032432555, "sampling/importance_sampling_ratio/mean": 0.9997432112693787, "sampling/importance_sampling_ratio/min": 0.7676762759685516, "sampling/sampling_logp_difference/max": 0.2961310148239136, "sampling/sampling_logp_difference/mean": 0.020141756720840932, "step": 21360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.6, "completions/max_terminated_length": 235.6, "completions/mean_length": 187.5375, "completions/mean_terminated_length": 187.5375, "completions/min_length": 117.3, "completions/min_terminated_length": 117.3, "entropy": 1.029591590166092, "epoch": 1.9646961478348808, "frac_reward_zero_std": 0.5, "grad_norm": 0.09044259786605835, "kl": 0.009246544662164524, "learning_rate": 1.4590745691054837e-06, "loss": 0.0004, "num_tokens": 93115070.0, "reward": -0.4, "reward_std": 0.43348987102508546, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.43348987102508546, "sampling/importance_sampling_ratio/max": 1.29266037940979, "sampling/importance_sampling_ratio/mean": 0.9997210204601288, "sampling/importance_sampling_ratio/min": 0.7800370156764984, "sampling/sampling_logp_difference/max": 0.29085339307785035, "sampling/sampling_logp_difference/mean": 0.02116508912295103, "step": 21370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.3, "completions/max_terminated_length": 301.3, "completions/mean_length": 205.825, "completions/mean_terminated_length": 205.825, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 1.0122291930019855, "epoch": 1.9656155189850142, "frac_reward_zero_std": 0.4, "grad_norm": 0.10318674147129059, "kl": 0.009902772711939178, "learning_rate": 1.456771545147187e-06, "loss": 0.0006, "num_tokens": 93157048.0, "reward": -0.05625, "reward_std": 0.5668363571166992, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.5668363869190216, "sampling/importance_sampling_ratio/max": 1.261973488330841, "sampling/importance_sampling_ratio/mean": 0.9994102597236634, "sampling/importance_sampling_ratio/min": 0.7515166163444519, "sampling/sampling_logp_difference/max": 0.2999048888683319, "sampling/sampling_logp_difference/mean": 0.020258005894720554, "step": 21380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.8, "completions/max_terminated_length": 291.6, "completions/mean_length": 206.05, "completions/mean_terminated_length": 202.8607147216797, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.0240629583597183, "epoch": 1.9665348901351476, "frac_reward_zero_std": 0.6, "grad_norm": 0.09480887651443481, "kl": 0.008436186477774754, "learning_rate": 1.4544695926681386e-06, "loss": 0.0003, "num_tokens": 93198932.0, "reward": -0.13125, "reward_std": 0.3551908671855927, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.3551908850669861, "sampling/importance_sampling_ratio/max": 1.2354193091392518, "sampling/importance_sampling_ratio/mean": 1.0002240002155305, "sampling/importance_sampling_ratio/min": 0.7820390880107879, "sampling/sampling_logp_difference/max": 0.24902713298797607, "sampling/sampling_logp_difference/mean": 0.020809478126466274, "step": 21390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 204.025, "completions/mean_terminated_length": 204.025, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.0275207482278348, "epoch": 1.967454261285281, "frac_reward_zero_std": 0.6, "grad_norm": 0.10051893442869186, "kl": 0.009772975876694545, "learning_rate": 1.452168714032627e-06, "loss": 0.0005, "num_tokens": 93239934.0, "reward": -0.225, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2659310936927795, "sampling/importance_sampling_ratio/mean": 0.9999298274517059, "sampling/importance_sampling_ratio/min": 0.7964859902858734, "sampling/sampling_logp_difference/max": 0.2518113136291504, "sampling/sampling_logp_difference/mean": 0.021426031924784185, "step": 21400 }, { "epoch": 1.967454261285281, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 293.35253054101224, "eval_completions/max_terminated_length": 292.2478184991274, "eval_completions/mean_length": 204.19328097731238, "eval_completions/mean_terminated_length": 203.9284156679483, "eval_completions/min_length": 132.17975567190226, "eval_completions/min_terminated_length": 132.17975567190226, "eval_entropy": 1.0674869646472664, "eval_frac_reward_zero_std": 0.4956369982547993, "eval_kl": 0.008830279561270022, "eval_loss": 0.00042557140113785863, "eval_num_tokens": 93239934.0, "eval_reward": -0.11092931937172774, "eval_reward_std": 0.442749046478388, "eval_rewards/reward_fn/mean": -0.11092931937172774, "eval_rewards/reward_fn/std": 0.4427490595851685, "eval_runtime": 1710.2262, "eval_samples_per_second": 0.335, "eval_sampling/importance_sampling_ratio/max": 1.2751935614341217, "eval_sampling/importance_sampling_ratio/mean": 1.0000706204777197, "eval_sampling/importance_sampling_ratio/min": 0.7732205846546832, "eval_sampling/sampling_logp_difference/max": 0.2919169586886494, "eval_sampling/sampling_logp_difference/mean": 0.020591774222361376, "eval_steps_per_second": 0.042, "step": 21400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 212.025, "completions/mean_terminated_length": 212.025, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.0434822984039784, "epoch": 1.9683736324354142, "frac_reward_zero_std": 0.5, "grad_norm": 0.08497728407382965, "kl": 0.008601493091555312, "learning_rate": 1.4498689116038408e-06, "loss": 0.001, "num_tokens": 93283088.0, "reward": -0.125, "reward_std": 0.4772168457508087, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4772168695926666, "sampling/importance_sampling_ratio/max": 1.2524933576583863, "sampling/importance_sampling_ratio/mean": 1.0003692090511322, "sampling/importance_sampling_ratio/min": 0.7504328787326813, "sampling/sampling_logp_difference/max": 0.30808199644088746, "sampling/sampling_logp_difference/mean": 0.020441557746380568, "step": 21410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.8, "completions/max_terminated_length": 291.8, "completions/mean_length": 203.1125, "completions/mean_terminated_length": 203.1125, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 1.0664639480412006, "epoch": 1.9692930035855474, "frac_reward_zero_std": 0.5, "grad_norm": 0.08093928545713425, "kl": 0.010116172200650909, "learning_rate": 1.4475701877438598e-06, "loss": 0.0005, "num_tokens": 93324825.0, "reward": -0.075, "reward_std": 0.4116185367107391, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4116185367107391, "sampling/importance_sampling_ratio/max": 1.2618757247924806, "sampling/importance_sampling_ratio/mean": 1.00005641579628, "sampling/importance_sampling_ratio/min": 0.7976100146770477, "sampling/sampling_logp_difference/max": 0.25101938247680666, "sampling/sampling_logp_difference/mean": 0.02109762318432331, "step": 21420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.1, "completions/max_terminated_length": 277.1, "completions/mean_length": 185.6875, "completions/mean_terminated_length": 185.6875, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.0510615192353725, "epoch": 1.9702123747356808, "frac_reward_zero_std": 0.4, "grad_norm": 6.695077896118164, "kl": 0.009869610078749248, "learning_rate": 1.4452725448136595e-06, "loss": 0.0004, "num_tokens": 93364648.0, "reward": -0.1, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.2931201100349425, "sampling/importance_sampling_ratio/mean": 0.9999848783016205, "sampling/importance_sampling_ratio/min": 0.7640893876552581, "sampling/sampling_logp_difference/max": 0.2887654542922974, "sampling/sampling_logp_difference/mean": 0.021000133454799653, "step": 21430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 201.125, "completions/mean_terminated_length": 201.125, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.1154683761298656, "epoch": 1.9711317458858142, "frac_reward_zero_std": 0.5, "grad_norm": 6.7460503578186035, "kl": 0.009734181276871822, "learning_rate": 1.4429759851731028e-06, "loss": 0.0005, "num_tokens": 93409410.0, "reward": -0.15, "reward_std": 0.45196654200553893, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4519665539264679, "sampling/importance_sampling_ratio/max": 1.2376177072525025, "sampling/importance_sampling_ratio/mean": 0.9997538983821869, "sampling/importance_sampling_ratio/min": 0.7984110057353974, "sampling/sampling_logp_difference/max": 0.24720749855041504, "sampling/sampling_logp_difference/mean": 0.021816975250840188, "step": 21440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.4, "completions/max_terminated_length": 323.4, "completions/mean_length": 221.4, "completions/mean_terminated_length": 221.4, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.1328480400145053, "epoch": 1.9720511170359474, "frac_reward_zero_std": 0.3, "grad_norm": 5.778748512268066, "kl": 0.008796060617896728, "learning_rate": 1.440680511180939e-06, "loss": 0.0004, "num_tokens": 93458602.0, "reward": 0.15, "reward_std": 0.673308688402176, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6733087241649628, "sampling/importance_sampling_ratio/max": 1.3038962960243226, "sampling/importance_sampling_ratio/mean": 0.9997676849365235, "sampling/importance_sampling_ratio/min": 0.7814196288585663, "sampling/sampling_logp_difference/max": 0.28362579345703126, "sampling/sampling_logp_difference/mean": 0.02207519318908453, "step": 21450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 209.05, "completions/mean_terminated_length": 209.05, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.0797960437834262, "epoch": 1.9729704881860806, "frac_reward_zero_std": 0.6, "grad_norm": 0.12045645713806152, "kl": 0.009199884481495247, "learning_rate": 1.4383861251948051e-06, "loss": 0.0005, "num_tokens": 93503070.0, "reward": -0.45, "reward_std": 0.3552303433418274, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.3552303433418274, "sampling/importance_sampling_ratio/max": 1.2975033402442933, "sampling/importance_sampling_ratio/mean": 1.0004236996173859, "sampling/importance_sampling_ratio/min": 0.707491996884346, "sampling/sampling_logp_difference/max": 0.40364747047424315, "sampling/sampling_logp_difference/mean": 0.022093219123780726, "step": 21460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 213.525, "completions/mean_terminated_length": 213.525, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.0651799865067004, "epoch": 1.973889859336214, "frac_reward_zero_std": 0.4, "grad_norm": 0.09465870261192322, "kl": 0.009218292962759733, "learning_rate": 1.43609282957122e-06, "loss": 0.0005, "num_tokens": 93547752.0, "reward": 0.175, "reward_std": 0.4789345502853394, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4789345622062683, "sampling/importance_sampling_ratio/max": 1.3072693467140197, "sampling/importance_sampling_ratio/mean": 1.0001603007316588, "sampling/importance_sampling_ratio/min": 0.8003500580787659, "sampling/sampling_logp_difference/max": 0.27794320583343507, "sampling/sampling_logp_difference/mean": 0.021253622882068158, "step": 21470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.6, "completions/max_terminated_length": 312.6, "completions/mean_length": 209.875, "completions/mean_terminated_length": 209.875, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 1.113317960500717, "epoch": 1.9748092304863474, "frac_reward_zero_std": 0.4, "grad_norm": 6.560885906219482, "kl": 0.009883717505726964, "learning_rate": 1.4338006266655818e-06, "loss": 0.0006, "num_tokens": 93591214.0, "reward": -0.325, "reward_std": 0.5117337048053742, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5117337286472321, "sampling/importance_sampling_ratio/max": 1.2596405267715454, "sampling/importance_sampling_ratio/mean": 0.9996323764324189, "sampling/importance_sampling_ratio/min": 0.7369710743427277, "sampling/sampling_logp_difference/max": 0.3234143704175949, "sampling/sampling_logp_difference/mean": 0.022100157104432583, "step": 21480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 283.8, "completions/max_terminated_length": 265.7, "completions/mean_length": 201.7875, "completions/mean_terminated_length": 198.725, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.0601686455309391, "epoch": 1.9757286016364808, "frac_reward_zero_std": 0.2, "grad_norm": 6.3760786056518555, "kl": 0.00943960048898589, "learning_rate": 1.4315095188321654e-06, "loss": 0.0001, "num_tokens": 93632269.0, "reward": -0.13125, "reward_std": 0.6830074071884156, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.6830074250698089, "sampling/importance_sampling_ratio/max": 1.234766912460327, "sampling/importance_sampling_ratio/mean": 1.0005300164222717, "sampling/importance_sampling_ratio/min": 0.7954684257507324, "sampling/sampling_logp_difference/max": 0.23304991722106932, "sampling/sampling_logp_difference/mean": 0.021271892450749876, "step": 21490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 198.575, "completions/mean_terminated_length": 198.575, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 0.9282707467675209, "epoch": 1.976647972786614, "frac_reward_zero_std": 0.7, "grad_norm": 0.07938052713871002, "kl": 0.00789312741253525, "learning_rate": 1.4292195084241238e-06, "loss": 0.0005, "num_tokens": 93677083.0, "reward": 0.075, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.295322549343109, "sampling/importance_sampling_ratio/mean": 0.9999298453330994, "sampling/importance_sampling_ratio/min": 0.772967517375946, "sampling/sampling_logp_difference/max": 0.3000467538833618, "sampling/sampling_logp_difference/mean": 0.01988964229822159, "step": 21500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.5, "completions/max_terminated_length": 327.5, "completions/mean_length": 214.2125, "completions/mean_terminated_length": 214.2125, "completions/min_length": 128.5, "completions/min_terminated_length": 128.5, "entropy": 1.1060553826391697, "epoch": 1.9775673439367472, "frac_reward_zero_std": 0.8, "grad_norm": 0.10893703997135162, "kl": 0.008911721856566145, "learning_rate": 1.4269305977934803e-06, "loss": 0.0002, "num_tokens": 93720588.0, "reward": -0.55, "reward_std": 0.1414213538169861, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.1414213538169861, "sampling/importance_sampling_ratio/max": 1.2637588620185851, "sampling/importance_sampling_ratio/mean": 0.9997640550136566, "sampling/importance_sampling_ratio/min": 0.7666120171546936, "sampling/sampling_logp_difference/max": 0.27713537216186523, "sampling/sampling_logp_difference/mean": 0.021581856533885002, "step": 21510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 211.65, "completions/mean_terminated_length": 211.65, "completions/min_length": 150.1, "completions/min_terminated_length": 150.1, "entropy": 1.1145466603338718, "epoch": 1.9784867150868806, "frac_reward_zero_std": 0.4, "grad_norm": 8.64287281036377, "kl": 0.008502916849101894, "learning_rate": 1.4246427892911305e-06, "loss": 0.0003, "num_tokens": 93767784.0, "reward": -0.175, "reward_std": 0.6025980114936829, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.6025980472564697, "sampling/importance_sampling_ratio/max": 1.2852711081504822, "sampling/importance_sampling_ratio/mean": 1.00031458735466, "sampling/importance_sampling_ratio/min": 0.7381415158510208, "sampling/sampling_logp_difference/max": 0.3540095388889313, "sampling/sampling_logp_difference/mean": 0.021285529434680938, "step": 21520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.4, "completions/max_terminated_length": 277.4, "completions/mean_length": 188.15, "completions/mean_terminated_length": 188.15, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0043067619204522, "epoch": 1.979406086237014, "frac_reward_zero_std": 0.6, "grad_norm": 5.609437942504883, "kl": 0.008237260664463975, "learning_rate": 1.4223560852668355e-06, "loss": 0.0005, "num_tokens": 93809212.0, "reward": 0.35, "reward_std": 0.3846505284309387, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.3846505284309387, "sampling/importance_sampling_ratio/max": 1.2382500052452088, "sampling/importance_sampling_ratio/mean": 1.000549989938736, "sampling/importance_sampling_ratio/min": 0.7563754439353942, "sampling/sampling_logp_difference/max": 0.31139219999313356, "sampling/sampling_logp_difference/mean": 0.020378333702683448, "step": 21530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.2, "completions/max_terminated_length": 302.2, "completions/mean_length": 210.7875, "completions/mean_terminated_length": 210.7875, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.043478263169527, "epoch": 1.9803254573871472, "frac_reward_zero_std": 0.8, "grad_norm": 0.10505077242851257, "kl": 0.009481960706762039, "learning_rate": 1.4200704880692257e-06, "loss": 0.0005, "num_tokens": 93852067.0, "reward": -0.05, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.291772711277008, "sampling/importance_sampling_ratio/mean": 0.9989653289318084, "sampling/importance_sampling_ratio/min": 0.7513624906539917, "sampling/sampling_logp_difference/max": 0.32402768135070803, "sampling/sampling_logp_difference/mean": 0.02105503622442484, "step": 21540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 277.4, "completions/max_terminated_length": 258.0, "completions/mean_length": 184.625, "completions/mean_terminated_length": 181.26071472167968, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 1.1135406002402306, "epoch": 1.9812448285372803, "frac_reward_zero_std": 0.3, "grad_norm": 0.08498881757259369, "kl": 0.008719134182319977, "learning_rate": 1.4177860000457904e-06, "loss": 0.0004, "num_tokens": 93891565.0, "reward": -0.30625, "reward_std": 0.5328060373663902, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5328060492873192, "sampling/importance_sampling_ratio/max": 1.2431395530700684, "sampling/importance_sampling_ratio/mean": 0.9996436476707459, "sampling/importance_sampling_ratio/min": 0.7553898394107819, "sampling/sampling_logp_difference/max": 0.28702686429023744, "sampling/sampling_logp_difference/mean": 0.022614802233874798, "step": 21550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.7, "completions/max_terminated_length": 262.7, "completions/mean_length": 175.775, "completions/mean_terminated_length": 175.775, "completions/min_length": 112.5, "completions/min_terminated_length": 112.5, "entropy": 1.03332701548934, "epoch": 1.9821641996874138, "frac_reward_zero_std": 0.6, "grad_norm": 0.13541345298290253, "kl": 0.009644349131849594, "learning_rate": 1.4155026235428845e-06, "loss": 0.0007, "num_tokens": 93930475.0, "reward": 0.05, "reward_std": 0.3846505284309387, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3846505284309387, "sampling/importance_sampling_ratio/max": 1.227338707447052, "sampling/importance_sampling_ratio/mean": 1.0001045703887939, "sampling/importance_sampling_ratio/min": 0.7862312436103821, "sampling/sampling_logp_difference/max": 0.2585774064064026, "sampling/sampling_logp_difference/mean": 0.02089589964598417, "step": 21560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 277.6, "completions/max_terminated_length": 261.2, "completions/mean_length": 211.8, "completions/mean_terminated_length": 204.3300018310547, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 0.9683435015380383, "epoch": 1.9830835708375472, "frac_reward_zero_std": 0.7, "grad_norm": 0.07572253793478012, "kl": 0.007884834599099121, "learning_rate": 1.4132203609057171e-06, "loss": 0.0006, "num_tokens": 93973283.0, "reward": -0.01875, "reward_std": 0.27819586992263795, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.27819588780403137, "sampling/importance_sampling_ratio/max": 1.2585960388183595, "sampling/importance_sampling_ratio/mean": 1.00076225399971, "sampling/importance_sampling_ratio/min": 0.754021281003952, "sampling/sampling_logp_difference/max": 0.2994255840778351, "sampling/sampling_logp_difference/mean": 0.01979872789233923, "step": 21570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 203.7, "completions/mean_terminated_length": 203.7, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0723406828939914, "epoch": 1.9840029419876806, "frac_reward_zero_std": 0.5, "grad_norm": 4.761844635009766, "kl": 0.009279154014075175, "learning_rate": 1.4109392144783543e-06, "loss": 0.0005, "num_tokens": 94018371.0, "reward": 0.175, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.3242321491241456, "sampling/importance_sampling_ratio/mean": 1.000029444694519, "sampling/importance_sampling_ratio/min": 0.7706795394420624, "sampling/sampling_logp_difference/max": 0.30409027338027955, "sampling/sampling_logp_difference/mean": 0.02159315850585699, "step": 21580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.5, "completions/max_terminated_length": 263.5, "completions/mean_length": 189.8625, "completions/mean_terminated_length": 189.8625, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 0.9628456979990005, "epoch": 1.9849223131378138, "frac_reward_zero_std": 0.6, "grad_norm": 17.248661041259766, "kl": 0.013962884564534761, "learning_rate": 1.408659186603717e-06, "loss": 0.0004, "num_tokens": 94060480.0, "reward": -0.175, "reward_std": 0.40311150550842284, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.40311154127120974, "sampling/importance_sampling_ratio/max": 1.2361288785934448, "sampling/importance_sampling_ratio/mean": 1.000268805027008, "sampling/importance_sampling_ratio/min": 0.7152362942695618, "sampling/sampling_logp_difference/max": 0.5103470325469971, "sampling/sampling_logp_difference/mean": 0.020344099774956704, "step": 21590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.5, "completions/max_terminated_length": 284.5, "completions/mean_length": 202.9, "completions/mean_terminated_length": 202.9, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.1127876043319702, "epoch": 1.985841684287947, "frac_reward_zero_std": 0.5, "grad_norm": 0.12353819608688354, "kl": 0.00845032866927795, "learning_rate": 1.4063802796235775e-06, "loss": 0.0005, "num_tokens": 94101072.0, "reward": -0.13125, "reward_std": 0.39283459186553954, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.39283459186553954, "sampling/importance_sampling_ratio/max": 1.2375221014022828, "sampling/importance_sampling_ratio/mean": 0.9994907915592194, "sampling/importance_sampling_ratio/min": 0.7888697206974029, "sampling/sampling_logp_difference/max": 0.24304187297821045, "sampling/sampling_logp_difference/mean": 0.02152929287403822, "step": 21600 }, { "epoch": 1.985841684287947, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 298.58813263525303, "eval_completions/max_terminated_length": 296.95986038394415, "eval_completions/mean_length": 207.47578534031413, "eval_completions/mean_terminated_length": 207.11543265936888, "eval_completions/min_length": 135.02966841186736, "eval_completions/min_terminated_length": 135.02966841186736, "eval_entropy": 1.0786393996828723, "eval_frac_reward_zero_std": 0.4816753926701571, "eval_kl": 0.008574061769419992, "eval_loss": 0.0004343462933320552, "eval_num_tokens": 94101072.0, "eval_reward": -0.09925828970331588, "eval_reward_std": 0.45884270380928877, "eval_rewards/reward_fn/mean": -0.09925828970331588, "eval_rewards/reward_fn/std": 0.4588427182683562, "eval_runtime": 1737.0166, "eval_samples_per_second": 0.33, "eval_sampling/importance_sampling_ratio/max": 1.2666532666687358, "eval_sampling/importance_sampling_ratio/mean": 1.0000479033064051, "eval_sampling/importance_sampling_ratio/min": 0.769036017787394, "eval_sampling/sampling_logp_difference/max": 0.289765881514258, "eval_sampling/sampling_logp_difference/mean": 0.021313723145836207, "eval_steps_per_second": 0.041, "step": 21600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.4, "completions/max_terminated_length": 300.4, "completions/mean_length": 211.25, "completions/mean_terminated_length": 211.25, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 0.9892764903604985, "epoch": 1.9867610554380803, "frac_reward_zero_std": 0.7, "grad_norm": 6.574105739593506, "kl": 0.00856005776731763, "learning_rate": 1.4041024958785558e-06, "loss": 0.0005, "num_tokens": 94142916.0, "reward": 0.225, "reward_std": 0.2449311852455139, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.24493119716644288, "sampling/importance_sampling_ratio/max": 1.2504769802093505, "sampling/importance_sampling_ratio/mean": 1.0001769483089447, "sampling/importance_sampling_ratio/min": 0.7565537124872208, "sampling/sampling_logp_difference/max": 0.30401015281677246, "sampling/sampling_logp_difference/mean": 0.020485647954046726, "step": 21610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 247.6, "completions/max_terminated_length": 247.6, "completions/mean_length": 189.3875, "completions/mean_terminated_length": 189.3875, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.1049678295850753, "epoch": 1.9876804265882138, "frac_reward_zero_std": 0.5, "grad_norm": 5.587435245513916, "kl": 0.008610535913612693, "learning_rate": 1.4018258377081162e-06, "loss": 0.0006, "num_tokens": 94186867.0, "reward": 0.225, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.2465537786483765, "sampling/importance_sampling_ratio/mean": 0.9999146223068237, "sampling/importance_sampling_ratio/min": 0.7948908567428589, "sampling/sampling_logp_difference/max": 0.23390791416168213, "sampling/sampling_logp_difference/mean": 0.02163659930229187, "step": 21620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.3, "completions/max_terminated_length": 294.3, "completions/mean_length": 210.475, "completions/mean_terminated_length": 210.475, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.0509258918464184, "epoch": 1.988599797738347, "frac_reward_zero_std": 0.4, "grad_norm": 6.857363224029541, "kl": 0.009603447839617729, "learning_rate": 1.3995503074505717e-06, "loss": 0.0002, "num_tokens": 94227161.0, "reward": -0.125, "reward_std": 0.5841213405132294, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5841213643550873, "sampling/importance_sampling_ratio/max": 1.2495840907096862, "sampling/importance_sampling_ratio/mean": 0.9997598171234131, "sampling/importance_sampling_ratio/min": 0.7905757248401641, "sampling/sampling_logp_difference/max": 0.24322044253349304, "sampling/sampling_logp_difference/mean": 0.02112125661224127, "step": 21630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.3, "completions/max_terminated_length": 299.3, "completions/mean_length": 213.1625, "completions/mean_terminated_length": 213.1625, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.1306994721293449, "epoch": 1.9895191688884801, "frac_reward_zero_std": 0.6, "grad_norm": 7.7051520347595215, "kl": 0.008982811882742681, "learning_rate": 1.3972759074430725e-06, "loss": 0.0007, "num_tokens": 94271918.0, "reward": -0.3, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.2526965975761413, "sampling/importance_sampling_ratio/mean": 0.9994704961776734, "sampling/importance_sampling_ratio/min": 0.7774495542049408, "sampling/sampling_logp_difference/max": 0.26359901428222654, "sampling/sampling_logp_difference/mean": 0.02154828142374754, "step": 21640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.1, "completions/max_terminated_length": 299.1, "completions/mean_length": 220.1375, "completions/mean_terminated_length": 220.1375, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 0.9914279371500015, "epoch": 1.9904385400386135, "frac_reward_zero_std": 0.4, "grad_norm": 0.09968572109937668, "kl": 0.008921533016837202, "learning_rate": 1.3950026400216117e-06, "loss": 0.0005, "num_tokens": 94315185.0, "reward": 0.025, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.2862029552459717, "sampling/importance_sampling_ratio/mean": 0.9999895334243775, "sampling/importance_sampling_ratio/min": 0.757888787984848, "sampling/sampling_logp_difference/max": 0.29788386821746826, "sampling/sampling_logp_difference/mean": 0.020578254014253616, "step": 21650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.7, "completions/max_terminated_length": 300.7, "completions/mean_length": 213.6875, "completions/mean_terminated_length": 213.6875, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.0507075265049934, "epoch": 1.991357911188747, "frac_reward_zero_std": 0.7, "grad_norm": 4.436825752258301, "kl": 0.00990993968443945, "learning_rate": 1.3927305075210157e-06, "loss": 0.0006, "num_tokens": 94359376.0, "reward": -0.45, "reward_std": 0.24832584857940673, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.24832584857940673, "sampling/importance_sampling_ratio/max": 1.3076974153518677, "sampling/importance_sampling_ratio/mean": 0.9999273717403412, "sampling/importance_sampling_ratio/min": 0.7501609206199646, "sampling/sampling_logp_difference/max": 0.32986524105072024, "sampling/sampling_logp_difference/mean": 0.02085710596293211, "step": 21660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 330.0, "completions/max_terminated_length": 303.9, "completions/mean_length": 214.175, "completions/mean_terminated_length": 210.1482147216797, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0783175371587277, "epoch": 1.9922772823388804, "frac_reward_zero_std": 0.2, "grad_norm": 8.575685501098633, "kl": 0.011483699802192859, "learning_rate": 1.390459512274946e-06, "loss": 0.0005, "num_tokens": 94404254.0, "reward": -0.15625, "reward_std": 0.7570965170860291, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.757096529006958, "sampling/importance_sampling_ratio/max": 1.3977884650230408, "sampling/importance_sampling_ratio/mean": 1.0004794955253602, "sampling/importance_sampling_ratio/min": 0.7740804433822632, "sampling/sampling_logp_difference/max": 0.3937244415283203, "sampling/sampling_logp_difference/mean": 0.021580276265740394, "step": 21670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.7, "completions/max_terminated_length": 285.7, "completions/mean_length": 207.85, "completions/mean_terminated_length": 207.85, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.0596449866890907, "epoch": 1.9931966534890135, "frac_reward_zero_std": 0.6, "grad_norm": 0.09813715517520905, "kl": 0.008392168453428894, "learning_rate": 1.3881896566158978e-06, "loss": 0.0004, "num_tokens": 94447490.0, "reward": -0.575, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": -0.575, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2874330043792725, "sampling/importance_sampling_ratio/mean": 0.9996242582798004, "sampling/importance_sampling_ratio/min": 0.7804895102977752, "sampling/sampling_logp_difference/max": 0.2660901784896851, "sampling/sampling_logp_difference/mean": 0.02100802268832922, "step": 21680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 207.9875, "completions/mean_terminated_length": 207.9875, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.0229016989469528, "epoch": 1.9941160246391467, "frac_reward_zero_std": 0.7, "grad_norm": 0.07404714822769165, "kl": 0.008166040873038582, "learning_rate": 1.3859209428751958e-06, "loss": 0.0004, "num_tokens": 94487777.0, "reward": -0.2, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2291371822357178, "sampling/importance_sampling_ratio/mean": 0.9995391488075256, "sampling/importance_sampling_ratio/min": 0.7843645751476288, "sampling/sampling_logp_difference/max": 0.2609483003616333, "sampling/sampling_logp_difference/mean": 0.02054117899388075, "step": 21690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.8, "completions/max_terminated_length": 283.8, "completions/mean_length": 190.5875, "completions/mean_terminated_length": 190.5875, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.03384063616395, "epoch": 1.9950353957892801, "frac_reward_zero_std": 0.5, "grad_norm": 0.08714472502470016, "kl": 0.008889124784036539, "learning_rate": 1.38365337338299e-06, "loss": 0.0004, "num_tokens": 94529144.0, "reward": -0.5, "reward_std": 0.4478123545646667, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.4478123545646667, "sampling/importance_sampling_ratio/max": 1.3024282097816466, "sampling/importance_sampling_ratio/mean": 0.9997686684131623, "sampling/importance_sampling_ratio/min": 0.7332465782761574, "sampling/sampling_logp_difference/max": 0.40854289531707766, "sampling/sampling_logp_difference/mean": 0.02139609158039093, "step": 21700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.7, "completions/max_terminated_length": 292.7, "completions/mean_length": 214.525, "completions/mean_terminated_length": 214.525, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.1551872469484805, "epoch": 1.9959547669394135, "frac_reward_zero_std": 0.6, "grad_norm": 0.09500318765640259, "kl": 0.008718881011009216, "learning_rate": 1.3813869504682545e-06, "loss": 0.0007, "num_tokens": 94573714.0, "reward": -0.35, "reward_std": 0.3737070143222809, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.37370702624320984, "sampling/importance_sampling_ratio/max": 1.283760666847229, "sampling/importance_sampling_ratio/mean": 0.9992623925209045, "sampling/importance_sampling_ratio/min": 0.7964522123336792, "sampling/sampling_logp_difference/max": 0.2626445531845093, "sampling/sampling_logp_difference/mean": 0.022201108187437056, "step": 21710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.1, "completions/max_terminated_length": 276.1, "completions/mean_length": 194.975, "completions/mean_terminated_length": 194.975, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0496687449514865, "epoch": 1.9968741380895467, "frac_reward_zero_std": 0.6, "grad_norm": 0.07640994340181351, "kl": 0.0077369788312353195, "learning_rate": 1.3791216764587898e-06, "loss": 0.0004, "num_tokens": 94616432.0, "reward": 0.125, "reward_std": 0.3190365254878998, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3190365254878998, "sampling/importance_sampling_ratio/max": 1.2828744530677796, "sampling/importance_sampling_ratio/mean": 0.999929791688919, "sampling/importance_sampling_ratio/min": 0.7616388380527497, "sampling/sampling_logp_difference/max": 0.2945637941360474, "sampling/sampling_logp_difference/mean": 0.021316534094512462, "step": 21720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.4, "completions/max_terminated_length": 271.4, "completions/mean_length": 207.8875, "completions/mean_terminated_length": 207.8875, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.0675595954060555, "epoch": 1.99779350923968, "frac_reward_zero_std": 0.3, "grad_norm": 8.848594665527344, "kl": 0.00961280326009728, "learning_rate": 1.3768575536812118e-06, "loss": 0.0004, "num_tokens": 94658951.0, "reward": -0.1, "reward_std": 0.6657755315303803, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.6657755434513092, "sampling/importance_sampling_ratio/max": 1.2642495393753053, "sampling/importance_sampling_ratio/mean": 0.9998249053955078, "sampling/importance_sampling_ratio/min": 0.7584084451198578, "sampling/sampling_logp_difference/max": 0.32046878933906553, "sampling/sampling_logp_difference/mean": 0.021368795447051524, "step": 21730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.4, "completions/max_terminated_length": 257.4, "completions/mean_length": 190.5875, "completions/mean_terminated_length": 190.5875, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.0232609782367945, "epoch": 1.9987128803898133, "frac_reward_zero_std": 0.4, "grad_norm": 5.397899150848389, "kl": 0.008146422638674267, "learning_rate": 1.3745945844609583e-06, "loss": 0.0005, "num_tokens": 94697174.0, "reward": 0.05, "reward_std": 0.5260718822479248, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5260718822479248, "sampling/importance_sampling_ratio/max": 1.335992980003357, "sampling/importance_sampling_ratio/mean": 0.9997416913509369, "sampling/importance_sampling_ratio/min": 0.7773600935935974, "sampling/sampling_logp_difference/max": 0.3143240213394165, "sampling/sampling_logp_difference/mean": 0.021101565659046174, "step": 21740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.2, "completions/max_terminated_length": 286.2, "completions/mean_length": 192.975, "completions/mean_terminated_length": 192.975, "completions/min_length": 116.8, "completions/min_terminated_length": 116.8, "entropy": 1.0499390862882136, "epoch": 1.9996322515399467, "frac_reward_zero_std": 0.5, "grad_norm": 0.0915284976363182, "kl": 0.007620611879974604, "learning_rate": 1.3723327711222772e-06, "loss": 0.0004, "num_tokens": 94738420.0, "reward": 0.125, "reward_std": 0.4444176912307739, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.44441770315170287, "sampling/importance_sampling_ratio/max": 1.2484152674674989, "sampling/importance_sampling_ratio/mean": 1.0004239678382874, "sampling/importance_sampling_ratio/min": 0.7854008674621582, "sampling/sampling_logp_difference/max": 0.2623586118221283, "sampling/sampling_logp_difference/mean": 0.02080706860870123, "step": 21750 } ], "logging_steps": 10, "max_steps": 32631, "num_input_tokens_seen": 94757271, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }