{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 200, "global_step": 10877, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 299.6928446771379, "eval_completions/max_terminated_length": 298.3961605584642, "eval_completions/mean_length": 209.70615183246073, "eval_completions/mean_terminated_length": 209.30374606200746, "eval_completions/min_length": 135.17102966841188, "eval_completions/min_terminated_length": 135.17102966841188, "eval_entropy": 1.0814493743745028, "eval_frac_reward_zero_std": 0.35776614310645727, "eval_kl": 0.0, "eval_loss": -2.7741376470657997e-05, "eval_num_tokens": 0.0, "eval_reward": -0.09794938917975567, "eval_reward_std": 0.5127268967464124, "eval_rewards/reward_fn/mean": -0.09794938917975567, "eval_rewards/reward_fn/std": 0.5127269140140757, "eval_runtime": 1749.9124, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2849910983240418, "eval_sampling/importance_sampling_ratio/mean": 0.9999885321911717, "eval_sampling/importance_sampling_ratio/min": 0.7644410355969041, "eval_sampling/sampling_logp_difference/max": 0.3089583640947392, "eval_sampling/sampling_logp_difference/mean": 0.021393064744315432, "eval_steps_per_second": 0.041, "step": 0 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 222.0, "completions/max_terminated_length": 222.0, "completions/mean_length": 151.625, "completions/mean_terminated_length": 151.625, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 1.0753526240587234, "epoch": 9.193711501333089e-05, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0, "num_tokens": 3285.0, "reward": 1.0, "reward_std": 0.0, "rewards/reward_fn/mean": 1.0, "rewards/reward_fn/std": 0.0, "sampling/importance_sampling_ratio/max": 1.169453501701355, "sampling/importance_sampling_ratio/mean": 0.9986120462417603, "sampling/importance_sampling_ratio/min": 0.7822896838188171, "sampling/sampling_logp_difference/max": 0.2455301284790039, "sampling/sampling_logp_difference/mean": 0.021735867485404015, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.77777777777777, "completions/max_terminated_length": 327.77777777777777, "completions/mean_length": 217.18055555555554, "completions/mean_terminated_length": 217.18055555555554, "completions/min_length": 142.33333333333334, "completions/min_terminated_length": 142.33333333333334, "entropy": 1.0297992668218083, "epoch": 0.0009193711501333088, "frac_reward_zero_std": 0.5555555555555556, "grad_norm": 0.031037449836730957, "kl": 0.0005116063932493691, "learning_rate": 2.757352941176471e-08, "loss": 0.0004, "num_tokens": 40962.0, "reward": -0.5277777777777778, "reward_std": 0.3398328456613753, "rewards/reward_fn/mean": -0.5277777777777778, "rewards/reward_fn/std": 0.3398328655295902, "sampling/importance_sampling_ratio/max": 1.2818556096818712, "sampling/importance_sampling_ratio/mean": 1.000329507721795, "sampling/importance_sampling_ratio/min": 0.7555213239457872, "sampling/sampling_logp_difference/max": 0.30765635437435573, "sampling/sampling_logp_difference/mean": 0.020564943758977786, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.3, "completions/max_terminated_length": 296.3, "completions/mean_length": 200.625, "completions/mean_terminated_length": 200.625, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.0524392507970333, "epoch": 0.0018387423002666175, "frac_reward_zero_std": 0.3, "grad_norm": 7.020498275756836, "kl": 0.0007292727361345897, "learning_rate": 5.8210784313725494e-08, "loss": 0.0002, "num_tokens": 82564.0, "reward": -0.0625, "reward_std": 0.5568657726049423, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.5568657964468002, "sampling/importance_sampling_ratio/max": 1.291491985321045, "sampling/importance_sampling_ratio/mean": 1.0000877737998963, "sampling/importance_sampling_ratio/min": 0.728471314907074, "sampling/sampling_logp_difference/max": 0.33972215056419375, "sampling/sampling_logp_difference/mean": 0.021487886272370815, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.3, "completions/max_terminated_length": 270.3, "completions/mean_length": 194.2875, "completions/mean_terminated_length": 194.2875, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.0034877896308898, "epoch": 0.0027581134503999265, "frac_reward_zero_std": 0.2, "grad_norm": 4.626897811889648, "kl": 0.000708105590820196, "learning_rate": 8.884803921568628e-08, "loss": -0.0005, "num_tokens": 123043.0, "reward": 0.21875, "reward_std": 0.6134829133749008, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.6134829312562943, "sampling/importance_sampling_ratio/max": 1.244255793094635, "sampling/importance_sampling_ratio/mean": 0.9995054066181183, "sampling/importance_sampling_ratio/min": 0.6939071506261826, "sampling/sampling_logp_difference/max": 0.4048956513404846, "sampling/sampling_logp_difference/mean": 0.020845489762723445, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 306.9, "completions/max_terminated_length": 297.1, "completions/mean_length": 211.3125, "completions/mean_terminated_length": 208.38214416503905, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.1050874687731267, "epoch": 0.003677484600533235, "frac_reward_zero_std": 0.3, "grad_norm": 7.555081367492676, "kl": 0.0007166219835198717, "learning_rate": 1.1948529411764705e-07, "loss": 0.0003, "num_tokens": 168684.0, "reward": -0.21875, "reward_std": 0.5781980097293854, "rewards/reward_fn/mean": -0.21875, "rewards/reward_fn/std": 0.5781980156898499, "sampling/importance_sampling_ratio/max": 1.284425187110901, "sampling/importance_sampling_ratio/mean": 1.0003512799739838, "sampling/importance_sampling_ratio/min": 0.7563988029956817, "sampling/sampling_logp_difference/max": 0.31729066371917725, "sampling/sampling_logp_difference/mean": 0.0213082455098629, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 205.8, "completions/mean_terminated_length": 205.8, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.0357854425907136, "epoch": 0.004596855750666544, "frac_reward_zero_std": 0.3, "grad_norm": 6.304925918579102, "kl": 0.0006978385630645789, "learning_rate": 1.5012254901960787e-07, "loss": 0.0003, "num_tokens": 212108.0, "reward": -0.05, "reward_std": 0.5523674607276916, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5523674786090851, "sampling/importance_sampling_ratio/max": 1.2830299735069275, "sampling/importance_sampling_ratio/mean": 0.9998844981193542, "sampling/importance_sampling_ratio/min": 0.8075290381908417, "sampling/sampling_logp_difference/max": 0.25767743587493896, "sampling/sampling_logp_difference/mean": 0.02069554980844259, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.4, "completions/max_terminated_length": 331.4, "completions/mean_length": 239.7875, "completions/mean_terminated_length": 239.7875, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.0629674822092057, "epoch": 0.005516226900799853, "frac_reward_zero_std": 0.2, "grad_norm": 0.019427070394158363, "kl": 0.0006944268512597773, "learning_rate": 1.8075980392156866e-07, "loss": 0.0001, "num_tokens": 260955.0, "reward": -0.3125, "reward_std": 0.6532785326242447, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.6532785505056381, "sampling/importance_sampling_ratio/max": 1.3109706163406372, "sampling/importance_sampling_ratio/mean": 1.0002642214298247, "sampling/importance_sampling_ratio/min": 0.764760285615921, "sampling/sampling_logp_difference/max": 0.3132571816444397, "sampling/sampling_logp_difference/mean": 0.021548254787921904, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 348.8, "completions/max_terminated_length": 344.9, "completions/mean_length": 229.325, "completions/mean_terminated_length": 227.15714416503906, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.0357748992741107, "epoch": 0.006435598050933162, "frac_reward_zero_std": 0.2, "grad_norm": 4.994715213775635, "kl": 0.0006824022573709953, "learning_rate": 2.1139705882352945e-07, "loss": -0.0003, "num_tokens": 306925.0, "reward": -0.23125, "reward_std": 0.5558717578649521, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5558717578649521, "sampling/importance_sampling_ratio/max": 1.2648253917694092, "sampling/importance_sampling_ratio/mean": 1.0005976200103759, "sampling/importance_sampling_ratio/min": 0.7736561179161072, "sampling/sampling_logp_difference/max": 0.26914299130439756, "sampling/sampling_logp_difference/mean": 0.019995733350515365, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 213.5875, "completions/mean_terminated_length": 213.5875, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.1275035381317138, "epoch": 0.00735496920106647, "frac_reward_zero_std": 0.3, "grad_norm": 7.821847915649414, "kl": 0.0007489944571716478, "learning_rate": 2.420343137254902e-07, "loss": -0.0001, "num_tokens": 348940.0, "reward": 0.0, "reward_std": 0.5335024356842041, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5335024356842041, "sampling/importance_sampling_ratio/max": 1.2393616318702698, "sampling/importance_sampling_ratio/mean": 1.0000521004199983, "sampling/importance_sampling_ratio/min": 0.7741144061088562, "sampling/sampling_logp_difference/max": 0.27061805725097654, "sampling/sampling_logp_difference/mean": 0.022378447465598583, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.9, "completions/max_terminated_length": 305.9, "completions/mean_length": 215.5375, "completions/mean_terminated_length": 215.5375, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1076096400618554, "epoch": 0.00827434035119978, "frac_reward_zero_std": 0.6, "grad_norm": 0.026966840028762817, "kl": 0.0007253596941154683, "learning_rate": 2.7267156862745103e-07, "loss": -0.0003, "num_tokens": 393743.0, "reward": 0.09375, "reward_std": 0.3553183853626251, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.35531840324401853, "sampling/importance_sampling_ratio/max": 1.2905601978302002, "sampling/importance_sampling_ratio/mean": 1.0002325773239136, "sampling/importance_sampling_ratio/min": 0.7468384265899658, "sampling/sampling_logp_difference/max": 0.31531521677970886, "sampling/sampling_logp_difference/mean": 0.022032690793275835, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 215.5, "completions/mean_terminated_length": 215.5, "completions/min_length": 120.1, "completions/min_terminated_length": 120.1, "entropy": 1.0822462134063244, "epoch": 0.009193711501333088, "frac_reward_zero_std": 0.3, "grad_norm": 7.044803142547607, "kl": 0.0007906028386059916, "learning_rate": 3.033088235294118e-07, "loss": 0.0005, "num_tokens": 439087.0, "reward": 0.1625, "reward_std": 0.5826104313135148, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.582610473036766, "sampling/importance_sampling_ratio/max": 1.2626904606819154, "sampling/importance_sampling_ratio/mean": 1.0001648664474487, "sampling/importance_sampling_ratio/min": 0.7490761339664459, "sampling/sampling_logp_difference/max": 0.30968942642211916, "sampling/sampling_logp_difference/mean": 0.02173212803900242, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.0, "completions/max_terminated_length": 264.0, "completions/mean_length": 183.9, "completions/mean_terminated_length": 183.9, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 1.075470769405365, "epoch": 0.010113082651466397, "frac_reward_zero_std": 0.3, "grad_norm": 0.02855364978313446, "kl": 0.0007278633313035244, "learning_rate": 3.3394607843137256e-07, "loss": -0.0003, "num_tokens": 477567.0, "reward": 0.40625, "reward_std": 0.5642594993114471, "rewards/reward_fn/mean": 0.40625, "rewards/reward_fn/std": 0.5642595171928406, "sampling/importance_sampling_ratio/max": 1.251785933971405, "sampling/importance_sampling_ratio/mean": 0.9998964726924896, "sampling/importance_sampling_ratio/min": 0.7826918303966522, "sampling/sampling_logp_difference/max": 0.25253567695617674, "sampling/sampling_logp_difference/mean": 0.02239386420696974, "step": 110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 201.75, "completions/mean_terminated_length": 201.75, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.021649543941021, "epoch": 0.011032453801599706, "frac_reward_zero_std": 0.1, "grad_norm": 4.212679862976074, "kl": 0.0007115950233128387, "learning_rate": 3.6458333333333337e-07, "loss": 0.0003, "num_tokens": 519043.0, "reward": 0.2625, "reward_std": 0.8656547248363495, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.8656547486782074, "sampling/importance_sampling_ratio/max": 1.288342821598053, "sampling/importance_sampling_ratio/mean": 1.0001618027687074, "sampling/importance_sampling_ratio/min": 0.770609176158905, "sampling/sampling_logp_difference/max": 0.2823983430862427, "sampling/sampling_logp_difference/mean": 0.02057406920939684, "step": 120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 199.3375, "completions/mean_terminated_length": 199.3375, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0881411455571652, "epoch": 0.011951824951733015, "frac_reward_zero_std": 0.2, "grad_norm": 7.2945027351379395, "kl": 0.0007851745387597476, "learning_rate": 3.9522058823529414e-07, "loss": -0.0002, "num_tokens": 562190.0, "reward": -0.0375, "reward_std": 0.6773951411247253, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.6773951649665833, "sampling/importance_sampling_ratio/max": 1.3473823308944701, "sampling/importance_sampling_ratio/mean": 1.0005816757678985, "sampling/importance_sampling_ratio/min": 0.7659225463867188, "sampling/sampling_logp_difference/max": 0.3466596841812134, "sampling/sampling_logp_difference/mean": 0.022084593400359155, "step": 130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.0, "completions/max_terminated_length": 307.0, "completions/mean_length": 210.625, "completions/mean_terminated_length": 210.625, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0419432297348976, "epoch": 0.012871196101866324, "frac_reward_zero_std": 0.6, "grad_norm": 0.04254626855254173, "kl": 0.0007189964184362907, "learning_rate": 4.2585784313725495e-07, "loss": -0.0001, "num_tokens": 608976.0, "reward": -0.525, "reward_std": 0.3205240786075592, "rewards/reward_fn/mean": -0.525, "rewards/reward_fn/std": 0.3205240905284882, "sampling/importance_sampling_ratio/max": 1.2970109224319457, "sampling/importance_sampling_ratio/mean": 1.000120460987091, "sampling/importance_sampling_ratio/min": 0.7898787200450897, "sampling/sampling_logp_difference/max": 0.2912539839744568, "sampling/sampling_logp_difference/mean": 0.02070650216192007, "step": 140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.6, "completions/max_terminated_length": 309.6, "completions/mean_length": 217.15, "completions/mean_terminated_length": 217.15, "completions/min_length": 134.9, "completions/min_terminated_length": 134.9, "entropy": 1.067040916532278, "epoch": 0.013790567251999633, "frac_reward_zero_std": 0.3, "grad_norm": 7.192220211029053, "kl": 0.0007091597442922648, "learning_rate": 4.564950980392157e-07, "loss": -0.0003, "num_tokens": 652860.0, "reward": -0.15, "reward_std": 0.5939657628536225, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5939657747745514, "sampling/importance_sampling_ratio/max": 1.2563103079795837, "sampling/importance_sampling_ratio/mean": 0.9997175633907318, "sampling/importance_sampling_ratio/min": 0.7330136477947236, "sampling/sampling_logp_difference/max": 0.3202814102172852, "sampling/sampling_logp_difference/mean": 0.021304642595350743, "step": 150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 179.15, "completions/mean_terminated_length": 179.15, "completions/min_length": 117.7, "completions/min_terminated_length": 117.7, "entropy": 1.0170505791902542, "epoch": 0.01470993840213294, "frac_reward_zero_std": 0.5, "grad_norm": 3.921785354614258, "kl": 0.0007678415953705553, "learning_rate": 4.871323529411765e-07, "loss": 0.0001, "num_tokens": 689424.0, "reward": -0.29375, "reward_std": 0.3700309723615646, "rewards/reward_fn/mean": -0.29375, "rewards/reward_fn/std": 0.37003099620342256, "sampling/importance_sampling_ratio/max": 1.264707350730896, "sampling/importance_sampling_ratio/mean": 1.0004965245723725, "sampling/importance_sampling_ratio/min": 0.7463383138179779, "sampling/sampling_logp_difference/max": 0.32123899459838867, "sampling/sampling_logp_difference/mean": 0.021498026326298714, "step": 160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 202.5625, "completions/mean_terminated_length": 202.5625, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0950105614960193, "epoch": 0.01562930955226625, "frac_reward_zero_std": 0.6, "grad_norm": 0.021143656224012375, "kl": 0.0006966964305320289, "learning_rate": 5.177696078431374e-07, "loss": -0.0001, "num_tokens": 734909.0, "reward": 0.1875, "reward_std": 0.3302770793437958, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.33027708530426025, "sampling/importance_sampling_ratio/max": 1.274586832523346, "sampling/importance_sampling_ratio/mean": 1.0002855062484741, "sampling/importance_sampling_ratio/min": 0.7830502808094024, "sampling/sampling_logp_difference/max": 0.2583516001701355, "sampling/sampling_logp_difference/mean": 0.021897630393505098, "step": 170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 215.975, "completions/mean_terminated_length": 215.975, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.1032920729368925, "epoch": 0.01654868070239956, "frac_reward_zero_std": 0.2, "grad_norm": 6.1822686195373535, "kl": 0.0006948265341634397, "learning_rate": 5.484068627450981e-07, "loss": 0.0001, "num_tokens": 784027.0, "reward": -0.03125, "reward_std": 0.676500204205513, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.676500216126442, "sampling/importance_sampling_ratio/max": 1.2685070157051086, "sampling/importance_sampling_ratio/mean": 1.000198769569397, "sampling/importance_sampling_ratio/min": 0.779697036743164, "sampling/sampling_logp_difference/max": 0.26588326692581177, "sampling/sampling_logp_difference/mean": 0.021168076153844594, "step": 180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.1, "completions/max_terminated_length": 272.1, "completions/mean_length": 197.775, "completions/mean_terminated_length": 197.775, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0987628474831581, "epoch": 0.01746805185253287, "frac_reward_zero_std": 0.4, "grad_norm": 0.030823295935988426, "kl": 0.0007605922815855593, "learning_rate": 5.790441176470589e-07, "loss": 0.0002, "num_tokens": 826953.0, "reward": -0.175, "reward_std": 0.5697988569736481, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.569798880815506, "sampling/importance_sampling_ratio/max": 1.3147042989730835, "sampling/importance_sampling_ratio/mean": 0.9999885618686676, "sampling/importance_sampling_ratio/min": 0.7843068599700928, "sampling/sampling_logp_difference/max": 0.29863872528076174, "sampling/sampling_logp_difference/mean": 0.02155501451343298, "step": 190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.0, "completions/max_terminated_length": 314.0, "completions/mean_length": 215.0, "completions/mean_terminated_length": 215.0, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.067587051168084, "epoch": 0.018387423002666176, "frac_reward_zero_std": 0.1, "grad_norm": 5.829469680786133, "kl": 0.0007320545670154388, "learning_rate": 6.096813725490197e-07, "loss": 0.0007, "num_tokens": 870889.0, "reward": 0.025, "reward_std": 0.840067332983017, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.8400673389434814, "sampling/importance_sampling_ratio/max": 1.2767957687377929, "sampling/importance_sampling_ratio/mean": 1.0000281572341918, "sampling/importance_sampling_ratio/min": 0.7662912130355835, "sampling/sampling_logp_difference/max": 0.27752859592437745, "sampling/sampling_logp_difference/mean": 0.020865828078240157, "step": 200 }, { "epoch": 0.018387423002666176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0034904013961605585, "eval_completions/max_length": 305.3438045375218, "eval_completions/max_terminated_length": 301.54275741710296, "eval_completions/mean_length": 209.8656195462478, "eval_completions/mean_terminated_length": 208.9629209561706, "eval_completions/min_length": 134.87085514834206, "eval_completions/min_terminated_length": 134.87085514834206, "eval_entropy": 1.085793704886711, "eval_frac_reward_zero_std": 0.3612565445026178, "eval_kl": 0.0007578833909915763, "eval_loss": 4.360772800282575e-05, "eval_num_tokens": 870889.0, "eval_reward": -0.10056719022687609, "eval_reward_std": 0.49869164122336823, "eval_rewards/reward_fn/mean": -0.10056719022687609, "eval_rewards/reward_fn/std": 0.4986916586990756, "eval_runtime": 1772.483, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2969242348928933, "eval_sampling/importance_sampling_ratio/mean": 0.9999848889222736, "eval_sampling/importance_sampling_ratio/min": 0.768506151011269, "eval_sampling/sampling_logp_difference/max": 0.30644968739770884, "eval_sampling/sampling_logp_difference/mean": 0.021337046208311956, "eval_steps_per_second": 0.041, "step": 200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 363.8, "completions/max_terminated_length": 363.8, "completions/mean_length": 235.2125, "completions/mean_terminated_length": 235.2125, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.0829980380833149, "epoch": 0.019306794152799484, "frac_reward_zero_std": 0.3, "grad_norm": 5.55156135559082, "kl": 0.0007172363591962494, "learning_rate": 6.403186274509804e-07, "loss": -0.0003, "num_tokens": 918906.0, "reward": -0.15, "reward_std": 0.5179141879081726, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5179141938686371, "sampling/importance_sampling_ratio/max": 1.2774083375930787, "sampling/importance_sampling_ratio/mean": 0.9995524644851684, "sampling/importance_sampling_ratio/min": 0.7407529830932618, "sampling/sampling_logp_difference/max": 0.3175777614116669, "sampling/sampling_logp_difference/mean": 0.021061371825635434, "step": 210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 359.2, "completions/max_terminated_length": 349.1, "completions/mean_length": 236.9, "completions/mean_terminated_length": 231.30893096923828, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1162165336310863, "epoch": 0.020226165302932794, "frac_reward_zero_std": 0.2, "grad_norm": 6.4660186767578125, "kl": 0.000740324639264145, "learning_rate": 6.709558823529412e-07, "loss": -0.0001, "num_tokens": 967226.0, "reward": 0.0375, "reward_std": 0.6329803034663201, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.6329803392291069, "sampling/importance_sampling_ratio/max": 1.2704672336578369, "sampling/importance_sampling_ratio/mean": 0.9998895943164825, "sampling/importance_sampling_ratio/min": 0.7882960200309753, "sampling/sampling_logp_difference/max": 0.26379434466362, "sampling/sampling_logp_difference/mean": 0.02106638755649328, "step": 220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 208.3875, "completions/mean_terminated_length": 208.3875, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.094875906407833, "epoch": 0.0211455364530661, "frac_reward_zero_std": 0.4, "grad_norm": 0.02850279025733471, "kl": 0.000748710538027808, "learning_rate": 7.01593137254902e-07, "loss": -0.0001, "num_tokens": 1010537.0, "reward": -0.26875, "reward_std": 0.3905462041497231, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.390546216070652, "sampling/importance_sampling_ratio/max": 1.2966946840286255, "sampling/importance_sampling_ratio/mean": 1.0003600001335144, "sampling/importance_sampling_ratio/min": 0.7723422288894654, "sampling/sampling_logp_difference/max": 0.2936273396015167, "sampling/sampling_logp_difference/mean": 0.021441088058054447, "step": 230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 395.5, "completions/max_terminated_length": 354.6, "completions/mean_length": 250.4125, "completions/mean_terminated_length": 241.62857513427736, "completions/min_length": 150.6, "completions/min_terminated_length": 150.6, "entropy": 1.126352274417877, "epoch": 0.022064907603199412, "frac_reward_zero_std": 0.3, "grad_norm": 0.024711329489946365, "kl": 0.0007412755752739031, "learning_rate": 7.322303921568629e-07, "loss": 0.0006, "num_tokens": 1058626.0, "reward": -0.30625, "reward_std": 0.5605054497718811, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.560505473613739, "sampling/importance_sampling_ratio/max": 1.3337203502655028, "sampling/importance_sampling_ratio/mean": 0.9998445868492126, "sampling/importance_sampling_ratio/min": 0.7922480344772339, "sampling/sampling_logp_difference/max": 0.29643134474754335, "sampling/sampling_logp_difference/mean": 0.02184301670640707, "step": 240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 224.6125, "completions/mean_terminated_length": 224.6125, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.0780830763280391, "epoch": 0.02298427875333272, "frac_reward_zero_std": 0.5, "grad_norm": 0.025984929874539375, "kl": 0.0007190637981693726, "learning_rate": 7.628676470588237e-07, "loss": -0.0, "num_tokens": 1102371.0, "reward": -0.58125, "reward_std": 0.4084311366081238, "rewards/reward_fn/mean": -0.58125, "rewards/reward_fn/std": 0.4084311366081238, "sampling/importance_sampling_ratio/max": 1.2953303456306458, "sampling/importance_sampling_ratio/mean": 1.000216645002365, "sampling/importance_sampling_ratio/min": 0.739750212430954, "sampling/sampling_logp_difference/max": 0.31953527331352233, "sampling/sampling_logp_difference/mean": 0.02097372431308031, "step": 250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.6, "completions/max_terminated_length": 307.5, "completions/mean_length": 213.625, "completions/mean_terminated_length": 209.9000015258789, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.076523457467556, "epoch": 0.02390364990346603, "frac_reward_zero_std": 0.3, "grad_norm": 7.000105381011963, "kl": 0.0007340486699831672, "learning_rate": 7.935049019607844e-07, "loss": -0.0002, "num_tokens": 1150237.0, "reward": -0.23125, "reward_std": 0.5087868988513946, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5087869048118592, "sampling/importance_sampling_ratio/max": 1.2860602378845214, "sampling/importance_sampling_ratio/mean": 0.9996052563190461, "sampling/importance_sampling_ratio/min": 0.7324400305747986, "sampling/sampling_logp_difference/max": 0.34504597783088686, "sampling/sampling_logp_difference/mean": 0.021130956523120404, "step": 260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 324.7, "completions/max_terminated_length": 312.1, "completions/mean_length": 232.3, "completions/mean_terminated_length": 225.69357299804688, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.0161486141383649, "epoch": 0.024823021053599337, "frac_reward_zero_std": 0.5, "grad_norm": 7.412708759307861, "kl": 0.0007453218102455139, "learning_rate": 8.241421568627452e-07, "loss": -0.0002, "num_tokens": 1197373.0, "reward": -0.025, "reward_std": 0.3877462804317474, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.3877462834119797, "sampling/importance_sampling_ratio/max": 1.3834476470947266, "sampling/importance_sampling_ratio/mean": 1.0002881050109864, "sampling/importance_sampling_ratio/min": 0.7464916050434113, "sampling/sampling_logp_difference/max": 0.40784192085266113, "sampling/sampling_logp_difference/mean": 0.02069284338504076, "step": 270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.3, "completions/max_terminated_length": 337.3, "completions/mean_length": 230.05, "completions/mean_terminated_length": 230.05, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.139074406027794, "epoch": 0.025742392203732648, "frac_reward_zero_std": 0.3, "grad_norm": 4.662224769592285, "kl": 0.0007581617432151688, "learning_rate": 8.54779411764706e-07, "loss": 0.0001, "num_tokens": 1239009.0, "reward": -0.23125, "reward_std": 0.5634288668632508, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.5634288728237152, "sampling/importance_sampling_ratio/max": 1.265760028362274, "sampling/importance_sampling_ratio/mean": 1.0004870474338532, "sampling/importance_sampling_ratio/min": 0.7602361142635345, "sampling/sampling_logp_difference/max": 0.29833831787109377, "sampling/sampling_logp_difference/mean": 0.02258503921329975, "step": 280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 211.4, "completions/mean_terminated_length": 211.4, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.0229365639388561, "epoch": 0.026661763353865955, "frac_reward_zero_std": 0.6, "grad_norm": 0.02211562730371952, "kl": 0.0006926543850568123, "learning_rate": 8.854166666666668e-07, "loss": 0.0002, "num_tokens": 1282361.0, "reward": -0.3, "reward_std": 0.22853160798549652, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.228531613945961, "sampling/importance_sampling_ratio/max": 1.298721969127655, "sampling/importance_sampling_ratio/mean": 0.9994909107685089, "sampling/importance_sampling_ratio/min": 0.7749456644058228, "sampling/sampling_logp_difference/max": 0.2961899220943451, "sampling/sampling_logp_difference/mean": 0.020687919482588767, "step": 290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.4, "completions/max_terminated_length": 286.4, "completions/mean_length": 204.0875, "completions/mean_terminated_length": 204.0875, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0738814190030097, "epoch": 0.027581134503999266, "frac_reward_zero_std": 0.3, "grad_norm": 6.201390266418457, "kl": 0.0007623104840604356, "learning_rate": 9.160539215686274e-07, "loss": -0.0002, "num_tokens": 1325656.0, "reward": -0.025, "reward_std": 0.5622500061988831, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5622500300407409, "sampling/importance_sampling_ratio/max": 1.3240941166877747, "sampling/importance_sampling_ratio/mean": 1.000314873456955, "sampling/importance_sampling_ratio/min": 0.7543622732162476, "sampling/sampling_logp_difference/max": 0.32526837587356566, "sampling/sampling_logp_difference/mean": 0.02190984096378088, "step": 300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.2, "completions/max_terminated_length": 294.2, "completions/mean_length": 205.9, "completions/mean_terminated_length": 205.9, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.020055215805769, "epoch": 0.028500505654132573, "frac_reward_zero_std": 0.2, "grad_norm": 0.03616155683994293, "kl": 0.0007322640267375391, "learning_rate": 9.466911764705883e-07, "loss": -0.0, "num_tokens": 1370376.0, "reward": -0.0625, "reward_std": 0.6827786028385162, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.6827786266803741, "sampling/importance_sampling_ratio/max": 1.2773687243461609, "sampling/importance_sampling_ratio/mean": 1.0002299845218658, "sampling/importance_sampling_ratio/min": 0.7852615237236023, "sampling/sampling_logp_difference/max": 0.2620736837387085, "sampling/sampling_logp_difference/mean": 0.020878583006560803, "step": 310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 198.475, "completions/mean_terminated_length": 198.475, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.0327361039817333, "epoch": 0.02941987680426588, "frac_reward_zero_std": 0.4, "grad_norm": 7.836579322814941, "kl": 0.0008560536036384292, "learning_rate": 9.77328431372549e-07, "loss": 0.0002, "num_tokens": 1413670.0, "reward": -0.075, "reward_std": 0.46159339845180514, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.46159341633319856, "sampling/importance_sampling_ratio/max": 1.2624885797500611, "sampling/importance_sampling_ratio/mean": 1.0000306367874146, "sampling/importance_sampling_ratio/min": 0.7686843037605285, "sampling/sampling_logp_difference/max": 0.2826682835817337, "sampling/sampling_logp_difference/mean": 0.020757396519184113, "step": 320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 367.2, "completions/max_terminated_length": 367.2, "completions/mean_length": 237.575, "completions/mean_terminated_length": 237.575, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.0333620056509971, "epoch": 0.03033924795439919, "frac_reward_zero_std": 0.5, "grad_norm": 5.881727695465088, "kl": 0.0007218352933705319, "learning_rate": 1.00796568627451e-06, "loss": 0.0001, "num_tokens": 1457732.0, "reward": 0.10625, "reward_std": 0.471607381105423, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.4716073930263519, "sampling/importance_sampling_ratio/max": 1.2482429027557373, "sampling/importance_sampling_ratio/mean": 1.0001922309398652, "sampling/importance_sampling_ratio/min": 0.7963117718696594, "sampling/sampling_logp_difference/max": 0.2448263645172119, "sampling/sampling_logp_difference/mean": 0.020515665877610445, "step": 330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 214.2875, "completions/mean_terminated_length": 214.2875, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0697659142315388, "epoch": 0.0312586191045325, "frac_reward_zero_std": 0.2, "grad_norm": 4.815767288208008, "kl": 0.0007912345174190705, "learning_rate": 1.0386029411764707e-06, "loss": 0.0003, "num_tokens": 1499787.0, "reward": -0.25, "reward_std": 0.5530398905277252, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5530398905277252, "sampling/importance_sampling_ratio/max": 1.2939372897148131, "sampling/importance_sampling_ratio/mean": 1.000212734937668, "sampling/importance_sampling_ratio/min": 0.7773324012756347, "sampling/sampling_logp_difference/max": 0.2824771046638489, "sampling/sampling_logp_difference/mean": 0.021793242543935776, "step": 340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.4, "completions/max_terminated_length": 270.4, "completions/mean_length": 191.7125, "completions/mean_terminated_length": 191.7125, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0357216469943524, "epoch": 0.032177990254665806, "frac_reward_zero_std": 0.7, "grad_norm": 7.155231475830078, "kl": 0.0008339654523297213, "learning_rate": 1.0692401960784314e-06, "loss": -0.0003, "num_tokens": 1537724.0, "reward": -0.2375, "reward_std": 0.23545143008232117, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.23545143604278565, "sampling/importance_sampling_ratio/max": 1.315459668636322, "sampling/importance_sampling_ratio/mean": 0.9998289704322815, "sampling/importance_sampling_ratio/min": 0.7348903477191925, "sampling/sampling_logp_difference/max": 0.32952979803085325, "sampling/sampling_logp_difference/mean": 0.021845722571015358, "step": 350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 217.0, "completions/mean_terminated_length": 217.0, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0911095909774304, "epoch": 0.03309736140479912, "frac_reward_zero_std": 0.1, "grad_norm": 5.255194664001465, "kl": 0.0007996212814759928, "learning_rate": 1.0998774509803921e-06, "loss": 0.0004, "num_tokens": 1585516.0, "reward": -0.64375, "reward_std": 0.6160528659820557, "rewards/reward_fn/mean": -0.64375, "rewards/reward_fn/std": 0.6160528898239136, "sampling/importance_sampling_ratio/max": 1.2698543906211852, "sampling/importance_sampling_ratio/mean": 1.0003864645957947, "sampling/importance_sampling_ratio/min": 0.7258599668741226, "sampling/sampling_logp_difference/max": 0.3434599757194519, "sampling/sampling_logp_difference/mean": 0.021987689286470415, "step": 360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.9, "completions/max_terminated_length": 299.9, "completions/mean_length": 215.075, "completions/mean_terminated_length": 215.075, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.0992125622928142, "epoch": 0.03401673255493243, "frac_reward_zero_std": 0.3, "grad_norm": 0.03416232019662857, "kl": 0.0008051389224419836, "learning_rate": 1.130514705882353e-06, "loss": 0.0004, "num_tokens": 1632426.0, "reward": -0.03125, "reward_std": 0.6317772924900055, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.6317772984504699, "sampling/importance_sampling_ratio/max": 1.27686687707901, "sampling/importance_sampling_ratio/mean": 0.999781858921051, "sampling/importance_sampling_ratio/min": 0.7379679620265961, "sampling/sampling_logp_difference/max": 0.3174197942018509, "sampling/sampling_logp_difference/mean": 0.0217969948425889, "step": 370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.2, "completions/max_terminated_length": 316.2, "completions/mean_length": 219.8375, "completions/mean_terminated_length": 219.8375, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0095836102962494, "epoch": 0.03493610370506574, "frac_reward_zero_std": 0.4, "grad_norm": 0.023060109466314316, "kl": 0.0008119727808661992, "learning_rate": 1.1611519607843138e-06, "loss": 0.0002, "num_tokens": 1675877.0, "reward": 0.075, "reward_std": 0.4590093195438385, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.4590093374252319, "sampling/importance_sampling_ratio/max": 1.4190353631973267, "sampling/importance_sampling_ratio/mean": 1.0000083804130555, "sampling/importance_sampling_ratio/min": 0.7434727549552917, "sampling/sampling_logp_difference/max": 0.36657058000564574, "sampling/sampling_logp_difference/mean": 0.01990796737372875, "step": 380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.7, "completions/max_terminated_length": 278.2, "completions/mean_length": 197.775, "completions/mean_terminated_length": 193.8000015258789, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 0.9564197130501271, "epoch": 0.03585547485519904, "frac_reward_zero_std": 0.5, "grad_norm": 0.02555416151881218, "kl": 0.0007471840333892032, "learning_rate": 1.1917892156862747e-06, "loss": -0.0001, "num_tokens": 1718051.0, "reward": 0.45625, "reward_std": 0.4634471356868744, "rewards/reward_fn/mean": 0.45625, "rewards/reward_fn/std": 0.46344714760780337, "sampling/importance_sampling_ratio/max": 1.3034681320190429, "sampling/importance_sampling_ratio/mean": 0.9996387720108032, "sampling/importance_sampling_ratio/min": 0.7818877398967743, "sampling/sampling_logp_difference/max": 0.305374014377594, "sampling/sampling_logp_difference/mean": 0.020192763954401016, "step": 390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.7, "completions/max_terminated_length": 331.7, "completions/mean_length": 221.2125, "completions/mean_terminated_length": 221.2125, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0457587964832782, "epoch": 0.03677484600533235, "frac_reward_zero_std": 0.6, "grad_norm": 0.032702621072530746, "kl": 0.000827704785115202, "learning_rate": 1.2224264705882354e-06, "loss": 0.0004, "num_tokens": 1764172.0, "reward": -0.1, "reward_std": 0.3355178117752075, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.33551783561706544, "sampling/importance_sampling_ratio/max": 1.2564478993415833, "sampling/importance_sampling_ratio/mean": 1.0000296652317047, "sampling/importance_sampling_ratio/min": 0.7436484932899475, "sampling/sampling_logp_difference/max": 0.31153822541236875, "sampling/sampling_logp_difference/mean": 0.020990101993083952, "step": 400 }, { "epoch": 0.03677484600533235, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 306.29842931937173, "eval_completions/max_terminated_length": 303.9197207678883, "eval_completions/mean_length": 211.96487783595114, "eval_completions/mean_terminated_length": 211.27420446053017, "eval_completions/min_length": 138.0, "eval_completions/min_terminated_length": 138.0, "eval_entropy": 1.082903455273227, "eval_frac_reward_zero_std": 0.3525305410122164, "eval_kl": 0.0007741701553322141, "eval_loss": -1.1047100088035222e-05, "eval_num_tokens": 1764172.0, "eval_reward": -0.09980366492146597, "eval_reward_std": 0.5139228775967271, "eval_rewards/reward_fn/mean": -0.09980366492146597, "eval_rewards/reward_fn/std": 0.5139228917437282, "eval_runtime": 1781.0892, "eval_samples_per_second": 0.322, "eval_sampling/importance_sampling_ratio/max": 1.275079117395491, "eval_sampling/importance_sampling_ratio/mean": 1.0000452339961268, "eval_sampling/importance_sampling_ratio/min": 0.7690545400347801, "eval_sampling/sampling_logp_difference/max": 0.2914872598845713, "eval_sampling/sampling_logp_difference/mean": 0.021694860621038533, "eval_steps_per_second": 0.04, "step": 400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.9, "completions/max_terminated_length": 270.9, "completions/mean_length": 191.9, "completions/mean_terminated_length": 191.9, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 0.9748659297823906, "epoch": 0.03769421715546566, "frac_reward_zero_std": 0.5, "grad_norm": 0.02381434105336666, "kl": 0.0007814693031832576, "learning_rate": 1.2530637254901961e-06, "loss": -0.0, "num_tokens": 1804452.0, "reward": 0.275, "reward_std": 0.43009520769119264, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.4300952196121216, "sampling/importance_sampling_ratio/max": 1.3290038228034973, "sampling/importance_sampling_ratio/mean": 1.0001201629638672, "sampling/importance_sampling_ratio/min": 0.7543022692203522, "sampling/sampling_logp_difference/max": 0.3361420124769211, "sampling/sampling_logp_difference/mean": 0.02122024167329073, "step": 410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 187.875, "completions/mean_terminated_length": 187.875, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0954497002065182, "epoch": 0.03861358830559897, "frac_reward_zero_std": 0.5, "grad_norm": 6.232787609100342, "kl": 0.0008645617057482013, "learning_rate": 1.2837009803921568e-06, "loss": 0.0002, "num_tokens": 1845370.0, "reward": -0.31875, "reward_std": 0.4481493324041367, "rewards/reward_fn/mean": -0.31875, "rewards/reward_fn/std": 0.44814935624599456, "sampling/importance_sampling_ratio/max": 1.2872082710266113, "sampling/importance_sampling_ratio/mean": 1.0000244319438933, "sampling/importance_sampling_ratio/min": 0.6757602855563164, "sampling/sampling_logp_difference/max": 0.566975736618042, "sampling/sampling_logp_difference/mean": 0.022479208186268806, "step": 420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.3, "completions/max_terminated_length": 259.3, "completions/mean_length": 195.05, "completions/mean_terminated_length": 195.05, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.0407501764595508, "epoch": 0.03953295945573228, "frac_reward_zero_std": 0.3, "grad_norm": 0.02399292029440403, "kl": 0.0007973959025548538, "learning_rate": 1.3143382352941177e-06, "loss": -0.0002, "num_tokens": 1884966.0, "reward": 0.0625, "reward_std": 0.6941522836685181, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.694152319431305, "sampling/importance_sampling_ratio/max": 1.2793169736862182, "sampling/importance_sampling_ratio/mean": 1.0003010928630829, "sampling/importance_sampling_ratio/min": 0.7122344195842742, "sampling/sampling_logp_difference/max": 0.3782411336898804, "sampling/sampling_logp_difference/mean": 0.02117371093481779, "step": 430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.2, "completions/max_terminated_length": 257.2, "completions/mean_length": 189.025, "completions/mean_terminated_length": 189.025, "completions/min_length": 122.2, "completions/min_terminated_length": 122.2, "entropy": 0.9857624679803848, "epoch": 0.04045233060586559, "frac_reward_zero_std": 0.6, "grad_norm": 8.941465377807617, "kl": 0.0007763271081785206, "learning_rate": 1.3449754901960785e-06, "loss": 0.0001, "num_tokens": 1921568.0, "reward": 0.34375, "reward_std": 0.3700077474117279, "rewards/reward_fn/mean": 0.34375, "rewards/reward_fn/std": 0.3700077533721924, "sampling/importance_sampling_ratio/max": 1.279254174232483, "sampling/importance_sampling_ratio/mean": 0.9998914003372192, "sampling/importance_sampling_ratio/min": 0.7792259633541108, "sampling/sampling_logp_difference/max": 0.28018001317977903, "sampling/sampling_logp_difference/mean": 0.020629195868968962, "step": 440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 313.2, "completions/max_terminated_length": 309.1, "completions/mean_length": 216.2875, "completions/mean_terminated_length": 208.6375, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 0.9929107904434205, "epoch": 0.0413717017559989, "frac_reward_zero_std": 0.3, "grad_norm": 8.332141876220703, "kl": 0.0007418615419737762, "learning_rate": 1.3756127450980394e-06, "loss": 0.0004, "num_tokens": 1966583.0, "reward": -0.39375, "reward_std": 0.6334246933460236, "rewards/reward_fn/mean": -0.39375, "rewards/reward_fn/std": 0.633424723148346, "sampling/importance_sampling_ratio/max": 1.3046724438667296, "sampling/importance_sampling_ratio/mean": 0.9997773408889771, "sampling/importance_sampling_ratio/min": 0.7649337887763977, "sampling/sampling_logp_difference/max": 0.3193891406059265, "sampling/sampling_logp_difference/mean": 0.020112727768719196, "step": 450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.2, "completions/max_terminated_length": 275.2, "completions/mean_length": 187.3375, "completions/mean_terminated_length": 187.3375, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.027133221924305, "epoch": 0.0422910729061322, "frac_reward_zero_std": 0.2, "grad_norm": 7.107820987701416, "kl": 0.0008454843980871373, "learning_rate": 1.40625e-06, "loss": 0.0005, "num_tokens": 2006882.0, "reward": -0.45625, "reward_std": 0.5439153432846069, "rewards/reward_fn/mean": -0.45625, "rewards/reward_fn/std": 0.5439153611660004, "sampling/importance_sampling_ratio/max": 1.3032878637313843, "sampling/importance_sampling_ratio/mean": 1.0005386471748352, "sampling/importance_sampling_ratio/min": 0.7848802864551544, "sampling/sampling_logp_difference/max": 0.29207401871681216, "sampling/sampling_logp_difference/mean": 0.021029272116720676, "step": 460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.1, "completions/max_terminated_length": 305.3, "completions/mean_length": 231.05, "completions/mean_terminated_length": 229.3732147216797, "completions/min_length": 158.4, "completions/min_terminated_length": 158.4, "entropy": 1.1479732871055603, "epoch": 0.043210444056265514, "frac_reward_zero_std": 0.2, "grad_norm": 5.885332107543945, "kl": 0.0008643940520414617, "learning_rate": 1.4368872549019608e-06, "loss": 0.0005, "num_tokens": 2053974.0, "reward": -0.26875, "reward_std": 0.6387705266475677, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.6387705385684967, "sampling/importance_sampling_ratio/max": 1.266026759147644, "sampling/importance_sampling_ratio/mean": 0.9996416211128235, "sampling/importance_sampling_ratio/min": 0.7661808729171753, "sampling/sampling_logp_difference/max": 0.2700227200984955, "sampling/sampling_logp_difference/mean": 0.02255709283053875, "step": 470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.8, "completions/max_terminated_length": 340.8, "completions/mean_length": 233.8, "completions/mean_terminated_length": 233.8, "completions/min_length": 151.9, "completions/min_terminated_length": 151.9, "entropy": 1.0974321976304053, "epoch": 0.044129815206398824, "frac_reward_zero_std": 0.3, "grad_norm": 4.995086193084717, "kl": 0.0011237188922677888, "learning_rate": 1.4675245098039217e-06, "loss": 0.0003, "num_tokens": 2101638.0, "reward": -0.0875, "reward_std": 0.5576324224472046, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.557632428407669, "sampling/importance_sampling_ratio/max": 1.265100634098053, "sampling/importance_sampling_ratio/mean": 0.9995315313339234, "sampling/importance_sampling_ratio/min": 0.731876790523529, "sampling/sampling_logp_difference/max": 0.34230992794036863, "sampling/sampling_logp_difference/mean": 0.022754467092454433, "step": 480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 332.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 227.625, "completions/mean_terminated_length": 223.8107147216797, "completions/min_length": 151.4, "completions/min_terminated_length": 151.4, "entropy": 1.1766859948635102, "epoch": 0.045049186356532135, "frac_reward_zero_std": 0.0, "grad_norm": 5.976254940032959, "kl": 0.001084200566401705, "learning_rate": 1.4981617647058824e-06, "loss": 0.0003, "num_tokens": 2146832.0, "reward": -0.16875, "reward_std": 0.8188398405909538, "rewards/reward_fn/mean": -0.16875, "rewards/reward_fn/std": 0.8188398584723473, "sampling/importance_sampling_ratio/max": 1.2931214451789856, "sampling/importance_sampling_ratio/mean": 1.0000166296958923, "sampling/importance_sampling_ratio/min": 0.7744659781455994, "sampling/sampling_logp_difference/max": 0.27478190660476687, "sampling/sampling_logp_difference/mean": 0.02285472545772791, "step": 490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.1, "completions/max_terminated_length": 310.1, "completions/mean_length": 225.25, "completions/mean_terminated_length": 225.25, "completions/min_length": 148.2, "completions/min_terminated_length": 148.2, "entropy": 1.027709399908781, "epoch": 0.04596855750666544, "frac_reward_zero_std": 0.3, "grad_norm": 5.856915473937988, "kl": 0.0011877536860993132, "learning_rate": 1.5287990196078434e-06, "loss": 0.0004, "num_tokens": 2192860.0, "reward": 0.1875, "reward_std": 0.6033036708831787, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.6033036708831787, "sampling/importance_sampling_ratio/max": 1.2909950613975525, "sampling/importance_sampling_ratio/mean": 0.9999686121940613, "sampling/importance_sampling_ratio/min": 0.7839114427566528, "sampling/sampling_logp_difference/max": 0.2868601202964783, "sampling/sampling_logp_difference/mean": 0.02084048669785261, "step": 500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 207.175, "completions/mean_terminated_length": 207.175, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0776195771992207, "epoch": 0.04688792865679875, "frac_reward_zero_std": 0.5, "grad_norm": 5.848415374755859, "kl": 0.0015265618094417732, "learning_rate": 1.559436274509804e-06, "loss": -0.0001, "num_tokens": 2235474.0, "reward": -0.2, "reward_std": 0.33271285593509675, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.33271287977695463, "sampling/importance_sampling_ratio/max": 1.3018338799476623, "sampling/importance_sampling_ratio/mean": 1.0002345502376557, "sampling/importance_sampling_ratio/min": 0.7495401531457901, "sampling/sampling_logp_difference/max": 0.32740806937217715, "sampling/sampling_logp_difference/mean": 0.02211202085018158, "step": 510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.4, "completions/max_terminated_length": 327.4, "completions/mean_length": 199.1625, "completions/mean_terminated_length": 199.1625, "completions/min_length": 118.2, "completions/min_terminated_length": 118.2, "entropy": 1.0149781927466393, "epoch": 0.04780729980693206, "frac_reward_zero_std": 0.1, "grad_norm": 5.296606063842773, "kl": 0.0015358019700215664, "learning_rate": 1.5900735294117648e-06, "loss": -0.0, "num_tokens": 2276727.0, "reward": -0.43125, "reward_std": 0.6722564369440078, "rewards/reward_fn/mean": -0.43125, "rewards/reward_fn/std": 0.6722564429044724, "sampling/importance_sampling_ratio/max": 1.3543675780296325, "sampling/importance_sampling_ratio/mean": 1.0003010928630829, "sampling/importance_sampling_ratio/min": 0.7615984201431274, "sampling/sampling_logp_difference/max": 0.3310415744781494, "sampling/sampling_logp_difference/mean": 0.021809526719152927, "step": 520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.6, "completions/max_terminated_length": 308.6, "completions/mean_length": 214.6875, "completions/mean_terminated_length": 214.6875, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.057837551087141, "epoch": 0.048726670957065364, "frac_reward_zero_std": 0.1, "grad_norm": 5.374033451080322, "kl": 0.0015048259701870847, "learning_rate": 1.6207107843137257e-06, "loss": 0.0001, "num_tokens": 2318302.0, "reward": -0.025, "reward_std": 0.7126178503036499, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.7126178562641143, "sampling/importance_sampling_ratio/max": 1.238474726676941, "sampling/importance_sampling_ratio/mean": 1.0000241696834564, "sampling/importance_sampling_ratio/min": 0.7692290544509888, "sampling/sampling_logp_difference/max": 0.2738600611686707, "sampling/sampling_logp_difference/mean": 0.021399349719285966, "step": 530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 212.95, "completions/mean_terminated_length": 212.95, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0474265605211257, "epoch": 0.049646042107198675, "frac_reward_zero_std": 0.3, "grad_norm": 5.015708923339844, "kl": 0.0019162637305271345, "learning_rate": 1.6513480392156864e-06, "loss": 0.0, "num_tokens": 2361114.0, "reward": -0.21875, "reward_std": 0.6456482112407684, "rewards/reward_fn/mean": -0.21875, "rewards/reward_fn/std": 0.6456482529640197, "sampling/importance_sampling_ratio/max": 1.2911459445953368, "sampling/importance_sampling_ratio/mean": 0.9999278366565705, "sampling/importance_sampling_ratio/min": 0.777782016992569, "sampling/sampling_logp_difference/max": 0.287069833278656, "sampling/sampling_logp_difference/mean": 0.021811645198613405, "step": 540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.6, "completions/max_terminated_length": 300.6, "completions/mean_length": 207.7, "completions/mean_terminated_length": 207.7, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 0.9286081437021494, "epoch": 0.050565413257331986, "frac_reward_zero_std": 0.2, "grad_norm": 5.826114654541016, "kl": 0.001771631084557157, "learning_rate": 1.6819852941176473e-06, "loss": 0.0005, "num_tokens": 2401458.0, "reward": -0.1375, "reward_std": 0.6216493517160415, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.6216493636369705, "sampling/importance_sampling_ratio/max": 1.300283133983612, "sampling/importance_sampling_ratio/mean": 1.0003273010253906, "sampling/importance_sampling_ratio/min": 0.7246978104114532, "sampling/sampling_logp_difference/max": 0.356741726398468, "sampling/sampling_logp_difference/mean": 0.020618940703570843, "step": 550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.1, "completions/max_terminated_length": 276.1, "completions/mean_length": 186.6, "completions/mean_terminated_length": 186.6, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0070199850946664, "epoch": 0.051484784407465296, "frac_reward_zero_std": 0.2, "grad_norm": 6.0504374504089355, "kl": 0.0017657156095083337, "learning_rate": 1.712622549019608e-06, "loss": -0.0005, "num_tokens": 2443154.0, "reward": -0.2625, "reward_std": 0.6940361440181733, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.6940361678600311, "sampling/importance_sampling_ratio/max": 1.2626020669937135, "sampling/importance_sampling_ratio/mean": 0.9996123135089874, "sampling/importance_sampling_ratio/min": 0.7820769190788269, "sampling/sampling_logp_difference/max": 0.26067387461662295, "sampling/sampling_logp_difference/mean": 0.021085712127387524, "step": 560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 353.7, "completions/max_terminated_length": 353.7, "completions/mean_length": 218.15, "completions/mean_terminated_length": 218.15, "completions/min_length": 114.2, "completions/min_terminated_length": 114.2, "entropy": 1.0417570002377032, "epoch": 0.0524041555575986, "frac_reward_zero_std": 0.2, "grad_norm": 5.932330131530762, "kl": 0.002104779648652766, "learning_rate": 1.7432598039215688e-06, "loss": 0.0008, "num_tokens": 2489278.0, "reward": -0.30625, "reward_std": 0.5829227596521378, "rewards/reward_fn/mean": -0.30625, "rewards/reward_fn/std": 0.5829227954149246, "sampling/importance_sampling_ratio/max": 1.2435389757156372, "sampling/importance_sampling_ratio/mean": 1.0000598549842834, "sampling/importance_sampling_ratio/min": 0.7541009902954101, "sampling/sampling_logp_difference/max": 0.29438533782958987, "sampling/sampling_logp_difference/mean": 0.0215120536275208, "step": 570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 339.0, "completions/max_terminated_length": 326.0, "completions/mean_length": 228.5625, "completions/mean_terminated_length": 217.375, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0102458983659743, "epoch": 0.05332352670773191, "frac_reward_zero_std": 0.1, "grad_norm": 4.449970722198486, "kl": 0.0017665868123003746, "learning_rate": 1.7738970588235297e-06, "loss": 0.0003, "num_tokens": 2534923.0, "reward": -0.21875, "reward_std": 0.8699533045291901, "rewards/reward_fn/mean": -0.21875, "rewards/reward_fn/std": 0.8699533402919769, "sampling/importance_sampling_ratio/max": 1.253454339504242, "sampling/importance_sampling_ratio/mean": 1.0003077864646912, "sampling/importance_sampling_ratio/min": 0.7230043232440948, "sampling/sampling_logp_difference/max": 0.34247471690177916, "sampling/sampling_logp_difference/mean": 0.020967343542724847, "step": 580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 190.15, "completions/mean_terminated_length": 190.15, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 1.0787615150213241, "epoch": 0.05424289785786522, "frac_reward_zero_std": 0.5, "grad_norm": 0.062219273298978806, "kl": 0.0014621297348639927, "learning_rate": 1.8045343137254904e-06, "loss": 0.0003, "num_tokens": 2575151.0, "reward": -0.175, "reward_std": 0.2748226135969162, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.2748226195573807, "sampling/importance_sampling_ratio/max": 1.2410045862197876, "sampling/importance_sampling_ratio/mean": 1.000088530778885, "sampling/importance_sampling_ratio/min": 0.7797856152057647, "sampling/sampling_logp_difference/max": 0.25548909306526185, "sampling/sampling_logp_difference/mean": 0.022826541773974897, "step": 590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 217.8125, "completions/mean_terminated_length": 217.8125, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.0992080330848695, "epoch": 0.05516226900799853, "frac_reward_zero_std": 0.4, "grad_norm": 5.946908950805664, "kl": 0.0015387784260383341, "learning_rate": 1.8351715686274509e-06, "loss": 0.0002, "num_tokens": 2622464.0, "reward": -0.1125, "reward_std": 0.49239068627357485, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.4923907041549683, "sampling/importance_sampling_ratio/max": 1.3550460577011108, "sampling/importance_sampling_ratio/mean": 1.0002450406551362, "sampling/importance_sampling_ratio/min": 0.7415886342525482, "sampling/sampling_logp_difference/max": 0.3970524609088898, "sampling/sampling_logp_difference/mean": 0.022138751484453677, "step": 600 }, { "epoch": 0.05516226900799853, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 291.7870855148342, "eval_completions/max_terminated_length": 290.39092495637, "eval_completions/mean_length": 203.46684118673647, "eval_completions/mean_terminated_length": 203.13759049820027, "eval_completions/min_length": 133.29493891797557, "eval_completions/min_terminated_length": 133.29493891797557, "eval_entropy": 1.0725050150411914, "eval_frac_reward_zero_std": 0.3717277486910995, "eval_kl": 0.0013345682309464232, "eval_loss": 9.026547922985628e-05, "eval_num_tokens": 2622464.0, "eval_reward": -0.07362565445026178, "eval_reward_std": 0.5034478372564283, "eval_rewards/reward_fn/mean": -0.07362565445026178, "eval_rewards/reward_fn/std": 0.503447853899959, "eval_runtime": 1706.6586, "eval_samples_per_second": 0.336, "eval_sampling/importance_sampling_ratio/max": 1.277582466082215, "eval_sampling/importance_sampling_ratio/mean": 1.0000125297701172, "eval_sampling/importance_sampling_ratio/min": 0.7611577148836944, "eval_sampling/sampling_logp_difference/max": 0.3050420353966026, "eval_sampling/sampling_logp_difference/mean": 0.021830966759531598, "eval_steps_per_second": 0.042, "step": 600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 204.3125, "completions/mean_terminated_length": 204.3125, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 1.059968823939562, "epoch": 0.056081640158131836, "frac_reward_zero_std": 0.4, "grad_norm": 4.471683502197266, "kl": 0.0012696869809587952, "learning_rate": 1.865808823529412e-06, "loss": 0.0, "num_tokens": 2663513.0, "reward": 0.0375, "reward_std": 0.5180439412593841, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.5180439591407776, "sampling/importance_sampling_ratio/max": 1.2942311406135558, "sampling/importance_sampling_ratio/mean": 0.9998626291751862, "sampling/importance_sampling_ratio/min": 0.7758565127849579, "sampling/sampling_logp_difference/max": 0.2907403588294983, "sampling/sampling_logp_difference/mean": 0.021082007698714732, "step": 610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.6, "completions/max_terminated_length": 324.6, "completions/mean_length": 221.0, "completions/mean_terminated_length": 221.0, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 1.1456563584506512, "epoch": 0.05700101130826515, "frac_reward_zero_std": 0.2, "grad_norm": 5.6595869064331055, "kl": 0.0012122080097469734, "learning_rate": 1.8964460784313725e-06, "loss": -0.0004, "num_tokens": 2711577.0, "reward": 0.0125, "reward_std": 0.7250636041164398, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.7250636458396912, "sampling/importance_sampling_ratio/max": 1.2696220874786377, "sampling/importance_sampling_ratio/mean": 1.0004298686981201, "sampling/importance_sampling_ratio/min": 0.7179760634899139, "sampling/sampling_logp_difference/max": 0.351926326751709, "sampling/sampling_logp_difference/mean": 0.022937136143445967, "step": 620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.4, "completions/max_terminated_length": 319.9, "completions/mean_length": 226.2375, "completions/mean_terminated_length": 223.80178833007812, "completions/min_length": 117.0, "completions/min_terminated_length": 117.0, "entropy": 1.134413019567728, "epoch": 0.05792038245839846, "frac_reward_zero_std": 0.3, "grad_norm": 4.442356586456299, "kl": 0.0011642233301245141, "learning_rate": 1.9270833333333334e-06, "loss": 0.0001, "num_tokens": 2756196.0, "reward": 0.10625, "reward_std": 0.4039553329348564, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.40395533889532087, "sampling/importance_sampling_ratio/max": 1.2688979506492615, "sampling/importance_sampling_ratio/mean": 0.9997282147407531, "sampling/importance_sampling_ratio/min": 0.7676404356956482, "sampling/sampling_logp_difference/max": 0.2864227324724197, "sampling/sampling_logp_difference/mean": 0.022699452936649323, "step": 630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 311.0, "completions/max_terminated_length": 307.8, "completions/mean_length": 224.5625, "completions/mean_terminated_length": 222.375, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.1004840791225434, "epoch": 0.05883975360853176, "frac_reward_zero_std": 0.5, "grad_norm": 6.381046772003174, "kl": 0.0013143535448762122, "learning_rate": 1.9577205882352944e-06, "loss": 0.0, "num_tokens": 2804593.0, "reward": 0.00625, "reward_std": 0.4376678436994553, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.43766787350177766, "sampling/importance_sampling_ratio/max": 1.2596550703048706, "sampling/importance_sampling_ratio/mean": 0.9996941030025482, "sampling/importance_sampling_ratio/min": 0.7166162163019181, "sampling/sampling_logp_difference/max": 0.3541571795940399, "sampling/sampling_logp_difference/mean": 0.022227655909955502, "step": 640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.6, "completions/max_terminated_length": 281.6, "completions/mean_length": 200.475, "completions/mean_terminated_length": 200.475, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 0.9952188201248646, "epoch": 0.05975912475866507, "frac_reward_zero_std": 0.4, "grad_norm": 5.908881187438965, "kl": 0.0011727314042218496, "learning_rate": 1.988357843137255e-06, "loss": 0.0003, "num_tokens": 2848967.0, "reward": -0.3375, "reward_std": 0.49928959012031554, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.49928959608078005, "sampling/importance_sampling_ratio/max": 1.3279521346092225, "sampling/importance_sampling_ratio/mean": 1.0001044511795043, "sampling/importance_sampling_ratio/min": 0.7699520707130432, "sampling/sampling_logp_difference/max": 0.3227164030075073, "sampling/sampling_logp_difference/mean": 0.020919199753552675, "step": 650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.7, "completions/max_terminated_length": 266.7, "completions/mean_length": 202.9625, "completions/mean_terminated_length": 202.9625, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0741405218839646, "epoch": 0.06067849590879838, "frac_reward_zero_std": 0.4, "grad_norm": 7.46884822845459, "kl": 0.0013214543490903453, "learning_rate": 2.018995098039216e-06, "loss": 0.0002, "num_tokens": 2894356.0, "reward": -0.0375, "reward_std": 0.46570733189582825, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.4657073378562927, "sampling/importance_sampling_ratio/max": 1.2397753715515136, "sampling/importance_sampling_ratio/mean": 1.0001684188842774, "sampling/importance_sampling_ratio/min": 0.7771768510341645, "sampling/sampling_logp_difference/max": 0.2563755869865417, "sampling/sampling_logp_difference/mean": 0.021589200757443906, "step": 660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.5, "completions/max_terminated_length": 328.5, "completions/mean_length": 220.1625, "completions/mean_terminated_length": 220.1625, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1027231484651565, "epoch": 0.06159786705893169, "frac_reward_zero_std": 0.0, "grad_norm": 5.490810871124268, "kl": 0.0015161238821747246, "learning_rate": 2.0496323529411767e-06, "loss": 0.0008, "num_tokens": 2937417.0, "reward": -0.4, "reward_std": 0.8197159588336944, "rewards/reward_fn/mean": -0.4, "rewards/reward_fn/std": 0.8197160124778747, "sampling/importance_sampling_ratio/max": 1.3400521159172059, "sampling/importance_sampling_ratio/mean": 1.0001618027687074, "sampling/importance_sampling_ratio/min": 0.7698067843914032, "sampling/sampling_logp_difference/max": 0.30671982765197753, "sampling/sampling_logp_difference/mean": 0.02209597863256931, "step": 670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 342.7, "completions/max_terminated_length": 306.3, "completions/mean_length": 228.9625, "completions/mean_terminated_length": 222.5482162475586, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.0638794735074044, "epoch": 0.062517238209065, "frac_reward_zero_std": 0.3, "grad_norm": 5.243893623352051, "kl": 0.0018352417064306792, "learning_rate": 2.0802696078431376e-06, "loss": 0.0001, "num_tokens": 2985102.0, "reward": 0.05625, "reward_std": 0.5753691732883454, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.5753691852092743, "sampling/importance_sampling_ratio/max": 1.3427037000656128, "sampling/importance_sampling_ratio/mean": 0.999526858329773, "sampling/importance_sampling_ratio/min": 0.7472610712051392, "sampling/sampling_logp_difference/max": 0.3482950508594513, "sampling/sampling_logp_difference/mean": 0.022305504232645035, "step": 680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.6, "completions/max_terminated_length": 314.6, "completions/mean_length": 208.5125, "completions/mean_terminated_length": 208.5125, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0907903552055358, "epoch": 0.06343660935919831, "frac_reward_zero_std": 0.2, "grad_norm": 0.1277386099100113, "kl": 0.0019939201170927844, "learning_rate": 2.110906862745098e-06, "loss": -0.0002, "num_tokens": 3030775.0, "reward": 0.0125, "reward_std": 0.6294712632894516, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.6294712632894516, "sampling/importance_sampling_ratio/max": 1.2345026850700378, "sampling/importance_sampling_ratio/mean": 1.0001937687397002, "sampling/importance_sampling_ratio/min": 0.7667145550251007, "sampling/sampling_logp_difference/max": 0.2702470779418945, "sampling/sampling_logp_difference/mean": 0.022137697972357272, "step": 690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.5, "completions/max_terminated_length": 321.5, "completions/mean_length": 226.5, "completions/mean_terminated_length": 226.5, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0019915945827962, "epoch": 0.06435598050933161, "frac_reward_zero_std": 0.1, "grad_norm": 6.810304641723633, "kl": 0.0017923590748978313, "learning_rate": 2.141544117647059e-06, "loss": 0.0005, "num_tokens": 3075815.0, "reward": -0.075, "reward_std": 0.7033166900277138, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.7033166900277138, "sampling/importance_sampling_ratio/max": 1.3033283948898315, "sampling/importance_sampling_ratio/mean": 0.9996226727962494, "sampling/importance_sampling_ratio/min": 0.7687257885932922, "sampling/sampling_logp_difference/max": 0.30762053430080416, "sampling/sampling_logp_difference/mean": 0.020816490799188615, "step": 700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.1, "completions/max_terminated_length": 263.1, "completions/mean_length": 181.625, "completions/mean_terminated_length": 181.625, "completions/min_length": 114.2, "completions/min_terminated_length": 114.2, "entropy": 1.052776575088501, "epoch": 0.06527535165946492, "frac_reward_zero_std": 0.3, "grad_norm": 6.5576653480529785, "kl": 0.0019860038912156595, "learning_rate": 2.17218137254902e-06, "loss": -0.0001, "num_tokens": 3116377.0, "reward": -0.0125, "reward_std": 0.5737014353275299, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.5737014710903168, "sampling/importance_sampling_ratio/max": 1.2989729404449464, "sampling/importance_sampling_ratio/mean": 1.0002965211868287, "sampling/importance_sampling_ratio/min": 0.7976649224758148, "sampling/sampling_logp_difference/max": 0.26987260580062866, "sampling/sampling_logp_difference/mean": 0.02193618509918451, "step": 710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 214.275, "completions/mean_terminated_length": 214.275, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.0446342997252942, "epoch": 0.06619472280959823, "frac_reward_zero_std": 0.4, "grad_norm": 0.06392182409763336, "kl": 0.0020218260469846427, "learning_rate": 2.2028186274509805e-06, "loss": 0.0003, "num_tokens": 3160351.0, "reward": 0.2375, "reward_std": 0.5578714311122894, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.5578714430332183, "sampling/importance_sampling_ratio/max": 1.3183648705482482, "sampling/importance_sampling_ratio/mean": 0.9995230078697205, "sampling/importance_sampling_ratio/min": 0.7686205208301544, "sampling/sampling_logp_difference/max": 0.31160278916358947, "sampling/sampling_logp_difference/mean": 0.02191967945545912, "step": 720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 213.0125, "completions/mean_terminated_length": 213.0125, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.016400897502899, "epoch": 0.06711409395973154, "frac_reward_zero_std": 0.4, "grad_norm": 5.154075622558594, "kl": 0.001864359564933693, "learning_rate": 2.2334558823529414e-06, "loss": 0.0003, "num_tokens": 3205568.0, "reward": -0.475, "reward_std": 0.5671322226524353, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5671322464942932, "sampling/importance_sampling_ratio/max": 1.231548273563385, "sampling/importance_sampling_ratio/mean": 1.0002602279186248, "sampling/importance_sampling_ratio/min": 0.7503688395023346, "sampling/sampling_logp_difference/max": 0.2985227793455124, "sampling/sampling_logp_difference/mean": 0.02064361944794655, "step": 730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 198.6875, "completions/mean_terminated_length": 198.6875, "completions/min_length": 123.5, "completions/min_terminated_length": 123.5, "entropy": 1.044501344114542, "epoch": 0.06803346510986485, "frac_reward_zero_std": 0.5, "grad_norm": 0.10327262431383133, "kl": 0.0023841019843530375, "learning_rate": 2.2640931372549023e-06, "loss": -0.0002, "num_tokens": 3247447.0, "reward": 0.0875, "reward_std": 0.4298454225063324, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.42984544634819033, "sampling/importance_sampling_ratio/max": 1.228196370601654, "sampling/importance_sampling_ratio/mean": 1.000384110212326, "sampling/importance_sampling_ratio/min": 0.7776436030864715, "sampling/sampling_logp_difference/max": 0.2547809600830078, "sampling/sampling_logp_difference/mean": 0.020780765824019908, "step": 740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.6, "completions/max_terminated_length": 284.6, "completions/mean_length": 199.3375, "completions/mean_terminated_length": 199.3375, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.0376607328653336, "epoch": 0.06895283625999817, "frac_reward_zero_std": 0.4, "grad_norm": 0.038123052567243576, "kl": 0.0029976150144648273, "learning_rate": 2.294730392156863e-06, "loss": 0.0002, "num_tokens": 3286538.0, "reward": 0.00625, "reward_std": 0.37166147828102114, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.37166147828102114, "sampling/importance_sampling_ratio/max": 1.2747241973876953, "sampling/importance_sampling_ratio/mean": 0.9998388707637786, "sampling/importance_sampling_ratio/min": 0.7131896018981934, "sampling/sampling_logp_difference/max": 0.3752714037895203, "sampling/sampling_logp_difference/mean": 0.021647820994257928, "step": 750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.4, "completions/max_terminated_length": 308.4, "completions/mean_length": 210.375, "completions/mean_terminated_length": 210.375, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0729804262518883, "epoch": 0.06987220741013148, "frac_reward_zero_std": 0.2, "grad_norm": 4.930842876434326, "kl": 0.002814591173955705, "learning_rate": 2.3253676470588237e-06, "loss": 0.0001, "num_tokens": 3330480.0, "reward": 0.0, "reward_std": 0.5096566110849381, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5096566289663315, "sampling/importance_sampling_ratio/max": 1.2650708317756654, "sampling/importance_sampling_ratio/mean": 0.9998493313789367, "sampling/importance_sampling_ratio/min": 0.7998102128505706, "sampling/sampling_logp_difference/max": 0.2504520356655121, "sampling/sampling_logp_difference/mean": 0.02120122741907835, "step": 760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 212.1125, "completions/mean_terminated_length": 212.1125, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 1.0424277201294898, "epoch": 0.07079157856026477, "frac_reward_zero_std": 0.2, "grad_norm": 4.779900074005127, "kl": 0.0019695476890774443, "learning_rate": 2.3560049019607847e-06, "loss": 0.0003, "num_tokens": 3374473.0, "reward": -0.425, "reward_std": 0.5967825591564179, "rewards/reward_fn/mean": -0.425, "rewards/reward_fn/std": 0.5967825591564179, "sampling/importance_sampling_ratio/max": 1.2580410718917847, "sampling/importance_sampling_ratio/mean": 0.9996964335441589, "sampling/importance_sampling_ratio/min": 0.7536938577890396, "sampling/sampling_logp_difference/max": 0.3311957329511642, "sampling/sampling_logp_difference/mean": 0.021131057664752005, "step": 770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 306.8, "completions/max_terminated_length": 302.1, "completions/mean_length": 214.1, "completions/mean_terminated_length": 211.575, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.0404712975025177, "epoch": 0.07171094971039808, "frac_reward_zero_std": 0.4, "grad_norm": 0.05750023201107979, "kl": 0.003591870867239777, "learning_rate": 2.3866421568627456e-06, "loss": 0.0004, "num_tokens": 3415681.0, "reward": -0.1875, "reward_std": 0.4439386010169983, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.44393863081932067, "sampling/importance_sampling_ratio/max": 1.3732612013816834, "sampling/importance_sampling_ratio/mean": 1.0003301322460174, "sampling/importance_sampling_ratio/min": 0.7330769240856171, "sampling/sampling_logp_difference/max": 0.38779781460762025, "sampling/sampling_logp_difference/mean": 0.021037154272198676, "step": 780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.6, "completions/max_terminated_length": 272.6, "completions/mean_length": 202.5125, "completions/mean_terminated_length": 202.5125, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.05287319496274, "epoch": 0.0726303208605314, "frac_reward_zero_std": 0.4, "grad_norm": 6.309450626373291, "kl": 0.002909520715184044, "learning_rate": 2.417279411764706e-06, "loss": -0.0001, "num_tokens": 3457842.0, "reward": 0.0, "reward_std": 0.5087977588176728, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5087977826595307, "sampling/importance_sampling_ratio/max": 1.2744810581207275, "sampling/importance_sampling_ratio/mean": 0.9999706447124481, "sampling/importance_sampling_ratio/min": 0.7788361012935638, "sampling/sampling_logp_difference/max": 0.2818069696426392, "sampling/sampling_logp_difference/mean": 0.021466401033103467, "step": 790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 200.7625, "completions/mean_terminated_length": 200.7625, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.089126033335924, "epoch": 0.0735496920106647, "frac_reward_zero_std": 0.5, "grad_norm": 7.358144760131836, "kl": 0.0025294901599409057, "learning_rate": 2.4479166666666666e-06, "loss": 0.0, "num_tokens": 3500271.0, "reward": -0.6375, "reward_std": 0.39377824664115907, "rewards/reward_fn/mean": -0.6375, "rewards/reward_fn/std": 0.39377824664115907, "sampling/importance_sampling_ratio/max": 1.2900829076766969, "sampling/importance_sampling_ratio/mean": 0.9997484087944031, "sampling/importance_sampling_ratio/min": 0.7701646745204925, "sampling/sampling_logp_difference/max": 0.29433541297912597, "sampling/sampling_logp_difference/mean": 0.02190215401351452, "step": 800 }, { "epoch": 0.0735496920106647, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 293.5549738219895, "eval_completions/max_terminated_length": 291.27748691099475, "eval_completions/mean_length": 204.33944153577661, "eval_completions/mean_terminated_length": 203.83570209389998, "eval_completions/min_length": 132.88830715532285, "eval_completions/min_terminated_length": 132.88830715532285, "eval_entropy": 1.0747235617088398, "eval_frac_reward_zero_std": 0.33507853403141363, "eval_kl": 0.003043472509462237, "eval_loss": 0.00012255448382347822, "eval_num_tokens": 3500271.0, "eval_reward": -0.06489965095986039, "eval_reward_std": 0.5076892879152797, "eval_rewards/reward_fn/mean": -0.06489965095986039, "eval_rewards/reward_fn/std": 0.5076893038306561, "eval_runtime": 1714.5099, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2751857127610717, "eval_sampling/importance_sampling_ratio/mean": 1.0000065688895514, "eval_sampling/importance_sampling_ratio/min": 0.7604119822148877, "eval_sampling/sampling_logp_difference/max": 0.30892977253720816, "eval_sampling/sampling_logp_difference/mean": 0.02196653231971915, "eval_steps_per_second": 0.042, "step": 800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.4, "completions/max_terminated_length": 326.4, "completions/mean_length": 210.1, "completions/mean_terminated_length": 210.1, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0941917210817338, "epoch": 0.07446906316079802, "frac_reward_zero_std": 0.3, "grad_norm": 0.0468394011259079, "kl": 0.003635933337500319, "learning_rate": 2.478553921568628e-06, "loss": 0.0004, "num_tokens": 3547407.0, "reward": -0.025, "reward_std": 0.4316474676132202, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.43164747953414917, "sampling/importance_sampling_ratio/max": 1.2295541405677795, "sampling/importance_sampling_ratio/mean": 0.9998320877552033, "sampling/importance_sampling_ratio/min": 0.7933319866657257, "sampling/sampling_logp_difference/max": 0.23681988716125488, "sampling/sampling_logp_difference/mean": 0.022012897953391074, "step": 810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.7, "completions/max_terminated_length": 308.7, "completions/mean_length": 210.875, "completions/mean_terminated_length": 210.875, "completions/min_length": 122.1, "completions/min_terminated_length": 122.1, "entropy": 1.0059361286461352, "epoch": 0.07538843431093133, "frac_reward_zero_std": 0.3, "grad_norm": 0.04860125854611397, "kl": 0.002773964193329448, "learning_rate": 2.5091911764705884e-06, "loss": 0.0003, "num_tokens": 3591517.0, "reward": -0.01875, "reward_std": 0.566309529542923, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.5663095355033875, "sampling/importance_sampling_ratio/max": 1.2595862746238708, "sampling/importance_sampling_ratio/mean": 1.0000875651836396, "sampling/importance_sampling_ratio/min": 0.7891337037086487, "sampling/sampling_logp_difference/max": 0.25611660480499265, "sampling/sampling_logp_difference/mean": 0.020061161369085312, "step": 820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 307.9, "completions/max_terminated_length": 289.1, "completions/mean_length": 207.45, "completions/mean_terminated_length": 203.99285888671875, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.060875914245844, "epoch": 0.07630780546106464, "frac_reward_zero_std": 0.1, "grad_norm": 0.04882955551147461, "kl": 0.004919398497440853, "learning_rate": 2.539828431372549e-06, "loss": 0.0004, "num_tokens": 3633353.0, "reward": -0.25, "reward_std": 0.5631370812654495, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.5631370931863785, "sampling/importance_sampling_ratio/max": 1.2809331178665162, "sampling/importance_sampling_ratio/mean": 0.9995364725589753, "sampling/importance_sampling_ratio/min": 0.7481603860855103, "sampling/sampling_logp_difference/max": 0.3214196145534515, "sampling/sampling_logp_difference/mean": 0.02176289279013872, "step": 830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.3, "completions/max_terminated_length": 292.3, "completions/mean_length": 194.4375, "completions/mean_terminated_length": 194.4375, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.0361232250928878, "epoch": 0.07722717661119793, "frac_reward_zero_std": 0.4, "grad_norm": 7.146517276763916, "kl": 0.0029901799985964316, "learning_rate": 2.57046568627451e-06, "loss": 0.0003, "num_tokens": 3674556.0, "reward": -0.5125, "reward_std": 0.4509626179933548, "rewards/reward_fn/mean": -0.5125, "rewards/reward_fn/std": 0.45096264183521273, "sampling/importance_sampling_ratio/max": 1.2726532220840454, "sampling/importance_sampling_ratio/mean": 0.9992108643054962, "sampling/importance_sampling_ratio/min": 0.7762696444988251, "sampling/sampling_logp_difference/max": 0.2775296330451965, "sampling/sampling_logp_difference/mean": 0.02096966076642275, "step": 840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.3, "completions/max_terminated_length": 298.3, "completions/mean_length": 201.3, "completions/mean_terminated_length": 201.3, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.1129839710891247, "epoch": 0.07814654776133124, "frac_reward_zero_std": 0.3, "grad_norm": 6.1760735511779785, "kl": 0.003231057215452893, "learning_rate": 2.601102941176471e-06, "loss": 0.0002, "num_tokens": 3719852.0, "reward": 0.0375, "reward_std": 0.5990100085735321, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.5990100562572479, "sampling/importance_sampling_ratio/max": 1.317863154411316, "sampling/importance_sampling_ratio/mean": 0.9996878027915954, "sampling/importance_sampling_ratio/min": 0.7930496752262115, "sampling/sampling_logp_difference/max": 0.2882010996341705, "sampling/sampling_logp_difference/mean": 0.021463465876877308, "step": 850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.5, "completions/max_terminated_length": 249.5, "completions/mean_length": 187.05, "completions/mean_terminated_length": 187.05, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0520827293395996, "epoch": 0.07906591891146456, "frac_reward_zero_std": 0.5, "grad_norm": 0.0969572439789772, "kl": 0.0027062737382948397, "learning_rate": 2.6317401960784317e-06, "loss": -0.0003, "num_tokens": 3758776.0, "reward": 0.05, "reward_std": 0.4006543099880219, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.4006543219089508, "sampling/importance_sampling_ratio/max": 1.2775512337684631, "sampling/importance_sampling_ratio/mean": 0.9999620974063873, "sampling/importance_sampling_ratio/min": 0.7652165651321411, "sampling/sampling_logp_difference/max": 0.288371616601944, "sampling/sampling_logp_difference/mean": 0.021277916617691517, "step": 860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.4, "completions/max_terminated_length": 315.8, "completions/mean_length": 218.25, "completions/mean_terminated_length": 215.00892944335936, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.1387375429272653, "epoch": 0.07998529006159787, "frac_reward_zero_std": 0.5, "grad_norm": 5.6979475021362305, "kl": 0.002763241647335235, "learning_rate": 2.662377450980392e-06, "loss": 0.0001, "num_tokens": 3804252.0, "reward": 0.18125, "reward_std": 0.41367453932762144, "rewards/reward_fn/mean": 0.18125, "rewards/reward_fn/std": 0.4136745512485504, "sampling/importance_sampling_ratio/max": 1.2411170601844788, "sampling/importance_sampling_ratio/mean": 1.0003732740879059, "sampling/importance_sampling_ratio/min": 0.7765689730644226, "sampling/sampling_logp_difference/max": 0.25886437892913816, "sampling/sampling_logp_difference/mean": 0.021854002587497236, "step": 870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.5, "completions/max_terminated_length": 267.5, "completions/mean_length": 192.725, "completions/mean_terminated_length": 192.725, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 0.9888180930167436, "epoch": 0.08090466121173118, "frac_reward_zero_std": 0.5, "grad_norm": 5.668768405914307, "kl": 0.0038015075013390744, "learning_rate": 2.693014705882353e-06, "loss": 0.0004, "num_tokens": 3847798.0, "reward": 0.3125, "reward_std": 0.41135978102684023, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.4113598048686981, "sampling/importance_sampling_ratio/max": 1.2296013355255127, "sampling/importance_sampling_ratio/mean": 0.9992747724056243, "sampling/importance_sampling_ratio/min": 0.7497190058231353, "sampling/sampling_logp_difference/max": 0.3144635081291199, "sampling/sampling_logp_difference/mean": 0.020920691825449467, "step": 880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.6, "completions/max_terminated_length": 271.6, "completions/mean_length": 185.275, "completions/mean_terminated_length": 185.275, "completions/min_length": 115.9, "completions/min_terminated_length": 115.9, "entropy": 0.9807270668447018, "epoch": 0.08182403236186449, "frac_reward_zero_std": 0.4, "grad_norm": 7.880890369415283, "kl": 0.0029675340076209976, "learning_rate": 2.7236519607843136e-06, "loss": 0.0002, "num_tokens": 3887212.0, "reward": 0.0125, "reward_std": 0.4339252322912216, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.4339252501726151, "sampling/importance_sampling_ratio/max": 1.2872227907180787, "sampling/importance_sampling_ratio/mean": 1.0002362549304962, "sampling/importance_sampling_ratio/min": 0.7751561105251312, "sampling/sampling_logp_difference/max": 0.3027405202388763, "sampling/sampling_logp_difference/mean": 0.02112089265137911, "step": 890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 197.3625, "completions/mean_terminated_length": 197.3625, "completions/min_length": 112.8, "completions/min_terminated_length": 112.8, "entropy": 1.0829822316765785, "epoch": 0.0827434035119978, "frac_reward_zero_std": 0.2, "grad_norm": 6.487550258636475, "kl": 0.007837075377756264, "learning_rate": 2.754289215686275e-06, "loss": 0.0001, "num_tokens": 3928489.0, "reward": 0.0625, "reward_std": 0.5465950816869736, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.5465950936079025, "sampling/importance_sampling_ratio/max": 1.346196746826172, "sampling/importance_sampling_ratio/mean": 0.9996935188770294, "sampling/importance_sampling_ratio/min": 0.7384186804294586, "sampling/sampling_logp_difference/max": 0.36049288511276245, "sampling/sampling_logp_difference/mean": 0.02240522187203169, "step": 900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.1, "completions/max_terminated_length": 273.1, "completions/mean_length": 191.075, "completions/mean_terminated_length": 191.075, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 0.9956984333693981, "epoch": 0.08366277466213111, "frac_reward_zero_std": 0.3, "grad_norm": 6.19687032699585, "kl": 0.006185506476322189, "learning_rate": 2.7849264705882355e-06, "loss": 0.0007, "num_tokens": 3968975.0, "reward": -0.375, "reward_std": 0.5235352247953415, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5235352486371994, "sampling/importance_sampling_ratio/max": 1.2452479362487794, "sampling/importance_sampling_ratio/mean": 1.0005298733711243, "sampling/importance_sampling_ratio/min": 0.7847002863883972, "sampling/sampling_logp_difference/max": 0.250644251704216, "sampling/sampling_logp_difference/mean": 0.021063000336289404, "step": 910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.0, "completions/max_terminated_length": 286.0, "completions/mean_length": 193.075, "completions/mean_terminated_length": 193.075, "completions/min_length": 121.4, "completions/min_terminated_length": 121.4, "entropy": 0.998225849866867, "epoch": 0.0845821458122644, "frac_reward_zero_std": 0.2, "grad_norm": 9.962566375732422, "kl": 0.006246131453372073, "learning_rate": 2.8155637254901964e-06, "loss": 0.0004, "num_tokens": 4009973.0, "reward": 0.2375, "reward_std": 0.7299546062946319, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.7299546241760254, "sampling/importance_sampling_ratio/max": 1.2650769233703614, "sampling/importance_sampling_ratio/mean": 1.000108653306961, "sampling/importance_sampling_ratio/min": 0.7777700006961823, "sampling/sampling_logp_difference/max": 0.26809492111206057, "sampling/sampling_logp_difference/mean": 0.021244184114038944, "step": 920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 221.25, "completions/mean_terminated_length": 221.25, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.0295479901134967, "epoch": 0.08550151696239772, "frac_reward_zero_std": 0.2, "grad_norm": 5.537407875061035, "kl": 0.006818726073834114, "learning_rate": 2.846200980392157e-06, "loss": 0.0002, "num_tokens": 4056449.0, "reward": 0.05, "reward_std": 0.5779510378837586, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5779510498046875, "sampling/importance_sampling_ratio/max": 1.317081832885742, "sampling/importance_sampling_ratio/mean": 0.9995251536369324, "sampling/importance_sampling_ratio/min": 0.7700926482677459, "sampling/sampling_logp_difference/max": 0.30742167532444, "sampling/sampling_logp_difference/mean": 0.02112684603780508, "step": 930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 203.975, "completions/mean_terminated_length": 203.975, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.0224198609590531, "epoch": 0.08642088811253103, "frac_reward_zero_std": 0.2, "grad_norm": 6.340763568878174, "kl": 0.0062911443543271165, "learning_rate": 2.876838235294118e-06, "loss": -0.0, "num_tokens": 4096711.0, "reward": -0.05625, "reward_std": 0.606641286611557, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.6066413044929504, "sampling/importance_sampling_ratio/max": 1.2410983204841615, "sampling/importance_sampling_ratio/mean": 0.9997718155384063, "sampling/importance_sampling_ratio/min": 0.7799016654491424, "sampling/sampling_logp_difference/max": 0.25803680419921876, "sampling/sampling_logp_difference/mean": 0.02161556761711836, "step": 940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.5, "completions/max_terminated_length": 308.5, "completions/mean_length": 211.025, "completions/mean_terminated_length": 211.025, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.0685635283589363, "epoch": 0.08734025926266434, "frac_reward_zero_std": 0.3, "grad_norm": 5.713547229766846, "kl": 0.006182058043486904, "learning_rate": 2.9074754901960787e-06, "loss": 0.0007, "num_tokens": 4139513.0, "reward": 0.00625, "reward_std": 0.5506034731864929, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.5506034791469574, "sampling/importance_sampling_ratio/max": 1.376935601234436, "sampling/importance_sampling_ratio/mean": 1.0003678441047668, "sampling/importance_sampling_ratio/min": 0.8011228561401367, "sampling/sampling_logp_difference/max": 0.33361477255821226, "sampling/sampling_logp_difference/mean": 0.02131375763565302, "step": 950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 351.2, "completions/max_terminated_length": 333.2, "completions/mean_length": 242.9375, "completions/mean_terminated_length": 239.1500015258789, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0816266678273678, "epoch": 0.08825963041279765, "frac_reward_zero_std": 0.1, "grad_norm": 5.561310768127441, "kl": 0.005243503677775152, "learning_rate": 2.9381127450980397e-06, "loss": -0.0003, "num_tokens": 4184116.0, "reward": 0.025, "reward_std": 0.735353896021843, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.7353539139032363, "sampling/importance_sampling_ratio/max": 1.2709896206855773, "sampling/importance_sampling_ratio/mean": 0.9998113334178924, "sampling/importance_sampling_ratio/min": 0.7179039746522904, "sampling/sampling_logp_difference/max": 0.3657096326351166, "sampling/sampling_logp_difference/mean": 0.021757134050130845, "step": 960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.8, "completions/max_terminated_length": 319.8, "completions/mean_length": 213.85, "completions/mean_terminated_length": 213.85, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0483902730047703, "epoch": 0.08917900156293096, "frac_reward_zero_std": 0.2, "grad_norm": 4.578107833862305, "kl": 0.011414260690798983, "learning_rate": 2.96875e-06, "loss": 0.0003, "num_tokens": 4226344.0, "reward": 0.025, "reward_std": 0.49959076642990113, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.499590790271759, "sampling/importance_sampling_ratio/max": 1.3289848446846009, "sampling/importance_sampling_ratio/mean": 1.0000871539115905, "sampling/importance_sampling_ratio/min": 0.7711245894432068, "sampling/sampling_logp_difference/max": 0.3512672662734985, "sampling/sampling_logp_difference/mean": 0.020528460573405027, "step": 970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 197.9375, "completions/mean_terminated_length": 197.9375, "completions/min_length": 129.7, "completions/min_terminated_length": 129.7, "entropy": 1.0756870269775392, "epoch": 0.09009837271306427, "frac_reward_zero_std": 0.4, "grad_norm": 6.964056491851807, "kl": 0.01016737746540457, "learning_rate": 2.999387254901961e-06, "loss": 0.0007, "num_tokens": 4267955.0, "reward": -0.08125, "reward_std": 0.4659160763025284, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.46591608226299286, "sampling/importance_sampling_ratio/max": 1.2798233151435852, "sampling/importance_sampling_ratio/mean": 1.000102198123932, "sampling/importance_sampling_ratio/min": 0.587386866658926, "sampling/sampling_logp_difference/max": 0.8602162718772888, "sampling/sampling_logp_difference/mean": 0.021850606054067613, "step": 980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 190.0375, "completions/mean_terminated_length": 190.0375, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.049230270832777, "epoch": 0.09101774386319757, "frac_reward_zero_std": 0.1, "grad_norm": 7.506882190704346, "kl": 0.010564835877448785, "learning_rate": 3.0300245098039216e-06, "loss": 0.0005, "num_tokens": 4307070.0, "reward": 0.2375, "reward_std": 0.5135475039482117, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.5135475158691406, "sampling/importance_sampling_ratio/max": 1.2762622237205505, "sampling/importance_sampling_ratio/mean": 0.9993451952934265, "sampling/importance_sampling_ratio/min": 0.7719105541706085, "sampling/sampling_logp_difference/max": 0.2895656704902649, "sampling/sampling_logp_difference/mean": 0.021446413174271584, "step": 990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 200.35, "completions/mean_terminated_length": 200.35, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.0577165380120277, "epoch": 0.09193711501333088, "frac_reward_zero_std": 0.1, "grad_norm": 5.163789749145508, "kl": 0.006241366888571065, "learning_rate": 3.060661764705883e-06, "loss": 0.0, "num_tokens": 4349418.0, "reward": -0.09375, "reward_std": 0.6604974716901779, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.6604974955320358, "sampling/importance_sampling_ratio/max": 1.2737963676452637, "sampling/importance_sampling_ratio/mean": 1.0003153443336488, "sampling/importance_sampling_ratio/min": 0.7728460252285003, "sampling/sampling_logp_difference/max": 0.2808604598045349, "sampling/sampling_logp_difference/mean": 0.021331284753978254, "step": 1000 }, { "epoch": 0.09193711501333088, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 288.0471204188482, "eval_completions/max_terminated_length": 285.41012216404886, "eval_completions/mean_length": 195.10362129144852, "eval_completions/mean_terminated_length": 194.4758478933604, "eval_completions/min_length": 123.08027923211169, "eval_completions/min_terminated_length": 123.08027923211169, "eval_entropy": 1.0918490369698557, "eval_frac_reward_zero_std": 0.2879581151832461, "eval_kl": 0.008752725989172596, "eval_loss": 0.00039568814099766314, "eval_num_tokens": 4349418.0, "eval_reward": -0.04144851657940663, "eval_reward_std": 0.49292232473275216, "eval_rewards/reward_fn/mean": -0.04144851657940663, "eval_rewards/reward_fn/std": 0.4929223387757313, "eval_runtime": 1688.7144, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2956086819500616, "eval_sampling/importance_sampling_ratio/mean": 0.9999952431748674, "eval_sampling/importance_sampling_ratio/min": 0.7587555581622515, "eval_sampling/sampling_logp_difference/max": 0.3259408835965301, "eval_sampling/sampling_logp_difference/mean": 0.021509873033883162, "eval_steps_per_second": 0.043, "step": 1000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.3, "completions/max_terminated_length": 279.3, "completions/mean_length": 204.9125, "completions/mean_terminated_length": 204.9125, "completions/min_length": 136.9, "completions/min_terminated_length": 136.9, "entropy": 1.1395965121686458, "epoch": 0.09285648616346419, "frac_reward_zero_std": 0.1, "grad_norm": 6.542971134185791, "kl": 0.008696513864560985, "learning_rate": 3.0912990196078434e-06, "loss": 0.0005, "num_tokens": 4396227.0, "reward": -0.275, "reward_std": 0.6093845218420029, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.6093845516443253, "sampling/importance_sampling_ratio/max": 1.2921066522598266, "sampling/importance_sampling_ratio/mean": 1.0000555276870728, "sampling/importance_sampling_ratio/min": 0.7674422830343246, "sampling/sampling_logp_difference/max": 0.30101699829101564, "sampling/sampling_logp_difference/mean": 0.022433127835392953, "step": 1010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.0, "completions/max_terminated_length": 301.0, "completions/mean_length": 206.125, "completions/mean_terminated_length": 206.125, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.1115202084183693, "epoch": 0.0937758573135975, "frac_reward_zero_std": 0.3, "grad_norm": 5.391073226928711, "kl": 0.006414841805235483, "learning_rate": 3.1219362745098044e-06, "loss": 0.0002, "num_tokens": 4440333.0, "reward": 0.1375, "reward_std": 0.535081484913826, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.5350814908742905, "sampling/importance_sampling_ratio/max": 1.2689568161964417, "sampling/importance_sampling_ratio/mean": 0.9998329162597657, "sampling/importance_sampling_ratio/min": 0.7038759469985962, "sampling/sampling_logp_difference/max": 0.3774571418762207, "sampling/sampling_logp_difference/mean": 0.02169222626835108, "step": 1020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.5, "completions/max_terminated_length": 266.5, "completions/mean_length": 194.575, "completions/mean_terminated_length": 194.575, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0383225880563258, "epoch": 0.09469522846373081, "frac_reward_zero_std": 0.1, "grad_norm": 4.7692670822143555, "kl": 0.01427452263305895, "learning_rate": 3.152573529411765e-06, "loss": 0.0006, "num_tokens": 4479491.0, "reward": -0.075, "reward_std": 0.6796805769205093, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.6796805888414383, "sampling/importance_sampling_ratio/max": 1.2769641399383544, "sampling/importance_sampling_ratio/mean": 1.000789326429367, "sampling/importance_sampling_ratio/min": 0.7282934546470642, "sampling/sampling_logp_difference/max": 0.352101719379425, "sampling/sampling_logp_difference/mean": 0.020846835151314736, "step": 1030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.9, "completions/max_terminated_length": 286.9, "completions/mean_length": 205.9375, "completions/mean_terminated_length": 205.9375, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.11657222956419, "epoch": 0.09561459961386412, "frac_reward_zero_std": 0.2, "grad_norm": 6.065827369689941, "kl": 0.010999771510250866, "learning_rate": 3.1832107843137254e-06, "loss": 0.0002, "num_tokens": 4523246.0, "reward": -0.375, "reward_std": 0.5288968861103058, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.5288969099521637, "sampling/importance_sampling_ratio/max": 1.2682188630104065, "sampling/importance_sampling_ratio/mean": 0.9997571289539338, "sampling/importance_sampling_ratio/min": 0.7869106948375701, "sampling/sampling_logp_difference/max": 0.2651927322149277, "sampling/sampling_logp_difference/mean": 0.02140223141759634, "step": 1040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.5, "completions/max_terminated_length": 339.5, "completions/mean_length": 221.075, "completions/mean_terminated_length": 221.075, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0718603052198887, "epoch": 0.09653397076399743, "frac_reward_zero_std": 0.2, "grad_norm": 6.041465759277344, "kl": 0.014499837646144442, "learning_rate": 3.2138480392156867e-06, "loss": 0.0006, "num_tokens": 4566860.0, "reward": 0.2875, "reward_std": 0.5085761934518814, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.5085762113332748, "sampling/importance_sampling_ratio/max": 1.2474358439445496, "sampling/importance_sampling_ratio/mean": 0.9997801959514618, "sampling/importance_sampling_ratio/min": 0.7986530303955078, "sampling/sampling_logp_difference/max": 0.24827976226806642, "sampling/sampling_logp_difference/mean": 0.02116443943232298, "step": 1050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.0, "completions/max_terminated_length": 325.0, "completions/mean_length": 220.5125, "completions/mean_terminated_length": 220.5125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.09085945263505, "epoch": 0.09745334191413073, "frac_reward_zero_std": 0.3, "grad_norm": 0.09769704937934875, "kl": 0.01425217889191117, "learning_rate": 3.2444852941176476e-06, "loss": 0.001, "num_tokens": 4611845.0, "reward": 0.1125, "reward_std": 0.5213924765586853, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.5213924765586853, "sampling/importance_sampling_ratio/max": 1.3713450193405152, "sampling/importance_sampling_ratio/mean": 1.0005628168582916, "sampling/importance_sampling_ratio/min": 0.7301262199878693, "sampling/sampling_logp_difference/max": 0.34665178060531615, "sampling/sampling_logp_difference/mean": 0.021962456032633783, "step": 1060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 201.025, "completions/mean_terminated_length": 201.025, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.209683421999216, "epoch": 0.09837271306426404, "frac_reward_zero_std": 0.2, "grad_norm": 4.830780982971191, "kl": 0.012879665949731134, "learning_rate": 3.275122549019608e-06, "loss": 0.0007, "num_tokens": 4655087.0, "reward": -0.125, "reward_std": 0.5285374969244003, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.5285375028848648, "sampling/importance_sampling_ratio/max": 1.3322084188461303, "sampling/importance_sampling_ratio/mean": 0.9994369089603424, "sampling/importance_sampling_ratio/min": 0.7903971552848816, "sampling/sampling_logp_difference/max": 0.3585920214653015, "sampling/sampling_logp_difference/mean": 0.023745033517479898, "step": 1070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.9, "completions/max_terminated_length": 326.9, "completions/mean_length": 210.5375, "completions/mean_terminated_length": 210.5375, "completions/min_length": 130.2, "completions/min_terminated_length": 130.2, "entropy": 1.1410760872066021, "epoch": 0.09929208421439735, "frac_reward_zero_std": 0.3, "grad_norm": 0.08446750044822693, "kl": 0.05767056857876014, "learning_rate": 3.3057598039215686e-06, "loss": 0.0024, "num_tokens": 4702522.0, "reward": -0.24375, "reward_std": 0.500537320971489, "rewards/reward_fn/mean": -0.24375, "rewards/reward_fn/std": 0.5005373388528824, "sampling/importance_sampling_ratio/max": 1.2505094289779664, "sampling/importance_sampling_ratio/mean": 1.0000601351261138, "sampling/importance_sampling_ratio/min": 0.7441504150629044, "sampling/sampling_logp_difference/max": 0.3169679373502731, "sampling/sampling_logp_difference/mean": 0.022094489261507987, "step": 1080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 202.2375, "completions/mean_terminated_length": 202.2375, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.072489184141159, "epoch": 0.10021145536453066, "frac_reward_zero_std": 0.2, "grad_norm": 6.569357872009277, "kl": 0.013545406749472022, "learning_rate": 3.3363970588235295e-06, "loss": 0.0008, "num_tokens": 4746397.0, "reward": 0.2375, "reward_std": 0.5542894184589386, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.554289436340332, "sampling/importance_sampling_ratio/max": 1.3320290684700011, "sampling/importance_sampling_ratio/mean": 1.0002446115016936, "sampling/importance_sampling_ratio/min": 0.7941021025180817, "sampling/sampling_logp_difference/max": 0.31126514077186584, "sampling/sampling_logp_difference/mean": 0.021482798084616662, "step": 1090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.6, "completions/max_terminated_length": 301.6, "completions/mean_length": 192.825, "completions/mean_terminated_length": 192.825, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0538658171892166, "epoch": 0.10113082651466397, "frac_reward_zero_std": 0.3, "grad_norm": 6.378344535827637, "kl": 0.01226953813165892, "learning_rate": 3.367034313725491e-06, "loss": 0.0009, "num_tokens": 4787183.0, "reward": -0.0375, "reward_std": 0.44131766855716703, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.44131766855716703, "sampling/importance_sampling_ratio/max": 1.3314331650733948, "sampling/importance_sampling_ratio/mean": 0.9999584257602692, "sampling/importance_sampling_ratio/min": 0.7500885725021362, "sampling/sampling_logp_difference/max": 0.32559962272644044, "sampling/sampling_logp_difference/mean": 0.021489018015563487, "step": 1100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.9, "completions/max_terminated_length": 292.9, "completions/mean_length": 199.3125, "completions/mean_terminated_length": 199.3125, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.0446888476610183, "epoch": 0.10205019766479728, "frac_reward_zero_std": 0.3, "grad_norm": 0.19413918256759644, "kl": 0.012133742007426918, "learning_rate": 3.3976715686274514e-06, "loss": 0.0006, "num_tokens": 4833664.0, "reward": 0.3125, "reward_std": 0.438364240527153, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.43836425244808197, "sampling/importance_sampling_ratio/max": 1.2479475140571594, "sampling/importance_sampling_ratio/mean": 1.000616192817688, "sampling/importance_sampling_ratio/min": 0.7562075972557067, "sampling/sampling_logp_difference/max": 0.2886667728424072, "sampling/sampling_logp_difference/mean": 0.022036554664373396, "step": 1110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 198.0875, "completions/mean_terminated_length": 198.0875, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.0662069909274579, "epoch": 0.10296956881493059, "frac_reward_zero_std": 0.3, "grad_norm": 7.773277282714844, "kl": 0.007365902060701046, "learning_rate": 3.428308823529412e-06, "loss": 0.0001, "num_tokens": 4875367.0, "reward": -0.3, "reward_std": 0.42112618386745454, "rewards/reward_fn/mean": -0.3, "rewards/reward_fn/std": 0.421126189827919, "sampling/importance_sampling_ratio/max": 1.2560564041137696, "sampling/importance_sampling_ratio/mean": 0.9996202647686004, "sampling/importance_sampling_ratio/min": 0.7204268157482148, "sampling/sampling_logp_difference/max": 0.3421809434890747, "sampling/sampling_logp_difference/mean": 0.022097828239202498, "step": 1120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 195.45, "completions/mean_terminated_length": 195.45, "completions/min_length": 118.5, "completions/min_terminated_length": 118.5, "entropy": 1.0936168320477009, "epoch": 0.10388893996506389, "frac_reward_zero_std": 0.1, "grad_norm": 8.33864974975586, "kl": 0.01408274565183092, "learning_rate": 3.458946078431373e-06, "loss": 0.0003, "num_tokens": 4918427.0, "reward": -0.31875, "reward_std": 0.5171559393405915, "rewards/reward_fn/mean": -0.31875, "rewards/reward_fn/std": 0.5171559512615204, "sampling/importance_sampling_ratio/max": 1.3096859097480773, "sampling/importance_sampling_ratio/mean": 0.9999238014221191, "sampling/importance_sampling_ratio/min": 0.723172527551651, "sampling/sampling_logp_difference/max": 0.35930281281471255, "sampling/sampling_logp_difference/mean": 0.02284015081822872, "step": 1130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 220.0625, "completions/mean_terminated_length": 220.0625, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0693205066025258, "epoch": 0.1048083111151972, "frac_reward_zero_std": 0.3, "grad_norm": 6.8803791999816895, "kl": 0.010877759483992121, "learning_rate": 3.4895833333333333e-06, "loss": 0.0007, "num_tokens": 4965360.0, "reward": -0.325, "reward_std": 0.5109704315662384, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.5109704434871674, "sampling/importance_sampling_ratio/max": 1.297501516342163, "sampling/importance_sampling_ratio/mean": 0.9997015535831452, "sampling/importance_sampling_ratio/min": 0.7729672253131866, "sampling/sampling_logp_difference/max": 0.29393945932388305, "sampling/sampling_logp_difference/mean": 0.02267507053911686, "step": 1140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.6, "completions/max_terminated_length": 296.6, "completions/mean_length": 216.875, "completions/mean_terminated_length": 216.875, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 0.9997228812426329, "epoch": 0.10572768226533051, "frac_reward_zero_std": 0.2, "grad_norm": 6.133396625518799, "kl": 0.013946997774473857, "learning_rate": 3.5202205882352947e-06, "loss": 0.0003, "num_tokens": 5007366.0, "reward": 0.15, "reward_std": 0.5775342971086502, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.5775343269109726, "sampling/importance_sampling_ratio/max": 1.3933629989624023, "sampling/importance_sampling_ratio/mean": 1.0001138508319856, "sampling/importance_sampling_ratio/min": 0.7607937216758728, "sampling/sampling_logp_difference/max": 0.4101886212825775, "sampling/sampling_logp_difference/mean": 0.021730512380599976, "step": 1150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.2, "completions/max_terminated_length": 293.7, "completions/mean_length": 208.8625, "completions/mean_terminated_length": 205.3732147216797, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0656747043132782, "epoch": 0.10664705341546382, "frac_reward_zero_std": 0.1, "grad_norm": 7.143483638763428, "kl": 0.010617410464328713, "learning_rate": 3.550857843137255e-06, "loss": 0.0005, "num_tokens": 5052755.0, "reward": -0.05625, "reward_std": 0.6774400025606155, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.6774400383234024, "sampling/importance_sampling_ratio/max": 1.4161882400512695, "sampling/importance_sampling_ratio/mean": 0.9996443450450897, "sampling/importance_sampling_ratio/min": 0.7458942472934723, "sampling/sampling_logp_difference/max": 0.40879332423210146, "sampling/sampling_logp_difference/mean": 0.022250896878540517, "step": 1160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.6, "completions/max_terminated_length": 276.6, "completions/mean_length": 206.2, "completions/mean_terminated_length": 206.2, "completions/min_length": 147.2, "completions/min_terminated_length": 147.2, "entropy": 1.0438854165375233, "epoch": 0.10756642456559713, "frac_reward_zero_std": 0.4, "grad_norm": 0.24372972548007965, "kl": 0.010773162270197645, "learning_rate": 3.581495098039216e-06, "loss": 0.0004, "num_tokens": 5095195.0, "reward": 0.075, "reward_std": 0.5114813864231109, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5114813983440399, "sampling/importance_sampling_ratio/max": 1.2707341670989991, "sampling/importance_sampling_ratio/mean": 1.000458264350891, "sampling/importance_sampling_ratio/min": 0.7152528584003448, "sampling/sampling_logp_difference/max": 0.35886303782463075, "sampling/sampling_logp_difference/mean": 0.022765771113336088, "step": 1170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.8, "completions/max_terminated_length": 271.8, "completions/mean_length": 188.375, "completions/mean_terminated_length": 188.375, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0524725951254368, "epoch": 0.10848579571573044, "frac_reward_zero_std": 0.3, "grad_norm": 0.10499633848667145, "kl": 0.0076308351097395645, "learning_rate": 3.6121323529411766e-06, "loss": -0.0001, "num_tokens": 5138401.0, "reward": -0.0625, "reward_std": 0.44819211661815644, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.4481921285390854, "sampling/importance_sampling_ratio/max": 1.2385743021965028, "sampling/importance_sampling_ratio/mean": 1.000013965368271, "sampling/importance_sampling_ratio/min": 0.7369125068187714, "sampling/sampling_logp_difference/max": 0.31007453203201296, "sampling/sampling_logp_difference/mean": 0.022578140906989574, "step": 1180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 286.9, "completions/max_terminated_length": 268.9, "completions/mean_length": 190.5625, "completions/mean_terminated_length": 187.53035888671874, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 0.9723918177187443, "epoch": 0.10940516686586375, "frac_reward_zero_std": 0.6, "grad_norm": 0.22342681884765625, "kl": 0.00678293082601158, "learning_rate": 3.6427696078431375e-06, "loss": 0.0004, "num_tokens": 5181374.0, "reward": 0.44375, "reward_std": 0.2882675379514694, "rewards/reward_fn/mean": 0.44375, "rewards/reward_fn/std": 0.2882675439119339, "sampling/importance_sampling_ratio/max": 1.3537864804267883, "sampling/importance_sampling_ratio/mean": 1.0001680910587312, "sampling/importance_sampling_ratio/min": 0.7728900909423828, "sampling/sampling_logp_difference/max": 0.32552298307418825, "sampling/sampling_logp_difference/mean": 0.020928254909813405, "step": 1190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 218.75, "completions/mean_terminated_length": 218.75, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.1200744204223156, "epoch": 0.11032453801599706, "frac_reward_zero_std": 0.3, "grad_norm": 5.122947692871094, "kl": 0.00601845326309558, "learning_rate": 3.6734068627450984e-06, "loss": 0.0, "num_tokens": 5228362.0, "reward": 0.1375, "reward_std": 0.5417128652334213, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.5417128771543502, "sampling/importance_sampling_ratio/max": 1.304845130443573, "sampling/importance_sampling_ratio/mean": 0.9999196469783783, "sampling/importance_sampling_ratio/min": 0.7648558259010315, "sampling/sampling_logp_difference/max": 0.29331214427948, "sampling/sampling_logp_difference/mean": 0.022312159277498723, "step": 1200 }, { "epoch": 0.11032453801599706, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 298.37696335078533, "eval_completions/max_terminated_length": 296.7312390924956, "eval_completions/mean_length": 210.15903141361255, "eval_completions/mean_terminated_length": 209.85418237815978, "eval_completions/min_length": 140.44851657940663, "eval_completions/min_terminated_length": 140.44851657940663, "eval_entropy": 1.068753021549804, "eval_frac_reward_zero_std": 0.33158813263525305, "eval_kl": 0.007995697537016117, "eval_loss": 0.0004092247982043773, "eval_num_tokens": 5228362.0, "eval_reward": -0.04712041884816754, "eval_reward_std": 0.4636780871614736, "eval_rewards/reward_fn/mean": -0.04712041884816754, "eval_rewards/reward_fn/std": 0.4636781009964086, "eval_runtime": 1739.5189, "eval_samples_per_second": 0.329, "eval_sampling/importance_sampling_ratio/max": 1.2854634885388518, "eval_sampling/importance_sampling_ratio/mean": 1.0000063097705807, "eval_sampling/importance_sampling_ratio/min": 0.7597562024990717, "eval_sampling/sampling_logp_difference/max": 0.3159886962664273, "eval_sampling/sampling_logp_difference/mean": 0.021668341415519803, "eval_steps_per_second": 0.041, "step": 1200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.9, "completions/max_terminated_length": 324.9, "completions/mean_length": 224.55, "completions/mean_terminated_length": 224.55, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.0434857800602912, "epoch": 0.11124390916613036, "frac_reward_zero_std": 0.3, "grad_norm": 0.1081586703658104, "kl": 0.006170208979165181, "learning_rate": 3.7040441176470593e-06, "loss": 0.0001, "num_tokens": 5274734.0, "reward": -0.2375, "reward_std": 0.40021162629127505, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.4002116322517395, "sampling/importance_sampling_ratio/max": 1.2809426426887511, "sampling/importance_sampling_ratio/mean": 0.999780023097992, "sampling/importance_sampling_ratio/min": 0.7142409771680832, "sampling/sampling_logp_difference/max": 0.44220882654190063, "sampling/sampling_logp_difference/mean": 0.021672898903489113, "step": 1210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.3, "completions/max_terminated_length": 289.3, "completions/mean_length": 210.675, "completions/mean_terminated_length": 210.675, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 0.968850920163095, "epoch": 0.11216328031626367, "frac_reward_zero_std": 0.2, "grad_norm": 5.864732265472412, "kl": 0.009097536602712352, "learning_rate": 3.73468137254902e-06, "loss": -0.0002, "num_tokens": 5316476.0, "reward": 0.2, "reward_std": 0.5061949193477631, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.506194931268692, "sampling/importance_sampling_ratio/max": 1.3063884854316712, "sampling/importance_sampling_ratio/mean": 0.9998998701572418, "sampling/importance_sampling_ratio/min": 0.737203985452652, "sampling/sampling_logp_difference/max": 0.3477449119091034, "sampling/sampling_logp_difference/mean": 0.02080409824848175, "step": 1220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.6, "completions/max_terminated_length": 298.8, "completions/mean_length": 222.725, "completions/mean_terminated_length": 219.13750152587892, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.1194646969437598, "epoch": 0.11308265146639698, "frac_reward_zero_std": 0.2, "grad_norm": 6.690155506134033, "kl": 0.011455826633027754, "learning_rate": 3.7653186274509808e-06, "loss": 0.0002, "num_tokens": 5358758.0, "reward": -0.6, "reward_std": 0.5313313931226731, "rewards/reward_fn/mean": -0.6, "rewards/reward_fn/std": 0.5313313931226731, "sampling/importance_sampling_ratio/max": 1.306610071659088, "sampling/importance_sampling_ratio/mean": 1.0002062559127807, "sampling/importance_sampling_ratio/min": 0.7746486663818359, "sampling/sampling_logp_difference/max": 0.2955535888671875, "sampling/sampling_logp_difference/mean": 0.021349863335490226, "step": 1230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.3, "completions/max_terminated_length": 300.3, "completions/mean_length": 228.9875, "completions/mean_terminated_length": 228.9875, "completions/min_length": 158.6, "completions/min_terminated_length": 158.6, "entropy": 1.1286462068557739, "epoch": 0.1140020226165303, "frac_reward_zero_std": 0.2, "grad_norm": 6.212457656860352, "kl": 0.00790575742139481, "learning_rate": 3.7959558823529413e-06, "loss": 0.0, "num_tokens": 5405533.0, "reward": 0.23125, "reward_std": 0.577428537607193, "rewards/reward_fn/mean": 0.23125, "rewards/reward_fn/std": 0.5774285733699799, "sampling/importance_sampling_ratio/max": 1.2457308173179626, "sampling/importance_sampling_ratio/mean": 1.0003167986869812, "sampling/importance_sampling_ratio/min": 0.8062489449977874, "sampling/sampling_logp_difference/max": 0.23042677640914916, "sampling/sampling_logp_difference/mean": 0.021200483851134776, "step": 1240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.0, "completions/max_terminated_length": 265.0, "completions/mean_length": 203.4, "completions/mean_terminated_length": 203.4, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.0522544912993907, "epoch": 0.1149213937666636, "frac_reward_zero_std": 0.2, "grad_norm": 0.13615116477012634, "kl": 0.009838560479693115, "learning_rate": 3.826593137254902e-06, "loss": 0.0005, "num_tokens": 5446733.0, "reward": 0.1625, "reward_std": 0.6745196104049682, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.6745196282863617, "sampling/importance_sampling_ratio/max": 1.283636200428009, "sampling/importance_sampling_ratio/mean": 1.0004195511341094, "sampling/importance_sampling_ratio/min": 0.734540593624115, "sampling/sampling_logp_difference/max": 0.3464024543762207, "sampling/sampling_logp_difference/mean": 0.021544823609292506, "step": 1250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 202.3875, "completions/mean_terminated_length": 202.3875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0066056750714778, "epoch": 0.11584076491679692, "frac_reward_zero_std": 0.4, "grad_norm": 5.153637409210205, "kl": 0.008987490230356342, "learning_rate": 3.857230392156863e-06, "loss": 0.0006, "num_tokens": 5484028.0, "reward": 0.2375, "reward_std": 0.4570765882730484, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.45707660615444184, "sampling/importance_sampling_ratio/max": 1.281479012966156, "sampling/importance_sampling_ratio/mean": 0.9997131764888764, "sampling/importance_sampling_ratio/min": 0.7385365486145019, "sampling/sampling_logp_difference/max": 0.3457703173160553, "sampling/sampling_logp_difference/mean": 0.020664984360337257, "step": 1260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.1, "completions/max_terminated_length": 305.1, "completions/mean_length": 217.125, "completions/mean_terminated_length": 217.125, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.1688122861087322, "epoch": 0.11676013606693023, "frac_reward_zero_std": 0.5, "grad_norm": 0.08216671645641327, "kl": 0.006927935587009415, "learning_rate": 3.887867647058824e-06, "loss": -0.0001, "num_tokens": 5528502.0, "reward": -0.0875, "reward_std": 0.32558145225048063, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.3255814641714096, "sampling/importance_sampling_ratio/max": 1.278193998336792, "sampling/importance_sampling_ratio/mean": 1.0003093004226684, "sampling/importance_sampling_ratio/min": 0.7788882911205292, "sampling/sampling_logp_difference/max": 0.27155337035655974, "sampling/sampling_logp_difference/mean": 0.023232090286910534, "step": 1270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 289.4, "completions/max_terminated_length": 271.4, "completions/mean_length": 193.1875, "completions/mean_terminated_length": 189.93035888671875, "completions/min_length": 120.2, "completions/min_terminated_length": 120.2, "entropy": 1.0665395490825176, "epoch": 0.11767950721706352, "frac_reward_zero_std": 0.4, "grad_norm": 6.827215194702148, "kl": 0.00874875822046306, "learning_rate": 3.918504901960785e-06, "loss": 0.0006, "num_tokens": 5571653.0, "reward": -0.03125, "reward_std": 0.2973824992775917, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.2973825052380562, "sampling/importance_sampling_ratio/max": 1.3046435356140136, "sampling/importance_sampling_ratio/mean": 1.0001174807548523, "sampling/importance_sampling_ratio/min": 0.7995468974113464, "sampling/sampling_logp_difference/max": 0.2812245011329651, "sampling/sampling_logp_difference/mean": 0.021241790615022182, "step": 1280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.9, "completions/max_terminated_length": 246.9, "completions/mean_length": 171.55, "completions/mean_terminated_length": 171.55, "completions/min_length": 115.2, "completions/min_terminated_length": 115.2, "entropy": 0.9502380028367042, "epoch": 0.11859887836719683, "frac_reward_zero_std": 0.3, "grad_norm": 5.376691818237305, "kl": 0.012134195037651807, "learning_rate": 3.9491421568627455e-06, "loss": 0.0008, "num_tokens": 5607961.0, "reward": -0.1375, "reward_std": 0.39840384423732755, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.39840384423732755, "sampling/importance_sampling_ratio/max": 1.3476706624031067, "sampling/importance_sampling_ratio/mean": 1.0007337510585785, "sampling/importance_sampling_ratio/min": 0.7455664277076721, "sampling/sampling_logp_difference/max": 0.36572521924972534, "sampling/sampling_logp_difference/mean": 0.021275184117257595, "step": 1290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 197.75, "completions/mean_terminated_length": 197.75, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.0110009878873825, "epoch": 0.11951824951733014, "frac_reward_zero_std": 0.4, "grad_norm": 8.256997108459473, "kl": 0.009620112203992903, "learning_rate": 3.979779411764706e-06, "loss": 0.0006, "num_tokens": 5651389.0, "reward": -0.1, "reward_std": 0.370937442779541, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.370937442779541, "sampling/importance_sampling_ratio/max": 1.2458324909210206, "sampling/importance_sampling_ratio/mean": 0.9999366641044617, "sampling/importance_sampling_ratio/min": 0.7940617024898529, "sampling/sampling_logp_difference/max": 0.23945897817611694, "sampling/sampling_logp_difference/mean": 0.020540885999798773, "step": 1300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 215.7125, "completions/mean_terminated_length": 215.7125, "completions/min_length": 144.6, "completions/min_terminated_length": 144.6, "entropy": 1.045546467602253, "epoch": 0.12043762066746345, "frac_reward_zero_std": 0.2, "grad_norm": 4.2061614990234375, "kl": 0.00735490906808991, "learning_rate": 4.010416666666667e-06, "loss": 0.0004, "num_tokens": 5696502.0, "reward": 0.15, "reward_std": 0.6018513649702072, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.6018513709306716, "sampling/importance_sampling_ratio/max": 1.2397029757499696, "sampling/importance_sampling_ratio/mean": 1.0003082811832429, "sampling/importance_sampling_ratio/min": 0.7763580977916718, "sampling/sampling_logp_difference/max": 0.2685515284538269, "sampling/sampling_logp_difference/mean": 0.021022022888064384, "step": 1310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 307.6, "completions/max_terminated_length": 294.9, "completions/mean_length": 231.25, "completions/mean_terminated_length": 226.32916870117188, "completions/min_length": 160.6, "completions/min_terminated_length": 160.6, "entropy": 1.1383093502372503, "epoch": 0.12135699181759677, "frac_reward_zero_std": 0.3, "grad_norm": 7.147118091583252, "kl": 0.00999721254920587, "learning_rate": 4.041053921568628e-06, "loss": 0.0004, "num_tokens": 5740474.0, "reward": 0.0625, "reward_std": 0.4475270092487335, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.44752702713012693, "sampling/importance_sampling_ratio/max": 1.2792275428771973, "sampling/importance_sampling_ratio/mean": 0.9997247695922852, "sampling/importance_sampling_ratio/min": 0.7661856114864349, "sampling/sampling_logp_difference/max": 0.3035975515842438, "sampling/sampling_logp_difference/mean": 0.022428595181554555, "step": 1320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.4, "completions/max_terminated_length": 324.1, "completions/mean_length": 241.2, "completions/mean_terminated_length": 238.375, "completions/min_length": 168.0, "completions/min_terminated_length": 168.0, "entropy": 1.043086964637041, "epoch": 0.12227636296773008, "frac_reward_zero_std": 0.2, "grad_norm": 0.07285313308238983, "kl": 0.009247479937039315, "learning_rate": 4.071691176470588e-06, "loss": -0.0001, "num_tokens": 5788738.0, "reward": 0.0125, "reward_std": 0.5789267063140869, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.5789267361164093, "sampling/importance_sampling_ratio/max": 1.2721714496612548, "sampling/importance_sampling_ratio/mean": 1.0001012861728669, "sampling/importance_sampling_ratio/min": 0.7748051226139069, "sampling/sampling_logp_difference/max": 0.27231634259223936, "sampling/sampling_logp_difference/mean": 0.020653835125267507, "step": 1330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.4, "completions/max_terminated_length": 317.4, "completions/mean_length": 210.8375, "completions/mean_terminated_length": 210.8375, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.041532776877284, "epoch": 0.12319573411786339, "frac_reward_zero_std": 0.2, "grad_norm": 6.706281661987305, "kl": 0.01274330764717888, "learning_rate": 4.102328431372549e-06, "loss": -0.0003, "num_tokens": 5833565.0, "reward": 0.1, "reward_std": 0.5324686735868454, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5324686974287033, "sampling/importance_sampling_ratio/max": 1.2627211689949036, "sampling/importance_sampling_ratio/mean": 1.000657558441162, "sampling/importance_sampling_ratio/min": 0.767503845691681, "sampling/sampling_logp_difference/max": 0.2748436689376831, "sampling/sampling_logp_difference/mean": 0.021594073809683322, "step": 1340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.6, "completions/max_terminated_length": 283.6, "completions/mean_length": 199.45, "completions/mean_terminated_length": 199.45, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.0785112269222736, "epoch": 0.12411510526799668, "frac_reward_zero_std": 0.4, "grad_norm": 5.028674125671387, "kl": 0.010370943482848815, "learning_rate": 4.13296568627451e-06, "loss": 0.0007, "num_tokens": 5878649.0, "reward": 0.3125, "reward_std": 0.4265191316604614, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.42651914358139037, "sampling/importance_sampling_ratio/max": 1.2595079779624938, "sampling/importance_sampling_ratio/mean": 1.0000385701656342, "sampling/importance_sampling_ratio/min": 0.7906409800052643, "sampling/sampling_logp_difference/max": 0.251240661740303, "sampling/sampling_logp_difference/mean": 0.021430917829275132, "step": 1350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 206.3625, "completions/mean_terminated_length": 206.3625, "completions/min_length": 126.9, "completions/min_terminated_length": 126.9, "entropy": 1.0548729564994574, "epoch": 0.12503447641813, "frac_reward_zero_std": 0.1, "grad_norm": 0.1075732409954071, "kl": 0.011580927346949465, "learning_rate": 4.1636029411764715e-06, "loss": 0.0005, "num_tokens": 5926526.0, "reward": 0.325, "reward_std": 0.7203713327646255, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.7203713744878769, "sampling/importance_sampling_ratio/max": 1.3204351902008056, "sampling/importance_sampling_ratio/mean": 0.9999918341636658, "sampling/importance_sampling_ratio/min": 0.7800904512405396, "sampling/sampling_logp_difference/max": 0.3146080493927002, "sampling/sampling_logp_difference/mean": 0.02158144675195217, "step": 1360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.9, "completions/max_terminated_length": 264.9, "completions/mean_length": 187.75, "completions/mean_terminated_length": 187.75, "completions/min_length": 113.1, "completions/min_terminated_length": 113.1, "entropy": 1.035442354902625, "epoch": 0.1259538475682633, "frac_reward_zero_std": 0.2, "grad_norm": 7.101892471313477, "kl": 0.016452591243432835, "learning_rate": 4.194240196078432e-06, "loss": 0.0013, "num_tokens": 5966618.0, "reward": 0.375, "reward_std": 0.4372564494609833, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.4372564613819122, "sampling/importance_sampling_ratio/max": 1.2982187032699586, "sampling/importance_sampling_ratio/mean": 0.9996362388134002, "sampling/importance_sampling_ratio/min": 0.6908780992031097, "sampling/sampling_logp_difference/max": 0.4562541127204895, "sampling/sampling_logp_difference/mean": 0.021488183457404376, "step": 1370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.7, "completions/max_terminated_length": 336.7, "completions/mean_length": 223.9875, "completions/mean_terminated_length": 223.9875, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.1679816335439681, "epoch": 0.12687321871839663, "frac_reward_zero_std": 0.2, "grad_norm": 7.767652988433838, "kl": 0.012456041463883594, "learning_rate": 4.2248774509803925e-06, "loss": 0.0008, "num_tokens": 6011585.0, "reward": 0.0, "reward_std": 0.5558438330888749, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5558438569307327, "sampling/importance_sampling_ratio/max": 1.334365463256836, "sampling/importance_sampling_ratio/mean": 0.9999495685100556, "sampling/importance_sampling_ratio/min": 0.7871855437755585, "sampling/sampling_logp_difference/max": 0.36275808811187743, "sampling/sampling_logp_difference/mean": 0.022078605182468892, "step": 1380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.8, "completions/max_terminated_length": 338.8, "completions/mean_length": 242.225, "completions/mean_terminated_length": 242.225, "completions/min_length": 173.9, "completions/min_terminated_length": 173.9, "entropy": 1.1290132313966752, "epoch": 0.12779258986852993, "frac_reward_zero_std": 0.3, "grad_norm": 5.37898588180542, "kl": 0.02128637230489403, "learning_rate": 4.255514705882353e-06, "loss": 0.0005, "num_tokens": 6060027.0, "reward": -0.1375, "reward_std": 0.45775099098682404, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.4577509969472885, "sampling/importance_sampling_ratio/max": 1.3249101161956787, "sampling/importance_sampling_ratio/mean": 1.000572180747986, "sampling/importance_sampling_ratio/min": 0.7810700714588166, "sampling/sampling_logp_difference/max": 0.3053713947534561, "sampling/sampling_logp_difference/mean": 0.021317550167441367, "step": 1390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.1, "completions/max_terminated_length": 327.1, "completions/mean_length": 236.4875, "completions/mean_terminated_length": 236.4875, "completions/min_length": 161.9, "completions/min_terminated_length": 161.9, "entropy": 1.135564436763525, "epoch": 0.12871196101866322, "frac_reward_zero_std": 0.2, "grad_norm": 6.217648983001709, "kl": 0.024721204745583238, "learning_rate": 4.2861519607843135e-06, "loss": 0.0009, "num_tokens": 6107578.0, "reward": -0.225, "reward_std": 0.4943925529718399, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4943925827741623, "sampling/importance_sampling_ratio/max": 1.3563641548156737, "sampling/importance_sampling_ratio/mean": 0.9998765587806702, "sampling/importance_sampling_ratio/min": 0.7215391218662262, "sampling/sampling_logp_difference/max": 0.34402431845664977, "sampling/sampling_logp_difference/mean": 0.021769707649946214, "step": 1400 }, { "epoch": 0.12871196101866322, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0030541012216404886, "eval_completions/max_length": 321.89877835951137, "eval_completions/max_terminated_length": 319.60034904013963, "eval_completions/mean_length": 225.65728621291447, "eval_completions/mean_terminated_length": 224.96231220238704, "eval_completions/min_length": 150.32460732984293, "eval_completions/min_terminated_length": 150.32460732984293, "eval_entropy": 1.1268045640116586, "eval_frac_reward_zero_std": 0.2530541012216405, "eval_kl": 0.019346026554996273, "eval_loss": 0.0010174785275012255, "eval_num_tokens": 6107578.0, "eval_reward": -0.027923211169284468, "eval_reward_std": 0.45558283786187, "eval_rewards/reward_fn/mean": -0.027923211169284468, "eval_rewards/reward_fn/std": 0.4555828505525622, "eval_runtime": 1859.0437, "eval_samples_per_second": 0.308, "eval_sampling/importance_sampling_ratio/max": 1.298839797108169, "eval_sampling/importance_sampling_ratio/mean": 0.9999097204957333, "eval_sampling/importance_sampling_ratio/min": 0.7600866199241882, "eval_sampling/sampling_logp_difference/max": 0.3255701512878478, "eval_sampling/sampling_logp_difference/mean": 0.02160237773271183, "eval_steps_per_second": 0.039, "step": 1400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 210.75, "completions/mean_terminated_length": 210.75, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.092452796176076, "epoch": 0.12963133216879655, "frac_reward_zero_std": 0.5, "grad_norm": 5.544597625732422, "kl": 0.014508970058523118, "learning_rate": 4.316789215686275e-06, "loss": 0.0009, "num_tokens": 6151286.0, "reward": 0.0375, "reward_std": 0.4303868621587753, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.4303868681192398, "sampling/importance_sampling_ratio/max": 1.2749321103096007, "sampling/importance_sampling_ratio/mean": 1.0000078678131104, "sampling/importance_sampling_ratio/min": 0.7807251989841462, "sampling/sampling_logp_difference/max": 0.2800210028886795, "sampling/sampling_logp_difference/mean": 0.021823861822485923, "step": 1410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 217.1125, "completions/mean_terminated_length": 217.1125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.101941668987274, "epoch": 0.13055070331892984, "frac_reward_zero_std": 0.4, "grad_norm": 5.590672969818115, "kl": 0.013525198728893883, "learning_rate": 4.347426470588235e-06, "loss": 0.001, "num_tokens": 6194975.0, "reward": -0.05, "reward_std": 0.34659049212932586, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.34659049212932586, "sampling/importance_sampling_ratio/max": 1.2486390590667724, "sampling/importance_sampling_ratio/mean": 1.0002484798431397, "sampling/importance_sampling_ratio/min": 0.73974589407444, "sampling/sampling_logp_difference/max": 0.31851269006729127, "sampling/sampling_logp_difference/mean": 0.021292382664978505, "step": 1420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.3, "completions/max_terminated_length": 328.3, "completions/mean_length": 226.0625, "completions/mean_terminated_length": 226.0625, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.0521254802122713, "epoch": 0.13147007446906317, "frac_reward_zero_std": 0.0, "grad_norm": 5.299581527709961, "kl": 0.025897987411008218, "learning_rate": 4.378063725490197e-06, "loss": 0.0015, "num_tokens": 6237108.0, "reward": -0.0125, "reward_std": 0.6083779186010361, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.6083779543638229, "sampling/importance_sampling_ratio/max": 1.3657152652740479, "sampling/importance_sampling_ratio/mean": 0.9999037444591522, "sampling/importance_sampling_ratio/min": 0.7124147206544876, "sampling/sampling_logp_difference/max": 0.41489108800888064, "sampling/sampling_logp_difference/mean": 0.021135816909372808, "step": 1430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 359.9, "completions/max_terminated_length": 328.8, "completions/mean_length": 244.0625, "completions/mean_terminated_length": 238.15535888671874, "completions/min_length": 155.9, "completions/min_terminated_length": 155.9, "entropy": 1.1280117265880107, "epoch": 0.13238944561919647, "frac_reward_zero_std": 0.0, "grad_norm": 5.672699928283691, "kl": 0.02738113821251318, "learning_rate": 4.408700980392157e-06, "loss": 0.0012, "num_tokens": 6281409.0, "reward": 0.1875, "reward_std": 0.6525094747543335, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.6525094985961915, "sampling/importance_sampling_ratio/max": 1.3170461773872375, "sampling/importance_sampling_ratio/mean": 0.9996939182281495, "sampling/importance_sampling_ratio/min": 0.7692372381687165, "sampling/sampling_logp_difference/max": 0.315592485666275, "sampling/sampling_logp_difference/mean": 0.02136859782040119, "step": 1440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.2, "completions/max_terminated_length": 306.2, "completions/mean_length": 206.0, "completions/mean_terminated_length": 206.0, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.0977587856352329, "epoch": 0.1333088167693298, "frac_reward_zero_std": 0.3, "grad_norm": 6.023177623748779, "kl": 0.015292615463840776, "learning_rate": 4.439338235294118e-06, "loss": 0.0007, "num_tokens": 6323561.0, "reward": -0.1625, "reward_std": 0.42727080881595614, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.42727082669734956, "sampling/importance_sampling_ratio/max": 1.3277016520500182, "sampling/importance_sampling_ratio/mean": 1.0003354728221894, "sampling/importance_sampling_ratio/min": 0.7874673068523407, "sampling/sampling_logp_difference/max": 0.30552958250045775, "sampling/sampling_logp_difference/mean": 0.021854222379624845, "step": 1450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.2, "completions/max_terminated_length": 327.2, "completions/mean_length": 229.3, "completions/mean_terminated_length": 229.3, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.0221133209764957, "epoch": 0.1342281879194631, "frac_reward_zero_std": 0.2, "grad_norm": 0.08343011140823364, "kl": 0.017870039975969122, "learning_rate": 4.469975490196079e-06, "loss": 0.0012, "num_tokens": 6369817.0, "reward": 0.1125, "reward_std": 0.5809791445732116, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.580979174375534, "sampling/importance_sampling_ratio/max": 1.2901687741279602, "sampling/importance_sampling_ratio/mean": 1.0001559674739837, "sampling/importance_sampling_ratio/min": 0.7431910753250122, "sampling/sampling_logp_difference/max": 0.3421707391738892, "sampling/sampling_logp_difference/mean": 0.020285730436444282, "step": 1460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.3, "completions/max_terminated_length": 275.3, "completions/mean_length": 201.275, "completions/mean_terminated_length": 201.275, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0342267520725728, "epoch": 0.13514755906959638, "frac_reward_zero_std": 0.4, "grad_norm": 5.92233943939209, "kl": 0.01684942019637674, "learning_rate": 4.5006127450980395e-06, "loss": 0.0006, "num_tokens": 6410055.0, "reward": 0.225, "reward_std": 0.3355525702238083, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3355525702238083, "sampling/importance_sampling_ratio/max": 1.2795489192008973, "sampling/importance_sampling_ratio/mean": 0.9997927188873291, "sampling/importance_sampling_ratio/min": 0.7806328356266021, "sampling/sampling_logp_difference/max": 0.28694812059402464, "sampling/sampling_logp_difference/mean": 0.020654335059225558, "step": 1470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.0, "completions/max_terminated_length": 269.0, "completions/mean_length": 199.2375, "completions/mean_terminated_length": 199.2375, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 0.9882201224565506, "epoch": 0.1360669302197297, "frac_reward_zero_std": 0.2, "grad_norm": 8.399575233459473, "kl": 0.026593866568873636, "learning_rate": 4.53125e-06, "loss": 0.0013, "num_tokens": 6452914.0, "reward": 0.2375, "reward_std": 0.4142331928014755, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.41423319876194, "sampling/importance_sampling_ratio/max": 1.2976352095603942, "sampling/importance_sampling_ratio/mean": 1.0002907395362854, "sampling/importance_sampling_ratio/min": 0.7600722372531891, "sampling/sampling_logp_difference/max": 0.32167547941207886, "sampling/sampling_logp_difference/mean": 0.020902066119015218, "step": 1480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 203.225, "completions/mean_terminated_length": 203.225, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0744589954614638, "epoch": 0.136986301369863, "frac_reward_zero_std": 0.1, "grad_norm": 8.661561012268066, "kl": 0.03353368609678, "learning_rate": 4.561887254901961e-06, "loss": 0.0015, "num_tokens": 6495732.0, "reward": -0.1125, "reward_std": 0.5512929946184159, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.5512930184602738, "sampling/importance_sampling_ratio/max": 1.3335409760475159, "sampling/importance_sampling_ratio/mean": 0.9998663485050201, "sampling/importance_sampling_ratio/min": 0.7325195729732513, "sampling/sampling_logp_difference/max": 0.4317288875579834, "sampling/sampling_logp_difference/mean": 0.021174842864274977, "step": 1490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.4, "completions/max_terminated_length": 281.4, "completions/mean_length": 199.4875, "completions/mean_terminated_length": 199.4875, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0627880647778511, "epoch": 0.13790567251999633, "frac_reward_zero_std": 0.3, "grad_norm": 0.16991867125034332, "kl": 0.022514234139816836, "learning_rate": 4.592524509803922e-06, "loss": 0.0011, "num_tokens": 6542563.0, "reward": -0.2875, "reward_std": 0.41911540925502777, "rewards/reward_fn/mean": -0.2875, "rewards/reward_fn/std": 0.4191154152154922, "sampling/importance_sampling_ratio/max": 1.3318233966827393, "sampling/importance_sampling_ratio/mean": 1.000492697954178, "sampling/importance_sampling_ratio/min": 0.755594152212143, "sampling/sampling_logp_difference/max": 0.3407777100801468, "sampling/sampling_logp_difference/mean": 0.02217220440506935, "step": 1500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.8, "completions/max_terminated_length": 253.8, "completions/mean_length": 181.0125, "completions/mean_terminated_length": 181.0125, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.0705828636884689, "epoch": 0.13882504367012963, "frac_reward_zero_std": 0.2, "grad_norm": 6.78505802154541, "kl": 0.029351890063844623, "learning_rate": 4.623161764705883e-06, "loss": 0.0017, "num_tokens": 6587300.0, "reward": -0.1875, "reward_std": 0.44087620079517365, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.44087620079517365, "sampling/importance_sampling_ratio/max": 1.3288820385932922, "sampling/importance_sampling_ratio/mean": 1.0002204418182372, "sampling/importance_sampling_ratio/min": 0.7853626430034637, "sampling/sampling_logp_difference/max": 0.3062667608261108, "sampling/sampling_logp_difference/mean": 0.02165432758629322, "step": 1510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 237.7, "completions/max_terminated_length": 237.7, "completions/mean_length": 182.9375, "completions/mean_terminated_length": 182.9375, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.050403956323862, "epoch": 0.13974441482026295, "frac_reward_zero_std": 0.2, "grad_norm": 6.823770999908447, "kl": 0.032556075253523886, "learning_rate": 4.653799019607844e-06, "loss": 0.0016, "num_tokens": 6629303.0, "reward": 0.1, "reward_std": 0.4424272894859314, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.44242729544639586, "sampling/importance_sampling_ratio/max": 1.4264759182929994, "sampling/importance_sampling_ratio/mean": 0.9998594403266907, "sampling/importance_sampling_ratio/min": 0.7270286232233047, "sampling/sampling_logp_difference/max": 0.3886754333972931, "sampling/sampling_logp_difference/mean": 0.021766609512269497, "step": 1520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 203.5875, "completions/mean_terminated_length": 203.5875, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.0652003578841687, "epoch": 0.14066378597039625, "frac_reward_zero_std": 0.2, "grad_norm": 15.634310722351074, "kl": 0.027967221062863246, "learning_rate": 4.684436274509804e-06, "loss": 0.0012, "num_tokens": 6672702.0, "reward": -0.2375, "reward_std": 0.3518119215965271, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.3518119215965271, "sampling/importance_sampling_ratio/max": 1.2514676809310914, "sampling/importance_sampling_ratio/mean": 1.0001472532749176, "sampling/importance_sampling_ratio/min": 0.8009488821029663, "sampling/sampling_logp_difference/max": 0.23702155351638793, "sampling/sampling_logp_difference/mean": 0.021141395904123783, "step": 1530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.9, "completions/max_terminated_length": 324.9, "completions/mean_length": 228.2875, "completions/mean_terminated_length": 225.59285888671874, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.1336680881679058, "epoch": 0.14158315712052955, "frac_reward_zero_std": 0.1, "grad_norm": 6.894134998321533, "kl": 0.0194727988622617, "learning_rate": 4.715073529411765e-06, "loss": 0.0015, "num_tokens": 6721093.0, "reward": 0.20625, "reward_std": 0.6140542179346085, "rewards/reward_fn/mean": 0.20625, "rewards/reward_fn/std": 0.6140542298555374, "sampling/importance_sampling_ratio/max": 1.303449809551239, "sampling/importance_sampling_ratio/mean": 1.0005681693553925, "sampling/importance_sampling_ratio/min": 0.7986957371234894, "sampling/sampling_logp_difference/max": 0.26659847497940065, "sampling/sampling_logp_difference/mean": 0.022179079987108708, "step": 1540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.4, "completions/max_terminated_length": 258.4, "completions/mean_length": 190.85, "completions/mean_terminated_length": 190.85, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.069445087760687, "epoch": 0.14250252827066287, "frac_reward_zero_std": 0.4, "grad_norm": 7.343962669372559, "kl": 0.020104502828326078, "learning_rate": 4.745710784313726e-06, "loss": 0.001, "num_tokens": 6762465.0, "reward": 0.0, "reward_std": 0.46734622716903684, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.46734623312950135, "sampling/importance_sampling_ratio/max": 1.3031768798828125, "sampling/importance_sampling_ratio/mean": 0.9997886836528778, "sampling/importance_sampling_ratio/min": 0.7159755289554596, "sampling/sampling_logp_difference/max": 0.35821783542633057, "sampling/sampling_logp_difference/mean": 0.02124423012137413, "step": 1550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.3, "completions/max_terminated_length": 290.3, "completions/mean_length": 202.2875, "completions/mean_terminated_length": 202.2875, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.1350328363478184, "epoch": 0.14342189942079617, "frac_reward_zero_std": 0.2, "grad_norm": 6.09912633895874, "kl": 0.014210601139348, "learning_rate": 4.7763480392156866e-06, "loss": 0.0008, "num_tokens": 6806336.0, "reward": -0.35, "reward_std": 0.5423088639974594, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5423088699579239, "sampling/importance_sampling_ratio/max": 1.2767892241477967, "sampling/importance_sampling_ratio/mean": 1.0004692196846008, "sampling/importance_sampling_ratio/min": 0.7695710301399231, "sampling/sampling_logp_difference/max": 0.27397539615631106, "sampling/sampling_logp_difference/mean": 0.02238109279423952, "step": 1560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.0, "completions/max_terminated_length": 243.0, "completions/mean_length": 174.2625, "completions/mean_terminated_length": 174.2625, "completions/min_length": 114.8, "completions/min_terminated_length": 114.8, "entropy": 1.1513236083090306, "epoch": 0.1443412705709295, "frac_reward_zero_std": 0.3, "grad_norm": 6.1542863845825195, "kl": 0.015671954827848823, "learning_rate": 4.806985294117648e-06, "loss": 0.0006, "num_tokens": 6847109.0, "reward": 0.3375, "reward_std": 0.5408749014139176, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.54087493121624, "sampling/importance_sampling_ratio/max": 1.225877845287323, "sampling/importance_sampling_ratio/mean": 0.9997000813484191, "sampling/importance_sampling_ratio/min": 0.7958498239517212, "sampling/sampling_logp_difference/max": 0.2364953875541687, "sampling/sampling_logp_difference/mean": 0.022214539349079132, "step": 1570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.6, "completions/max_terminated_length": 243.6, "completions/mean_length": 163.0625, "completions/mean_terminated_length": 163.0625, "completions/min_length": 106.9, "completions/min_terminated_length": 106.9, "entropy": 1.007540236413479, "epoch": 0.1452606417210628, "frac_reward_zero_std": 0.2, "grad_norm": 6.1776862144470215, "kl": 0.024297343887155876, "learning_rate": 4.837622549019608e-06, "loss": 0.0014, "num_tokens": 6884130.0, "reward": 0.325, "reward_std": 0.42642940282821656, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.42642942070961, "sampling/importance_sampling_ratio/max": 1.2720715641975402, "sampling/importance_sampling_ratio/mean": 0.9998812735080719, "sampling/importance_sampling_ratio/min": 0.7560014605522156, "sampling/sampling_logp_difference/max": 0.2988255798816681, "sampling/sampling_logp_difference/mean": 0.02160265054553747, "step": 1580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 231.2, "completions/max_terminated_length": 231.2, "completions/mean_length": 165.8875, "completions/mean_terminated_length": 165.8875, "completions/min_length": 106.7, "completions/min_terminated_length": 106.7, "entropy": 1.0177243258804083, "epoch": 0.1461800128711961, "frac_reward_zero_std": 0.6, "grad_norm": 0.14829987287521362, "kl": 0.01376922951603774, "learning_rate": 4.868259803921569e-06, "loss": 0.0006, "num_tokens": 6922569.0, "reward": 0.0625, "reward_std": 0.24237500131130219, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.2423750251531601, "sampling/importance_sampling_ratio/max": 1.235771894454956, "sampling/importance_sampling_ratio/mean": 1.000098603963852, "sampling/importance_sampling_ratio/min": 0.7932318806648254, "sampling/sampling_logp_difference/max": 0.24366546869277955, "sampling/sampling_logp_difference/mean": 0.02123898882418871, "step": 1590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.4, "completions/max_terminated_length": 254.4, "completions/mean_length": 190.575, "completions/mean_terminated_length": 190.575, "completions/min_length": 126.2, "completions/min_terminated_length": 126.2, "entropy": 1.1313216343522072, "epoch": 0.1470993840213294, "frac_reward_zero_std": 0.2, "grad_norm": 6.902104377746582, "kl": 0.018426794398692437, "learning_rate": 4.898897058823529e-06, "loss": 0.0007, "num_tokens": 6962631.0, "reward": -0.225, "reward_std": 0.47630349099636077, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4763035088777542, "sampling/importance_sampling_ratio/max": 1.3305252194404602, "sampling/importance_sampling_ratio/mean": 0.9998793840408325, "sampling/importance_sampling_ratio/min": 0.7743998110294342, "sampling/sampling_logp_difference/max": 0.3400513678789139, "sampling/sampling_logp_difference/mean": 0.02234724797308445, "step": 1600 }, { "epoch": 0.1470993840213294, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 295.5130890052356, "eval_completions/max_terminated_length": 294.8726003490401, "eval_completions/mean_length": 207.77748691099475, "eval_completions/mean_terminated_length": 207.45817768303303, "eval_completions/min_length": 136.47993019197207, "eval_completions/min_terminated_length": 136.47993019197207, "eval_entropy": 1.1100842952728271, "eval_frac_reward_zero_std": 0.2914485165794066, "eval_kl": 0.017346000372810876, "eval_loss": 0.0008573353989049792, "eval_num_tokens": 6962631.0, "eval_reward": -0.009925828970331589, "eval_reward_std": 0.44791377201858407, "eval_rewards/reward_fn/mean": -0.009925828970331589, "eval_rewards/reward_fn/std": 0.4479137832529674, "eval_runtime": 1726.0089, "eval_samples_per_second": 0.332, "eval_sampling/importance_sampling_ratio/max": 1.2872572171958536, "eval_sampling/importance_sampling_ratio/mean": 1.0000051697927412, "eval_sampling/importance_sampling_ratio/min": 0.7703749284174222, "eval_sampling/sampling_logp_difference/max": 0.3057308903851434, "eval_sampling/sampling_logp_difference/mean": 0.02126904747515501, "eval_steps_per_second": 0.042, "step": 1600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.6, "completions/max_terminated_length": 319.6, "completions/mean_length": 225.15, "completions/mean_terminated_length": 225.15, "completions/min_length": 159.6, "completions/min_terminated_length": 159.6, "entropy": 1.1622169733047485, "epoch": 0.1480187551714627, "frac_reward_zero_std": 0.1, "grad_norm": 4.971400260925293, "kl": 0.022326471004635095, "learning_rate": 4.929534313725491e-06, "loss": 0.001, "num_tokens": 7006403.0, "reward": -0.1375, "reward_std": 0.5463842064142227, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.5463842123746872, "sampling/importance_sampling_ratio/max": 1.3966349363327026, "sampling/importance_sampling_ratio/mean": 1.0003197193145752, "sampling/importance_sampling_ratio/min": 0.7974103271961213, "sampling/sampling_logp_difference/max": 0.3287259519100189, "sampling/sampling_logp_difference/mean": 0.022303739003837107, "step": 1610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 209.2625, "completions/mean_terminated_length": 209.2625, "completions/min_length": 141.8, "completions/min_terminated_length": 141.8, "entropy": 1.1289937548339366, "epoch": 0.14893812632159603, "frac_reward_zero_std": 0.2, "grad_norm": 0.15305055677890778, "kl": 0.025693328343913892, "learning_rate": 4.960171568627451e-06, "loss": 0.0015, "num_tokens": 7050240.0, "reward": 0.09375, "reward_std": 0.4124096900224686, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.41240971386432645, "sampling/importance_sampling_ratio/max": 1.375335943698883, "sampling/importance_sampling_ratio/mean": 1.0000707685947419, "sampling/importance_sampling_ratio/min": 0.6895323842763901, "sampling/sampling_logp_difference/max": 0.4378436326980591, "sampling/sampling_logp_difference/mean": 0.021447959914803506, "step": 1620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 366.7, "completions/max_terminated_length": 361.2, "completions/mean_length": 236.7375, "completions/mean_terminated_length": 233.58392944335938, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.1081803828477859, "epoch": 0.14985749747172933, "frac_reward_zero_std": 0.2, "grad_norm": 4.14241886138916, "kl": 0.015311846393160522, "learning_rate": 4.990808823529413e-06, "loss": 0.0009, "num_tokens": 7095291.0, "reward": -0.15, "reward_std": 0.5733177453279495, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.5733177691698075, "sampling/importance_sampling_ratio/max": 1.2535099864006043, "sampling/importance_sampling_ratio/mean": 1.000777542591095, "sampling/importance_sampling_ratio/min": 0.7887921273708344, "sampling/sampling_logp_difference/max": 0.2520232915878296, "sampling/sampling_logp_difference/mean": 0.021039466373622417, "step": 1630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 196.2, "completions/mean_terminated_length": 196.2, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.1308213829994203, "epoch": 0.15077686862186265, "frac_reward_zero_std": 0.3, "grad_norm": 0.11484629660844803, "kl": 0.019536224391777067, "learning_rate": 4.9999993709133756e-06, "loss": 0.0008, "num_tokens": 7135819.0, "reward": -0.125, "reward_std": 0.4179248958826065, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.4179249078035355, "sampling/importance_sampling_ratio/max": 1.2677364349365234, "sampling/importance_sampling_ratio/mean": 1.0001580357551574, "sampling/importance_sampling_ratio/min": 0.773081511259079, "sampling/sampling_logp_difference/max": 0.2969019651412964, "sampling/sampling_logp_difference/mean": 0.02234613448381424, "step": 1640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.5, "completions/max_terminated_length": 272.5, "completions/mean_length": 191.1375, "completions/mean_terminated_length": 191.1375, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.0719189211726188, "epoch": 0.15169623977199595, "frac_reward_zero_std": 0.3, "grad_norm": 0.09037373214960098, "kl": 0.022338721586856992, "learning_rate": 4.999996289673523e-06, "loss": 0.0009, "num_tokens": 7174798.0, "reward": 0.325, "reward_std": 0.4644810765981674, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.46448108851909636, "sampling/importance_sampling_ratio/max": 1.3307189226150513, "sampling/importance_sampling_ratio/mean": 1.0003674447536468, "sampling/importance_sampling_ratio/min": 0.7516764968633651, "sampling/sampling_logp_difference/max": 0.37522636651992797, "sampling/sampling_logp_difference/mean": 0.02092442624270916, "step": 1650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.5, "completions/max_terminated_length": 273.5, "completions/mean_length": 196.6, "completions/mean_terminated_length": 196.6, "completions/min_length": 114.3, "completions/min_terminated_length": 114.3, "entropy": 1.0820644252002238, "epoch": 0.15261561092212927, "frac_reward_zero_std": 0.4, "grad_norm": 5.1205830574035645, "kl": 0.018708533188328148, "learning_rate": 4.9999906407370825e-06, "loss": 0.0008, "num_tokens": 7216278.0, "reward": -0.2375, "reward_std": 0.39638522267341614, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.3963852345943451, "sampling/importance_sampling_ratio/max": 1.4229997515678405, "sampling/importance_sampling_ratio/mean": 0.9998858988285064, "sampling/importance_sampling_ratio/min": 0.758237111568451, "sampling/sampling_logp_difference/max": 0.39238523244857787, "sampling/sampling_logp_difference/mean": 0.02201388143002987, "step": 1660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.9, "completions/max_terminated_length": 310.4, "completions/mean_length": 217.075, "completions/mean_terminated_length": 213.76250152587892, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.0664160154759883, "epoch": 0.15353498207226257, "frac_reward_zero_std": 0.1, "grad_norm": 7.3463592529296875, "kl": 0.018984398437896743, "learning_rate": 4.999982424109855e-06, "loss": 0.0001, "num_tokens": 7257796.0, "reward": -0.14375, "reward_std": 0.4889191150665283, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.4889191150665283, "sampling/importance_sampling_ratio/max": 1.2739620685577393, "sampling/importance_sampling_ratio/mean": 0.9997388124465942, "sampling/importance_sampling_ratio/min": 0.7940850377082824, "sampling/sampling_logp_difference/max": 0.24902696609497071, "sampling/sampling_logp_difference/mean": 0.021570471115410328, "step": 1670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 352.3, "completions/max_terminated_length": 352.3, "completions/mean_length": 236.175, "completions/mean_terminated_length": 236.175, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.0556964889168738, "epoch": 0.15445435322239587, "frac_reward_zero_std": 0.3, "grad_norm": 4.638572692871094, "kl": 0.03635619452106766, "learning_rate": 4.999971639800281e-06, "loss": 0.0015, "num_tokens": 7304154.0, "reward": 0.18125, "reward_std": 0.45312740206718444, "rewards/reward_fn/mean": 0.18125, "rewards/reward_fn/std": 0.45312740802764895, "sampling/importance_sampling_ratio/max": 1.2885913491249084, "sampling/importance_sampling_ratio/mean": 0.999603909254074, "sampling/importance_sampling_ratio/min": 0.7212844133377075, "sampling/sampling_logp_difference/max": 0.36110965013504026, "sampling/sampling_logp_difference/mean": 0.02110386434942484, "step": 1680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.9, "completions/max_terminated_length": 281.9, "completions/mean_length": 210.8, "completions/mean_terminated_length": 210.8, "completions/min_length": 156.5, "completions/min_terminated_length": 156.5, "entropy": 1.1038669049739838, "epoch": 0.1553737243725292, "frac_reward_zero_std": 0.3, "grad_norm": 0.09199784696102142, "kl": 0.032090726867318155, "learning_rate": 4.9999582878194345e-06, "loss": 0.0018, "num_tokens": 7348554.0, "reward": 0.15, "reward_std": 0.4254381716251373, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.4254381775856018, "sampling/importance_sampling_ratio/max": 1.2427798151969909, "sampling/importance_sampling_ratio/mean": 1.000288063287735, "sampling/importance_sampling_ratio/min": 0.7444603860378265, "sampling/sampling_logp_difference/max": 0.3002180427312851, "sampling/sampling_logp_difference/mean": 0.021241777949035166, "step": 1690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.6, "completions/max_terminated_length": 284.6, "completions/mean_length": 199.9625, "completions/mean_terminated_length": 199.9625, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.1785475850105285, "epoch": 0.1562930955226625, "frac_reward_zero_std": 0.4, "grad_norm": 6.7228684425354, "kl": 0.04097896607709117, "learning_rate": 4.99994236818103e-06, "loss": 0.0017, "num_tokens": 7388871.0, "reward": -0.14375, "reward_std": 0.2651960164308548, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.2651960164308548, "sampling/importance_sampling_ratio/max": 1.2273439764976501, "sampling/importance_sampling_ratio/mean": 0.9998808383941651, "sampling/importance_sampling_ratio/min": 0.7310256153345108, "sampling/sampling_logp_difference/max": 0.33525035381317136, "sampling/sampling_logp_difference/mean": 0.022543698735535144, "step": 1700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.1, "completions/max_terminated_length": 299.1, "completions/mean_length": 210.7375, "completions/mean_terminated_length": 210.7375, "completions/min_length": 151.5, "completions/min_terminated_length": 151.5, "entropy": 1.0985402591526507, "epoch": 0.15721246667279581, "frac_reward_zero_std": 0.5, "grad_norm": 6.276334762573242, "kl": 0.0512921616260428, "learning_rate": 4.999923880901419e-06, "loss": 0.0026, "num_tokens": 7435130.0, "reward": -0.0375, "reward_std": 0.2757775604724884, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.2757775604724884, "sampling/importance_sampling_ratio/max": 1.3340021610260009, "sampling/importance_sampling_ratio/mean": 1.000907826423645, "sampling/importance_sampling_ratio/min": 0.7723042488098144, "sampling/sampling_logp_difference/max": 0.31889861822128296, "sampling/sampling_logp_difference/mean": 0.021154037117958067, "step": 1710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.2, "completions/max_terminated_length": 308.2, "completions/mean_length": 211.4125, "completions/mean_terminated_length": 211.4125, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.0608094729483128, "epoch": 0.1581318378229291, "frac_reward_zero_std": 0.4, "grad_norm": 0.1368068903684616, "kl": 0.03478015947039239, "learning_rate": 4.999902825999588e-06, "loss": 0.0015, "num_tokens": 7481155.0, "reward": 0.2625, "reward_std": 0.2917597532272339, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.2917597651481628, "sampling/importance_sampling_ratio/max": 1.3059454321861268, "sampling/importance_sampling_ratio/mean": 1.0001837849617004, "sampling/importance_sampling_ratio/min": 0.7194568514823914, "sampling/sampling_logp_difference/max": 0.4368997931480408, "sampling/sampling_logp_difference/mean": 0.02122602015733719, "step": 1720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 206.5875, "completions/mean_terminated_length": 206.5875, "completions/min_length": 131.1, "completions/min_terminated_length": 131.1, "entropy": 1.0386747267097234, "epoch": 0.15905120897306244, "frac_reward_zero_std": 0.4, "grad_norm": 5.215572357177734, "kl": 0.04280963434721343, "learning_rate": 4.9998792034971645e-06, "loss": 0.002, "num_tokens": 7524506.0, "reward": 0.3, "reward_std": 0.3547274947166443, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.3547275006771088, "sampling/importance_sampling_ratio/max": 1.281489098072052, "sampling/importance_sampling_ratio/mean": 0.9999694466590882, "sampling/importance_sampling_ratio/min": 0.7720149755477905, "sampling/sampling_logp_difference/max": 0.2822927713394165, "sampling/sampling_logp_difference/mean": 0.021051739901304246, "step": 1730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 212.0625, "completions/mean_terminated_length": 212.0625, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.1329169027507304, "epoch": 0.15997058012319573, "frac_reward_zero_std": 0.2, "grad_norm": 6.036554336547852, "kl": 0.033280878106597814, "learning_rate": 4.999853013418409e-06, "loss": 0.0019, "num_tokens": 7571895.0, "reward": -0.3375, "reward_std": 0.47455789148807526, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.4745579093694687, "sampling/importance_sampling_ratio/max": 1.2642733454704285, "sampling/importance_sampling_ratio/mean": 1.000291782617569, "sampling/importance_sampling_ratio/min": 0.7963159680366516, "sampling/sampling_logp_difference/max": 0.24890742301940919, "sampling/sampling_logp_difference/mean": 0.02223680019378662, "step": 1740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.1, "completions/max_terminated_length": 268.1, "completions/mean_length": 187.6125, "completions/mean_terminated_length": 187.6125, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0342995330691338, "epoch": 0.16088995127332903, "frac_reward_zero_std": 0.2, "grad_norm": 5.907735824584961, "kl": 0.03509269545902498, "learning_rate": 4.99982425579022e-06, "loss": 0.0021, "num_tokens": 7612040.0, "reward": -0.0375, "reward_std": 0.4307587772607803, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.4307588070631027, "sampling/importance_sampling_ratio/max": 1.2999369382858277, "sampling/importance_sampling_ratio/mean": 1.0000859439373015, "sampling/importance_sampling_ratio/min": 0.7732944309711456, "sampling/sampling_logp_difference/max": 0.28751913607120516, "sampling/sampling_logp_difference/mean": 0.021382975578308105, "step": 1750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 210.825, "completions/mean_terminated_length": 210.825, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.1162607789039611, "epoch": 0.16180932242346235, "frac_reward_zero_std": 0.1, "grad_norm": 5.0578179359436035, "kl": 0.026344594033434986, "learning_rate": 4.999792930642135e-06, "loss": 0.0007, "num_tokens": 7654626.0, "reward": 0.025, "reward_std": 0.4763862133026123, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.47638622522354124, "sampling/importance_sampling_ratio/max": 1.286192572116852, "sampling/importance_sampling_ratio/mean": 1.0006832718849181, "sampling/importance_sampling_ratio/min": 0.7646889507770538, "sampling/sampling_logp_difference/max": 0.3069689214229584, "sampling/sampling_logp_difference/mean": 0.02153454050421715, "step": 1760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 348.0, "completions/max_terminated_length": 323.8, "completions/mean_length": 227.3375, "completions/mean_terminated_length": 223.60535736083983, "completions/min_length": 145.3, "completions/min_terminated_length": 145.3, "entropy": 1.1258797220885755, "epoch": 0.16272869357359565, "frac_reward_zero_std": 0.4, "grad_norm": 7.659244537353516, "kl": 0.021348016592673956, "learning_rate": 4.999759038006328e-06, "loss": 0.0006, "num_tokens": 7701701.0, "reward": 0.15625, "reward_std": 0.41403928101062776, "rewards/reward_fn/mean": 0.15625, "rewards/reward_fn/std": 0.4140392869710922, "sampling/importance_sampling_ratio/max": 1.3567353725433349, "sampling/importance_sampling_ratio/mean": 0.9997352540493012, "sampling/importance_sampling_ratio/min": 0.7304510235786438, "sampling/sampling_logp_difference/max": 0.3939854145050049, "sampling/sampling_logp_difference/mean": 0.021448280289769174, "step": 1770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.4, "completions/max_terminated_length": 252.4, "completions/mean_length": 187.225, "completions/mean_terminated_length": 187.225, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.1231333188712598, "epoch": 0.16364806472372898, "frac_reward_zero_std": 0.3, "grad_norm": 6.789471626281738, "kl": 0.015499754762277006, "learning_rate": 4.9997225779176075e-06, "loss": 0.0011, "num_tokens": 7744951.0, "reward": 0.2, "reward_std": 0.36909410953521726, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.36909412145614623, "sampling/importance_sampling_ratio/max": 1.2488541722297668, "sampling/importance_sampling_ratio/mean": 1.0003271102905273, "sampling/importance_sampling_ratio/min": 0.6873427987098694, "sampling/sampling_logp_difference/max": 0.45946853756904604, "sampling/sampling_logp_difference/mean": 0.0220858970656991, "step": 1780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.1, "completions/max_terminated_length": 253.1, "completions/mean_length": 187.0375, "completions/mean_terminated_length": 187.0375, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.0134456224739552, "epoch": 0.16456743587386227, "frac_reward_zero_std": 0.2, "grad_norm": 6.742940425872803, "kl": 0.020215615181950853, "learning_rate": 4.999683550413424e-06, "loss": 0.0006, "num_tokens": 7783418.0, "reward": 0.29375, "reward_std": 0.6668038606643677, "rewards/reward_fn/mean": 0.29375, "rewards/reward_fn/std": 0.6668038606643677, "sampling/importance_sampling_ratio/max": 1.299957811832428, "sampling/importance_sampling_ratio/mean": 0.9998450696468353, "sampling/importance_sampling_ratio/min": 0.7644662320613861, "sampling/sampling_logp_difference/max": 0.3087465226650238, "sampling/sampling_logp_difference/mean": 0.02149232681840658, "step": 1790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 298.3, "completions/max_terminated_length": 296.8, "completions/mean_length": 203.375, "completions/mean_terminated_length": 200.50357360839843, "completions/min_length": 117.8, "completions/min_terminated_length": 117.8, "entropy": 1.0240340236574412, "epoch": 0.1654868070239956, "frac_reward_zero_std": 0.5, "grad_norm": 0.15658947825431824, "kl": 0.01568760203081183, "learning_rate": 4.999641955533858e-06, "loss": 0.0002, "num_tokens": 7829024.0, "reward": -0.26875, "reward_std": 0.20908173173666, "rewards/reward_fn/mean": -0.26875, "rewards/reward_fn/std": 0.2090817376971245, "sampling/importance_sampling_ratio/max": 1.3214632987976074, "sampling/importance_sampling_ratio/mean": 0.9997395277023315, "sampling/importance_sampling_ratio/min": 0.7777888774871826, "sampling/sampling_logp_difference/max": 0.3096400260925293, "sampling/sampling_logp_difference/mean": 0.020993327908217906, "step": 1800 }, { "epoch": 0.1654868070239956, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 268.956369982548, "eval_completions/max_terminated_length": 267.6928446771379, "eval_completions/mean_length": 189.07089877835952, "eval_completions/mean_terminated_length": 188.78922550008352, "eval_completions/min_length": 121.76788830715532, "eval_completions/min_terminated_length": 121.76788830715532, "eval_entropy": 1.0206009637414888, "eval_frac_reward_zero_std": 0.33507853403141363, "eval_kl": 0.016771220259147374, "eval_loss": 0.0008007879951037467, "eval_num_tokens": 7829024.0, "eval_reward": -0.033595113438045374, "eval_reward_std": 0.4418367014557487, "eval_rewards/reward_fn/mean": -0.033595113438045374, "eval_rewards/reward_fn/std": 0.441836712690132, "eval_runtime": 1591.0901, "eval_samples_per_second": 0.36, "eval_sampling/importance_sampling_ratio/max": 1.2806214154494786, "eval_sampling/importance_sampling_ratio/mean": 1.0000317281869486, "eval_sampling/importance_sampling_ratio/min": 0.7608642992912995, "eval_sampling/sampling_logp_difference/max": 0.30833629657876843, "eval_sampling/sampling_logp_difference/mean": 0.021213743801555395, "eval_steps_per_second": 0.045, "step": 1800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 220.1625, "completions/mean_terminated_length": 220.1625, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1191354081034661, "epoch": 0.1664061781741289, "frac_reward_zero_std": 0.1, "grad_norm": 5.302118301391602, "kl": 0.019197515666019172, "learning_rate": 4.999597793321635e-06, "loss": 0.0011, "num_tokens": 7875549.0, "reward": -0.35, "reward_std": 0.5776334583759308, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.5776334643363953, "sampling/importance_sampling_ratio/max": 1.2857922315597534, "sampling/importance_sampling_ratio/mean": 1.0001758277416228, "sampling/importance_sampling_ratio/min": 0.7315779685974121, "sampling/sampling_logp_difference/max": 0.3537005364894867, "sampling/sampling_logp_difference/mean": 0.02154194638133049, "step": 1810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.4, "completions/max_terminated_length": 276.4, "completions/mean_length": 202.7875, "completions/mean_terminated_length": 202.7875, "completions/min_length": 141.4, "completions/min_terminated_length": 141.4, "entropy": 0.9648481421172619, "epoch": 0.16732554932426222, "frac_reward_zero_std": 0.1, "grad_norm": 6.462383270263672, "kl": 0.020946410379838198, "learning_rate": 4.9995510638221086e-06, "loss": 0.0015, "num_tokens": 7920572.0, "reward": -0.475, "reward_std": 0.5935844153165817, "rewards/reward_fn/mean": -0.475, "rewards/reward_fn/std": 0.5935844212770462, "sampling/importance_sampling_ratio/max": 1.3535225510597229, "sampling/importance_sampling_ratio/mean": 1.000580257177353, "sampling/importance_sampling_ratio/min": 0.7677641868591308, "sampling/sampling_logp_difference/max": 0.34211663603782655, "sampling/sampling_logp_difference/mean": 0.021085841208696367, "step": 1820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 212.3, "completions/mean_terminated_length": 212.3, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0180262207984925, "epoch": 0.16824492047439552, "frac_reward_zero_std": 0.2, "grad_norm": 5.7959442138671875, "kl": 0.030683148233219982, "learning_rate": 4.999501767083278e-06, "loss": 0.0018, "num_tokens": 7965572.0, "reward": -0.0375, "reward_std": 0.5805714815855026, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.5805714935064316, "sampling/importance_sampling_ratio/max": 1.3044957280158997, "sampling/importance_sampling_ratio/mean": 0.9997461616992951, "sampling/importance_sampling_ratio/min": 0.7525037109851838, "sampling/sampling_logp_difference/max": 0.3274673104286194, "sampling/sampling_logp_difference/mean": 0.02111899834126234, "step": 1830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 300.6, "completions/max_terminated_length": 279.2, "completions/mean_length": 224.1125, "completions/mean_terminated_length": 220.81964416503905, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.122595849633217, "epoch": 0.1691642916245288, "frac_reward_zero_std": 0.0, "grad_norm": 4.7362141609191895, "kl": 0.02047282544663176, "learning_rate": 4.999449903155772e-06, "loss": 0.0008, "num_tokens": 8009557.0, "reward": -0.1625, "reward_std": 0.7461822956800461, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.7461823433637619, "sampling/importance_sampling_ratio/max": 1.3128098130226136, "sampling/importance_sampling_ratio/mean": 1.0002018332481384, "sampling/importance_sampling_ratio/min": 0.7588152348995209, "sampling/sampling_logp_difference/max": 0.30289416909217837, "sampling/sampling_logp_difference/mean": 0.022252458706498147, "step": 1840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.3, "completions/max_terminated_length": 315.3, "completions/mean_length": 219.6125, "completions/mean_terminated_length": 219.6125, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.1157530531287194, "epoch": 0.17008366277466214, "frac_reward_zero_std": 0.4, "grad_norm": 0.11803943663835526, "kl": 0.02386214191792533, "learning_rate": 4.999395472092861e-06, "loss": 0.0014, "num_tokens": 8053110.0, "reward": -0.55, "reward_std": 0.41486670076847076, "rewards/reward_fn/mean": -0.55, "rewards/reward_fn/std": 0.4148667067289352, "sampling/importance_sampling_ratio/max": 1.3096130967140198, "sampling/importance_sampling_ratio/mean": 0.9998457193374634, "sampling/importance_sampling_ratio/min": 0.7483814895153046, "sampling/sampling_logp_difference/max": 0.3229205310344696, "sampling/sampling_logp_difference/mean": 0.02229196336120367, "step": 1850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 211.675, "completions/mean_terminated_length": 211.675, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 0.9884705692529678, "epoch": 0.17100303392479543, "frac_reward_zero_std": 0.2, "grad_norm": 5.273815155029297, "kl": 0.029997169808484613, "learning_rate": 4.999338473950448e-06, "loss": 0.0016, "num_tokens": 8092468.0, "reward": -0.1125, "reward_std": 0.4584876537322998, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.45848766565322874, "sampling/importance_sampling_ratio/max": 1.2812983393669128, "sampling/importance_sampling_ratio/mean": 1.000443696975708, "sampling/importance_sampling_ratio/min": 0.7582194089889527, "sampling/sampling_logp_difference/max": 0.302225387096405, "sampling/sampling_logp_difference/mean": 0.02075035348534584, "step": 1860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 360.2, "completions/max_terminated_length": 357.7, "completions/mean_length": 240.2875, "completions/mean_terminated_length": 237.825, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.1315513290464878, "epoch": 0.17192240507492876, "frac_reward_zero_std": 0.2, "grad_norm": 4.1036810874938965, "kl": 0.014520496840123086, "learning_rate": 4.9992789087870764e-06, "loss": 0.0006, "num_tokens": 8137827.0, "reward": 0.14375, "reward_std": 0.5457315862178802, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.5457316160202026, "sampling/importance_sampling_ratio/max": 1.2495078206062318, "sampling/importance_sampling_ratio/mean": 0.9996249377727509, "sampling/importance_sampling_ratio/min": 0.7946069240570068, "sampling/sampling_logp_difference/max": 0.24551107883453369, "sampling/sampling_logp_difference/mean": 0.021386440843343735, "step": 1870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.2, "completions/max_terminated_length": 269.2, "completions/mean_length": 196.55, "completions/mean_terminated_length": 196.55, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.038776909559965, "epoch": 0.17284177622506205, "frac_reward_zero_std": 0.4, "grad_norm": 0.1355523020029068, "kl": 0.018370905792107806, "learning_rate": 4.999216776663924e-06, "loss": 0.0011, "num_tokens": 8175687.0, "reward": 0.525, "reward_std": 0.39513265490531924, "rewards/reward_fn/mean": 0.525, "rewards/reward_fn/std": 0.3951326608657837, "sampling/importance_sampling_ratio/max": 1.3396273612976075, "sampling/importance_sampling_ratio/mean": 1.0000875353813172, "sampling/importance_sampling_ratio/min": 0.7652070045471191, "sampling/sampling_logp_difference/max": 0.3233711004257202, "sampling/sampling_logp_difference/mean": 0.021475713327527048, "step": 1880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.5, "completions/max_terminated_length": 320.5, "completions/mean_length": 224.575, "completions/mean_terminated_length": 224.575, "completions/min_length": 142.8, "completions/min_terminated_length": 142.8, "entropy": 1.1039319053292274, "epoch": 0.17376114737519538, "frac_reward_zero_std": 0.0, "grad_norm": 6.80249547958374, "kl": 0.02133035861188546, "learning_rate": 4.999152077644805e-06, "loss": 0.0007, "num_tokens": 8220229.0, "reward": -0.1, "reward_std": 0.7276197880506515, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.7276198178529739, "sampling/importance_sampling_ratio/max": 1.297184681892395, "sampling/importance_sampling_ratio/mean": 0.9996058583259583, "sampling/importance_sampling_ratio/min": 0.7719779908657074, "sampling/sampling_logp_difference/max": 0.28920464515686034, "sampling/sampling_logp_difference/mean": 0.020804373547434807, "step": 1890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.2, "completions/max_terminated_length": 273.2, "completions/mean_length": 182.775, "completions/mean_terminated_length": 182.775, "completions/min_length": 119.5, "completions/min_terminated_length": 119.5, "entropy": 1.1487253315746784, "epoch": 0.17468051852532868, "frac_reward_zero_std": 0.5, "grad_norm": 6.879741191864014, "kl": 0.015241934260120615, "learning_rate": 4.9990848117961705e-06, "loss": 0.0009, "num_tokens": 8260819.0, "reward": -0.0375, "reward_std": 0.40172620117664337, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.40172623097896576, "sampling/importance_sampling_ratio/max": 1.2542536377906799, "sampling/importance_sampling_ratio/mean": 1.0002043008804322, "sampling/importance_sampling_ratio/min": 0.7620341658592225, "sampling/sampling_logp_difference/max": 0.2907947599887848, "sampling/sampling_logp_difference/mean": 0.022572824731469154, "step": 1900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.5, "completions/max_terminated_length": 329.5, "completions/mean_length": 209.6625, "completions/mean_terminated_length": 209.6625, "completions/min_length": 100.4, "completions/min_terminated_length": 100.4, "entropy": 1.107984083145857, "epoch": 0.17559988967546197, "frac_reward_zero_std": 0.2, "grad_norm": 5.747552871704102, "kl": 0.017384014307754114, "learning_rate": 4.999014979187108e-06, "loss": 0.0011, "num_tokens": 8302392.0, "reward": 0.4375, "reward_std": 0.5283277183771133, "rewards/reward_fn/mean": 0.4375, "rewards/reward_fn/std": 0.5283277422189713, "sampling/importance_sampling_ratio/max": 1.2413341641426086, "sampling/importance_sampling_ratio/mean": 0.9996440470218658, "sampling/importance_sampling_ratio/min": 0.7668880820274353, "sampling/sampling_logp_difference/max": 0.276617693901062, "sampling/sampling_logp_difference/mean": 0.021286442503333093, "step": 1910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.1, "completions/max_terminated_length": 308.1, "completions/mean_length": 209.2, "completions/mean_terminated_length": 209.2, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.077923546358943, "epoch": 0.1765192608255953, "frac_reward_zero_std": 0.2, "grad_norm": 5.904787063598633, "kl": 0.020174477086402475, "learning_rate": 4.9989425798893415e-06, "loss": 0.0014, "num_tokens": 8342456.0, "reward": -0.5, "reward_std": 0.48138763904571535, "rewards/reward_fn/mean": -0.5, "rewards/reward_fn/std": 0.48138765096664426, "sampling/importance_sampling_ratio/max": 1.2854896545410157, "sampling/importance_sampling_ratio/mean": 0.9998751997947692, "sampling/importance_sampling_ratio/min": 0.7877361714839936, "sampling/sampling_logp_difference/max": 0.27977050840854645, "sampling/sampling_logp_difference/mean": 0.020843944512307644, "step": 1920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.2, "completions/max_terminated_length": 289.2, "completions/mean_length": 215.05, "completions/mean_terminated_length": 215.05, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.1482290662825108, "epoch": 0.1774386319757286, "frac_reward_zero_std": 0.2, "grad_norm": 6.440964698791504, "kl": 0.017538489843718707, "learning_rate": 4.998867613977231e-06, "loss": 0.0011, "num_tokens": 8389636.0, "reward": -0.00625, "reward_std": 0.4986860066652298, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.49868602454662325, "sampling/importance_sampling_ratio/max": 1.2459492087364197, "sampling/importance_sampling_ratio/mean": 1.0001853168010713, "sampling/importance_sampling_ratio/min": 0.8000842928886414, "sampling/sampling_logp_difference/max": 0.2414076328277588, "sampling/sampling_logp_difference/mean": 0.021038925647735594, "step": 1930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.4, "completions/max_terminated_length": 336.4, "completions/mean_length": 233.7, "completions/mean_terminated_length": 233.7, "completions/min_length": 151.5, "completions/min_terminated_length": 151.5, "entropy": 1.1576966300606728, "epoch": 0.17835800312586192, "frac_reward_zero_std": 0.3, "grad_norm": 0.10882382839918137, "kl": 0.019720542541472243, "learning_rate": 4.998790081527771e-06, "loss": 0.001, "num_tokens": 8433580.0, "reward": -0.0875, "reward_std": 0.41069733500480654, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.410697340965271, "sampling/importance_sampling_ratio/max": 1.2426087141036988, "sampling/importance_sampling_ratio/mean": 0.9995519280433655, "sampling/importance_sampling_ratio/min": 0.772708636522293, "sampling/sampling_logp_difference/max": 0.26426082849502563, "sampling/sampling_logp_difference/mean": 0.021607748977839947, "step": 1940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.7, "completions/max_terminated_length": 322.7, "completions/mean_length": 226.7625, "completions/mean_terminated_length": 226.7625, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.116366022080183, "epoch": 0.17927737427599522, "frac_reward_zero_std": 0.2, "grad_norm": 4.747546195983887, "kl": 0.017638581997016445, "learning_rate": 4.9987099826205966e-06, "loss": 0.0009, "num_tokens": 8479513.0, "reward": 0.0375, "reward_std": 0.5626426249742508, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.5626426488161087, "sampling/importance_sampling_ratio/max": 1.262166678905487, "sampling/importance_sampling_ratio/mean": 1.000122892856598, "sampling/importance_sampling_ratio/min": 0.7566198885440827, "sampling/sampling_logp_difference/max": 0.2945895254611969, "sampling/sampling_logp_difference/mean": 0.02078951820731163, "step": 1950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.5, "completions/max_terminated_length": 272.5, "completions/mean_length": 194.15, "completions/mean_terminated_length": 194.15, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.085212154686451, "epoch": 0.18019674542612854, "frac_reward_zero_std": 0.4, "grad_norm": 0.095575712621212, "kl": 0.03095030563999899, "learning_rate": 4.998627317337973e-06, "loss": 0.0016, "num_tokens": 8518621.0, "reward": -0.0875, "reward_std": 0.3881663382053375, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.3881663382053375, "sampling/importance_sampling_ratio/max": 1.3164339065551758, "sampling/importance_sampling_ratio/mean": 1.0007083892822266, "sampling/importance_sampling_ratio/min": 0.760808676481247, "sampling/sampling_logp_difference/max": 0.33363351225852966, "sampling/sampling_logp_difference/mean": 0.02128539700061083, "step": 1960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.6, "completions/max_terminated_length": 302.6, "completions/mean_length": 223.075, "completions/mean_terminated_length": 223.075, "completions/min_length": 152.3, "completions/min_terminated_length": 152.3, "entropy": 1.1523349031805992, "epoch": 0.18111611657626184, "frac_reward_zero_std": 0.4, "grad_norm": 6.747213363647461, "kl": 0.02937217736034654, "learning_rate": 4.9985420857648044e-06, "loss": 0.0015, "num_tokens": 8568403.0, "reward": 0.05, "reward_std": 0.4174750655889511, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.4174750715494156, "sampling/importance_sampling_ratio/max": 1.3106306672096253, "sampling/importance_sampling_ratio/mean": 0.9993778169155121, "sampling/importance_sampling_ratio/min": 0.7692589163780212, "sampling/sampling_logp_difference/max": 0.3071877956390381, "sampling/sampling_logp_difference/mean": 0.021547301672399044, "step": 1970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.3, "completions/max_terminated_length": 293.3, "completions/mean_length": 216.2125, "completions/mean_terminated_length": 216.2125, "completions/min_length": 161.6, "completions/min_terminated_length": 161.6, "entropy": 1.0855547845363618, "epoch": 0.18203548772639513, "frac_reward_zero_std": 0.1, "grad_norm": 0.09698199480772018, "kl": 0.022887393622659145, "learning_rate": 4.998454287988632e-06, "loss": 0.0011, "num_tokens": 8610244.0, "reward": -0.0625, "reward_std": 0.5482046812772751, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.5482046931982041, "sampling/importance_sampling_ratio/max": 1.2999603390693664, "sampling/importance_sampling_ratio/mean": 0.9996606171131134, "sampling/importance_sampling_ratio/min": 0.7765462398529053, "sampling/sampling_logp_difference/max": 0.30179101824760435, "sampling/sampling_logp_difference/mean": 0.020937790907919408, "step": 1980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 209.0375, "completions/mean_terminated_length": 209.0375, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.029877895116806, "epoch": 0.18295485887652846, "frac_reward_zero_std": 0.2, "grad_norm": 0.12309105694293976, "kl": 0.0291645442659501, "learning_rate": 4.998363924099629e-06, "loss": 0.0014, "num_tokens": 8654591.0, "reward": 0.05, "reward_std": 0.36818075478076934, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.36818076074123385, "sampling/importance_sampling_ratio/max": 1.2720123529434204, "sampling/importance_sampling_ratio/mean": 0.999754011631012, "sampling/importance_sampling_ratio/min": 0.7930947542190552, "sampling/sampling_logp_difference/max": 0.25222728550434115, "sampling/sampling_logp_difference/mean": 0.020741439983248712, "step": 1990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.2, "completions/max_terminated_length": 258.2, "completions/mean_length": 195.075, "completions/mean_terminated_length": 195.075, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 0.9862498935312033, "epoch": 0.18387423002666176, "frac_reward_zero_std": 0.3, "grad_norm": 6.373353004455566, "kl": 0.03954175267135725, "learning_rate": 4.998270994190609e-06, "loss": 0.0017, "num_tokens": 8697589.0, "reward": 0.1625, "reward_std": 0.4119541674852371, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.4119541734457016, "sampling/importance_sampling_ratio/max": 1.2410858035087586, "sampling/importance_sampling_ratio/mean": 0.9996942818164826, "sampling/importance_sampling_ratio/min": 0.7882209658622742, "sampling/sampling_logp_difference/max": 0.2585688352584839, "sampling/sampling_logp_difference/mean": 0.020708341896533967, "step": 2000 }, { "epoch": 0.18387423002666176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 294.13961605584643, "eval_completions/max_terminated_length": 292.0069808027923, "eval_completions/mean_length": 209.1326352530541, "eval_completions/mean_terminated_length": 208.5953214156066, "eval_completions/min_length": 141.1064572425829, "eval_completions/min_terminated_length": 141.1064572425829, "eval_entropy": 1.0833796258060095, "eval_frac_reward_zero_std": 0.23734729493891799, "eval_kl": 0.037226662446092976, "eval_loss": 0.0019072559662163258, "eval_num_tokens": 8697589.0, "eval_reward": 0.00981675392670157, "eval_reward_std": 0.4129836211861948, "eval_rewards/reward_fn/mean": 0.00981675392670157, "eval_rewards/reward_fn/std": 0.41298363491710777, "eval_runtime": 1720.6011, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2999582694255036, "eval_sampling/importance_sampling_ratio/mean": 0.9999931461940053, "eval_sampling/importance_sampling_ratio/min": 0.7586964579724933, "eval_sampling/sampling_logp_difference/max": 0.32477144077810316, "eval_sampling/sampling_logp_difference/mean": 0.021163378978703972, "eval_steps_per_second": 0.042, "step": 2000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.1, "completions/max_terminated_length": 265.1, "completions/mean_length": 204.45, "completions/mean_terminated_length": 204.45, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0727780736982822, "epoch": 0.18479360117679508, "frac_reward_zero_std": 0.3, "grad_norm": 5.539535045623779, "kl": 0.05420771196950227, "learning_rate": 4.9981754983570165e-06, "loss": 0.0028, "num_tokens": 8739433.0, "reward": -0.225, "reward_std": 0.3548214554786682, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3548214614391327, "sampling/importance_sampling_ratio/max": 1.3256034851074219, "sampling/importance_sampling_ratio/mean": 0.9996547698974609, "sampling/importance_sampling_ratio/min": 0.7573504507541656, "sampling/sampling_logp_difference/max": 0.3263516664505005, "sampling/sampling_logp_difference/mean": 0.021330700255930424, "step": 2010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 208.7875, "completions/mean_terminated_length": 208.7875, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.1074910007417202, "epoch": 0.18571297232692838, "frac_reward_zero_std": 0.3, "grad_norm": 0.1339806318283081, "kl": 0.04188211368164048, "learning_rate": 4.998077436696934e-06, "loss": 0.0021, "num_tokens": 8783264.0, "reward": -0.1125, "reward_std": 0.3818357288837433, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.38183574080467225, "sampling/importance_sampling_ratio/max": 1.2332355976104736, "sampling/importance_sampling_ratio/mean": 1.000568288564682, "sampling/importance_sampling_ratio/min": 0.7717979192733765, "sampling/sampling_logp_difference/max": 0.2694129407405853, "sampling/sampling_logp_difference/mean": 0.02108699418604374, "step": 2020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.4, "completions/max_terminated_length": 276.4, "completions/mean_length": 205.95, "completions/mean_terminated_length": 205.95, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.0832145676016807, "epoch": 0.1866323434770617, "frac_reward_zero_std": 0.5, "grad_norm": 6.657064437866211, "kl": 0.054080014571081844, "learning_rate": 4.997976809311078e-06, "loss": 0.0027, "num_tokens": 8824580.0, "reward": 0.3, "reward_std": 0.2328435957431793, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.2328436017036438, "sampling/importance_sampling_ratio/max": 1.3020219087600708, "sampling/importance_sampling_ratio/mean": 1.0002191722393037, "sampling/importance_sampling_ratio/min": 0.7339301750063896, "sampling/sampling_logp_difference/max": 0.3908567070960999, "sampling/sampling_logp_difference/mean": 0.02179529834538698, "step": 2030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 186.1, "completions/mean_terminated_length": 186.1, "completions/min_length": 115.9, "completions/min_terminated_length": 115.9, "entropy": 1.0329186499118805, "epoch": 0.187551714627195, "frac_reward_zero_std": 0.5, "grad_norm": 0.15627823770046234, "kl": 0.03850859795929864, "learning_rate": 4.997873616302802e-06, "loss": 0.0021, "num_tokens": 8866460.0, "reward": 0.25625, "reward_std": 0.3145385950803757, "rewards/reward_fn/mean": 0.25625, "rewards/reward_fn/std": 0.31453860104084014, "sampling/importance_sampling_ratio/max": 1.2636144042015076, "sampling/importance_sampling_ratio/mean": 1.000742393732071, "sampling/importance_sampling_ratio/min": 0.7930640876293182, "sampling/sampling_logp_difference/max": 0.2515856772661209, "sampling/sampling_logp_difference/mean": 0.02052618097513914, "step": 2040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.0, "completions/max_terminated_length": 311.0, "completions/mean_length": 220.4, "completions/mean_terminated_length": 220.4, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.055208795145154, "epoch": 0.1884710857773283, "frac_reward_zero_std": 0.3, "grad_norm": 6.259217739105225, "kl": 0.03475877782329917, "learning_rate": 4.997767857778093e-06, "loss": 0.0019, "num_tokens": 8912868.0, "reward": 0.2375, "reward_std": 0.3383516758680344, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.3383516877889633, "sampling/importance_sampling_ratio/max": 1.283850336074829, "sampling/importance_sampling_ratio/mean": 1.0002785205841065, "sampling/importance_sampling_ratio/min": 0.7336234390735626, "sampling/sampling_logp_difference/max": 0.33705209493637084, "sampling/sampling_logp_difference/mean": 0.021786217391490937, "step": 2050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.3, "completions/max_terminated_length": 306.3, "completions/mean_length": 225.3875, "completions/mean_terminated_length": 225.3875, "completions/min_length": 157.2, "completions/min_terminated_length": 157.2, "entropy": 1.0984139777719975, "epoch": 0.18939045692746162, "frac_reward_zero_std": 0.4, "grad_norm": 0.13372617959976196, "kl": 0.027556859265314416, "learning_rate": 4.997659533845574e-06, "loss": 0.0013, "num_tokens": 8959131.0, "reward": 0.1875, "reward_std": 0.2394672751426697, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.23946728110313414, "sampling/importance_sampling_ratio/max": 1.242374837398529, "sampling/importance_sampling_ratio/mean": 0.9999833285808564, "sampling/importance_sampling_ratio/min": 0.7497252464294434, "sampling/sampling_logp_difference/max": 0.29617769122123716, "sampling/sampling_logp_difference/mean": 0.021633453853428363, "step": 2060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 222.7125, "completions/mean_terminated_length": 222.7125, "completions/min_length": 154.1, "completions/min_terminated_length": 154.1, "entropy": 1.066904890537262, "epoch": 0.19030982807759492, "frac_reward_zero_std": 0.4, "grad_norm": 0.10214605182409286, "kl": 0.04522109412937425, "learning_rate": 4.997548644616501e-06, "loss": 0.0021, "num_tokens": 9002820.0, "reward": -0.1625, "reward_std": 0.3167852282524109, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.3167852461338043, "sampling/importance_sampling_ratio/max": 1.280836009979248, "sampling/importance_sampling_ratio/mean": 0.9996492743492127, "sampling/importance_sampling_ratio/min": 0.7761459469795227, "sampling/sampling_logp_difference/max": 0.2953964829444885, "sampling/sampling_logp_difference/mean": 0.021134966053068638, "step": 2070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.4, "completions/max_terminated_length": 332.4, "completions/mean_length": 245.9125, "completions/mean_terminated_length": 245.9125, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "entropy": 1.1535887710750103, "epoch": 0.19122919922772824, "frac_reward_zero_std": 0.5, "grad_norm": 0.10864507406949997, "kl": 0.04408502578153275, "learning_rate": 4.997435190204767e-06, "loss": 0.0022, "num_tokens": 9050621.0, "reward": 0.0875, "reward_std": 0.2856272906064987, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.2856272965669632, "sampling/importance_sampling_ratio/max": 1.27427978515625, "sampling/importance_sampling_ratio/mean": 1.0002832472324372, "sampling/importance_sampling_ratio/min": 0.7479039609432221, "sampling/sampling_logp_difference/max": 0.32640727460384367, "sampling/sampling_logp_difference/mean": 0.021294852904975414, "step": 2080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.2, "completions/max_terminated_length": 304.2, "completions/mean_length": 216.5625, "completions/mean_terminated_length": 216.5625, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.1119799844920635, "epoch": 0.19214857037786154, "frac_reward_zero_std": 0.2, "grad_norm": 6.983915328979492, "kl": 0.045881410402944314, "learning_rate": 4.9973191707268986e-06, "loss": 0.0027, "num_tokens": 9098066.0, "reward": 0.2875, "reward_std": 0.39642935395240786, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.3964293718338013, "sampling/importance_sampling_ratio/max": 1.2645806550979615, "sampling/importance_sampling_ratio/mean": 0.9998108804225921, "sampling/importance_sampling_ratio/min": 0.7577556669712067, "sampling/sampling_logp_difference/max": 0.30794984102249146, "sampling/sampling_logp_difference/mean": 0.02104753293097019, "step": 2090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.5, "completions/max_terminated_length": 315.5, "completions/mean_length": 234.3125, "completions/mean_terminated_length": 234.3125, "completions/min_length": 153.5, "completions/min_terminated_length": 153.5, "entropy": 1.180136325210333, "epoch": 0.19306794152799486, "frac_reward_zero_std": 0.1, "grad_norm": 5.488556385040283, "kl": 0.03945447752485052, "learning_rate": 4.9972005863020574e-06, "loss": 0.0022, "num_tokens": 9146147.0, "reward": -0.0625, "reward_std": 0.47202224731445314, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.47202226519584656, "sampling/importance_sampling_ratio/max": 1.380826508998871, "sampling/importance_sampling_ratio/mean": 1.0002896964550019, "sampling/importance_sampling_ratio/min": 0.792996472120285, "sampling/sampling_logp_difference/max": 0.3363222122192383, "sampling/sampling_logp_difference/mean": 0.021930642798542975, "step": 2100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 223.3375, "completions/mean_terminated_length": 223.3375, "completions/min_length": 160.7, "completions/min_terminated_length": 160.7, "entropy": 1.12664475440979, "epoch": 0.19398731267812816, "frac_reward_zero_std": 0.3, "grad_norm": 3.8552494049072266, "kl": 0.028041312610730527, "learning_rate": 4.997079437052039e-06, "loss": 0.0019, "num_tokens": 9191158.0, "reward": -0.05, "reward_std": 0.3670209854841232, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3670209974050522, "sampling/importance_sampling_ratio/max": 1.3053725361824036, "sampling/importance_sampling_ratio/mean": 0.9995544373989105, "sampling/importance_sampling_ratio/min": 0.7901838839054107, "sampling/sampling_logp_difference/max": 0.2744863986968994, "sampling/sampling_logp_difference/mean": 0.021696880273520947, "step": 2110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 242.4625, "completions/mean_terminated_length": 242.4625, "completions/min_length": 174.8, "completions/min_terminated_length": 174.8, "entropy": 1.1719475172460079, "epoch": 0.19490668382826146, "frac_reward_zero_std": 0.1, "grad_norm": 5.561178207397461, "kl": 0.039780617982614785, "learning_rate": 4.996955723101273e-06, "loss": 0.0021, "num_tokens": 9239419.0, "reward": -0.1625, "reward_std": 0.4653678238391876, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.46536784172058104, "sampling/importance_sampling_ratio/max": 1.3244190573692323, "sampling/importance_sampling_ratio/mean": 0.999659663438797, "sampling/importance_sampling_ratio/min": 0.7307224631309509, "sampling/sampling_logp_difference/max": 0.3514132142066956, "sampling/sampling_logp_difference/mean": 0.0220487168058753, "step": 2120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.7, "completions/max_terminated_length": 321.7, "completions/mean_length": 241.0625, "completions/mean_terminated_length": 241.0625, "completions/min_length": 161.0, "completions/min_terminated_length": 161.0, "entropy": 1.1713617824018001, "epoch": 0.19582605497839478, "frac_reward_zero_std": 0.3, "grad_norm": 5.935511589050293, "kl": 0.035165926109766585, "learning_rate": 4.996829444576825e-06, "loss": 0.0018, "num_tokens": 9288672.0, "reward": -0.2375, "reward_std": 0.34120489954948424, "rewards/reward_fn/mean": -0.2375, "rewards/reward_fn/std": 0.34120491743087766, "sampling/importance_sampling_ratio/max": 1.2524248003959655, "sampling/importance_sampling_ratio/mean": 0.9999395251274109, "sampling/importance_sampling_ratio/min": 0.7554161071777343, "sampling/sampling_logp_difference/max": 0.312109237909317, "sampling/sampling_logp_difference/mean": 0.020913349092006685, "step": 2130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.8, "completions/max_terminated_length": 323.8, "completions/mean_length": 243.2, "completions/mean_terminated_length": 243.2, "completions/min_length": 158.3, "completions/min_terminated_length": 158.3, "entropy": 1.1296326026320458, "epoch": 0.19674542612852808, "frac_reward_zero_std": 0.2, "grad_norm": 7.870914936065674, "kl": 0.04121863882755861, "learning_rate": 4.996700601608391e-06, "loss": 0.002, "num_tokens": 9335976.0, "reward": -0.025, "reward_std": 0.3774735927581787, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.37747361660003664, "sampling/importance_sampling_ratio/max": 1.2856929779052735, "sampling/importance_sampling_ratio/mean": 1.0002261340618133, "sampling/importance_sampling_ratio/min": 0.7678008794784545, "sampling/sampling_logp_difference/max": 0.2914842367172241, "sampling/sampling_logp_difference/mean": 0.02136395815759897, "step": 2140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.6, "completions/max_terminated_length": 304.6, "completions/mean_length": 209.0625, "completions/mean_terminated_length": 209.0625, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.2393813714385034, "epoch": 0.1976647972786614, "frac_reward_zero_std": 0.4, "grad_norm": 5.041493892669678, "kl": 0.037880709511227906, "learning_rate": 4.9965691943283045e-06, "loss": 0.0018, "num_tokens": 9378029.0, "reward": -0.20625, "reward_std": 0.29540800750255586, "rewards/reward_fn/mean": -0.20625, "rewards/reward_fn/std": 0.29540803134441374, "sampling/importance_sampling_ratio/max": 1.2303287863731385, "sampling/importance_sampling_ratio/mean": 1.0006081283092498, "sampling/importance_sampling_ratio/min": 0.7902114391326904, "sampling/sampling_logp_difference/max": 0.24944406747817993, "sampling/sampling_logp_difference/mean": 0.022588260658085346, "step": 2150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.3, "completions/max_terminated_length": 294.3, "completions/mean_length": 217.4125, "completions/mean_terminated_length": 217.4125, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.1623989813029767, "epoch": 0.1985841684287947, "frac_reward_zero_std": 0.3, "grad_norm": 6.053800582885742, "kl": 0.04897128135198727, "learning_rate": 4.996435222871531e-06, "loss": 0.0026, "num_tokens": 9424854.0, "reward": 0.3375, "reward_std": 0.29839127957820893, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.29839127957820893, "sampling/importance_sampling_ratio/max": 1.2595209956169129, "sampling/importance_sampling_ratio/mean": 0.9999686002731323, "sampling/importance_sampling_ratio/min": 0.7652659296989441, "sampling/sampling_logp_difference/max": 0.29065966606140137, "sampling/sampling_logp_difference/mean": 0.021971743553876877, "step": 2160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 196.2375, "completions/mean_terminated_length": 196.2375, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 1.12191246971488, "epoch": 0.19950353957892802, "frac_reward_zero_std": 0.3, "grad_norm": 4.84433650970459, "kl": 0.034589039994170886, "learning_rate": 4.996298687375669e-06, "loss": 0.0018, "num_tokens": 9464745.0, "reward": -0.1, "reward_std": 0.3570065200328827, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.35700652599334715, "sampling/importance_sampling_ratio/max": 1.2351934671401978, "sampling/importance_sampling_ratio/mean": 1.0003180503845215, "sampling/importance_sampling_ratio/min": 0.7983351945877075, "sampling/sampling_logp_difference/max": 0.23011383414268494, "sampling/sampling_logp_difference/mean": 0.02162060271948576, "step": 2170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 244.7, "completions/max_terminated_length": 244.7, "completions/mean_length": 169.0875, "completions/mean_terminated_length": 169.0875, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 1.080114208161831, "epoch": 0.20042291072906132, "frac_reward_zero_std": 0.4, "grad_norm": 0.12924686074256897, "kl": 0.04256691740592942, "learning_rate": 4.996159587980953e-06, "loss": 0.002, "num_tokens": 9501200.0, "reward": 0.175, "reward_std": 0.3474793940782547, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.34747940599918364, "sampling/importance_sampling_ratio/max": 1.2207862496376038, "sampling/importance_sampling_ratio/mean": 0.9995071887969971, "sampling/importance_sampling_ratio/min": 0.775526350736618, "sampling/sampling_logp_difference/max": 0.2593968629837036, "sampling/sampling_logp_difference/mean": 0.02124457973986864, "step": 2180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.4, "completions/max_terminated_length": 340.4, "completions/mean_length": 218.15, "completions/mean_terminated_length": 218.15, "completions/min_length": 146.1, "completions/min_terminated_length": 146.1, "entropy": 1.096110636740923, "epoch": 0.20134228187919462, "frac_reward_zero_std": 0.3, "grad_norm": 0.14402176439762115, "kl": 0.03449008559109643, "learning_rate": 4.9960179248302485e-06, "loss": 0.001, "num_tokens": 9545660.0, "reward": -0.375, "reward_std": 0.3575767487287521, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.3575767546892166, "sampling/importance_sampling_ratio/max": 1.2408198356628417, "sampling/importance_sampling_ratio/mean": 0.9994823038578033, "sampling/importance_sampling_ratio/min": 0.7583486199378967, "sampling/sampling_logp_difference/max": 0.2966102182865143, "sampling/sampling_logp_difference/mean": 0.020903543941676615, "step": 2190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.1, "completions/max_terminated_length": 280.1, "completions/mean_length": 187.3125, "completions/mean_terminated_length": 187.3125, "completions/min_length": 114.7, "completions/min_terminated_length": 114.7, "entropy": 0.9106440592557192, "epoch": 0.20226165302932794, "frac_reward_zero_std": 0.2, "grad_norm": 6.621983051300049, "kl": 0.04620505248894915, "learning_rate": 4.995873698069054e-06, "loss": 0.0023, "num_tokens": 9584253.0, "reward": 0.1875, "reward_std": 0.34637176990509033, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3463717758655548, "sampling/importance_sampling_ratio/max": 1.3062060832977296, "sampling/importance_sampling_ratio/mean": 0.9998080670833588, "sampling/importance_sampling_ratio/min": 0.766309243440628, "sampling/sampling_logp_difference/max": 0.30480992794036865, "sampling/sampling_logp_difference/mean": 0.01946581043303013, "step": 2200 }, { "epoch": 0.20226165302932794, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0004363001745200698, "eval_completions/max_length": 270.21465968586386, "eval_completions/max_terminated_length": 269.76439790575915, "eval_completions/mean_length": 186.1173647469459, "eval_completions/mean_terminated_length": 185.99071306957623, "eval_completions/min_length": 116.75043630017451, "eval_completions/min_terminated_length": 116.75043630017451, "eval_entropy": 1.0583471444160735, "eval_frac_reward_zero_std": 0.27225130890052357, "eval_kl": 0.03981750488547806, "eval_loss": 0.0020967598538845778, "eval_num_tokens": 9584253.0, "eval_reward": 0.029668411867364748, "eval_reward_std": 0.39895731117088756, "eval_rewards/reward_fn/mean": 0.029668411867364748, "eval_rewards/reward_fn/std": 0.3989573223012488, "eval_runtime": 1597.1237, "eval_samples_per_second": 0.359, "eval_sampling/importance_sampling_ratio/max": 1.275554472863362, "eval_sampling/importance_sampling_ratio/mean": 0.9999324976877808, "eval_sampling/importance_sampling_ratio/min": 0.7752521586771827, "eval_sampling/sampling_logp_difference/max": 0.2901335429874806, "eval_sampling/sampling_logp_difference/mean": 0.020724559201164915, "eval_steps_per_second": 0.045, "step": 2200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.6, "completions/max_terminated_length": 299.6, "completions/mean_length": 200.3, "completions/mean_terminated_length": 200.3, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0624324716627598, "epoch": 0.20318102417946124, "frac_reward_zero_std": 0.2, "grad_norm": 7.511288166046143, "kl": 0.035200098413042724, "learning_rate": 4.995726907845505e-06, "loss": 0.002, "num_tokens": 9626693.0, "reward": -0.3125, "reward_std": 0.42686644196510315, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.4268664479255676, "sampling/importance_sampling_ratio/max": 1.2750413417816162, "sampling/importance_sampling_ratio/mean": 1.0001168131828309, "sampling/importance_sampling_ratio/min": 0.7514230638742447, "sampling/sampling_logp_difference/max": 0.31315724849700927, "sampling/sampling_logp_difference/mean": 0.021705342829227446, "step": 2210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 190.1, "completions/mean_terminated_length": 190.1, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 1.0477397292852402, "epoch": 0.20410039532959456, "frac_reward_zero_std": 0.4, "grad_norm": 0.10544715076684952, "kl": 0.031700442923465744, "learning_rate": 4.995577554310363e-06, "loss": 0.0012, "num_tokens": 9668269.0, "reward": -0.25, "reward_std": 0.3466205418109894, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3466205596923828, "sampling/importance_sampling_ratio/max": 1.2567079782485961, "sampling/importance_sampling_ratio/mean": 1.0003982841968537, "sampling/importance_sampling_ratio/min": 0.73931904733181, "sampling/sampling_logp_difference/max": 0.3210598468780518, "sampling/sampling_logp_difference/mean": 0.020927842892706394, "step": 2220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 278.0, "completions/max_terminated_length": 277.3, "completions/mean_length": 187.125, "completions/mean_terminated_length": 181.6041687011719, "completions/min_length": 110.1, "completions/min_terminated_length": 110.1, "entropy": 0.9990961596369743, "epoch": 0.20501976647972786, "frac_reward_zero_std": 0.1, "grad_norm": 5.903346538543701, "kl": 0.03451996381045319, "learning_rate": 4.995425637617027e-06, "loss": 0.0018, "num_tokens": 9708199.0, "reward": 0.0875, "reward_std": 0.5238449484109878, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.5238449543714523, "sampling/importance_sampling_ratio/max": 1.237573754787445, "sampling/importance_sampling_ratio/mean": 1.0002052724361419, "sampling/importance_sampling_ratio/min": 0.7941884994506836, "sampling/sampling_logp_difference/max": 0.247905957698822, "sampling/sampling_logp_difference/mean": 0.020801369659602643, "step": 2230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.3, "completions/max_terminated_length": 276.3, "completions/mean_length": 191.15, "completions/mean_terminated_length": 191.15, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.10877006649971, "epoch": 0.20593913762986119, "frac_reward_zero_std": 0.3, "grad_norm": 0.24242615699768066, "kl": 0.033766067633405325, "learning_rate": 4.995271157921529e-06, "loss": 0.0018, "num_tokens": 9748195.0, "reward": 0.3, "reward_std": 0.46596879661083224, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.46596881449222566, "sampling/importance_sampling_ratio/max": 1.2661152005195617, "sampling/importance_sampling_ratio/mean": 0.999598640203476, "sampling/importance_sampling_ratio/min": 0.7782447814941407, "sampling/sampling_logp_difference/max": 0.2654001295566559, "sampling/sampling_logp_difference/mean": 0.022385939210653304, "step": 2240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.2, "completions/max_terminated_length": 260.2, "completions/mean_length": 186.35, "completions/mean_terminated_length": 186.35, "completions/min_length": 104.4, "completions/min_terminated_length": 104.4, "entropy": 1.0151292737573385, "epoch": 0.20685850877999448, "frac_reward_zero_std": 0.2, "grad_norm": 7.4661545753479, "kl": 0.06271298103383742, "learning_rate": 4.995114115382532e-06, "loss": 0.0029, "num_tokens": 9790023.0, "reward": -0.0875, "reward_std": 0.4487059205770493, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.4487059205770493, "sampling/importance_sampling_ratio/max": 1.281790828704834, "sampling/importance_sampling_ratio/mean": 1.0002649962902068, "sampling/importance_sampling_ratio/min": 0.7612417340278625, "sampling/sampling_logp_difference/max": 0.2945103049278259, "sampling/sampling_logp_difference/mean": 0.020727747585624456, "step": 2250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 212.8, "completions/max_terminated_length": 212.8, "completions/mean_length": 158.3125, "completions/mean_terminated_length": 158.3125, "completions/min_length": 104.9, "completions/min_terminated_length": 104.9, "entropy": 0.9757637903094292, "epoch": 0.20777787993012778, "frac_reward_zero_std": 0.3, "grad_norm": 6.1468634605407715, "kl": 0.03781939910841174, "learning_rate": 4.994954510161331e-06, "loss": 0.0025, "num_tokens": 9825448.0, "reward": 0.125, "reward_std": 0.36428422927856446, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.36428422927856446, "sampling/importance_sampling_ratio/max": 1.2629591584205628, "sampling/importance_sampling_ratio/mean": 1.0006554782390595, "sampling/importance_sampling_ratio/min": 0.7807676434516907, "sampling/sampling_logp_difference/max": 0.26751006841659547, "sampling/sampling_logp_difference/mean": 0.021309599094092847, "step": 2260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.1, "completions/max_terminated_length": 253.1, "completions/mean_length": 187.125, "completions/mean_terminated_length": 187.125, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.0190150149166584, "epoch": 0.2086972510802611, "frac_reward_zero_std": 0.4, "grad_norm": 5.6369476318359375, "kl": 0.028186429140623658, "learning_rate": 4.994792342421852e-06, "loss": 0.0011, "num_tokens": 9866826.0, "reward": 0.10625, "reward_std": 0.3279952198266983, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.32799523174762724, "sampling/importance_sampling_ratio/max": 1.2821413159370423, "sampling/importance_sampling_ratio/mean": 0.9998392701148987, "sampling/importance_sampling_ratio/min": 0.7528982877731323, "sampling/sampling_logp_difference/max": 0.3191342711448669, "sampling/sampling_logp_difference/mean": 0.021119408123195172, "step": 2270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.9, "completions/max_terminated_length": 321.9, "completions/mean_length": 227.5375, "completions/mean_terminated_length": 227.5375, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0744174726307392, "epoch": 0.2096166222303944, "frac_reward_zero_std": 0.2, "grad_norm": 6.216409206390381, "kl": 0.034819805121514946, "learning_rate": 4.994627612330655e-06, "loss": 0.0023, "num_tokens": 9916917.0, "reward": -0.0125, "reward_std": 0.4910866290330887, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4910866469144821, "sampling/importance_sampling_ratio/max": 1.2545270442962646, "sampling/importance_sampling_ratio/mean": 0.9997605562210083, "sampling/importance_sampling_ratio/min": 0.7645127654075623, "sampling/sampling_logp_difference/max": 0.275495845079422, "sampling/sampling_logp_difference/mean": 0.020799391716718674, "step": 2280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.3, "completions/max_terminated_length": 305.3, "completions/mean_length": 216.8875, "completions/mean_terminated_length": 216.8875, "completions/min_length": 153.7, "completions/min_terminated_length": 153.7, "entropy": 1.061875542998314, "epoch": 0.21053599338052773, "frac_reward_zero_std": 0.2, "grad_norm": 7.102997779846191, "kl": 0.02847459077311214, "learning_rate": 4.994460320056932e-06, "loss": 0.0016, "num_tokens": 9961100.0, "reward": 0.00625, "reward_std": 0.5160910218954087, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.5160910278558731, "sampling/importance_sampling_ratio/max": 1.2617968320846558, "sampling/importance_sampling_ratio/mean": 1.000396978855133, "sampling/importance_sampling_ratio/min": 0.7838475465774536, "sampling/sampling_logp_difference/max": 0.2614006817340851, "sampling/sampling_logp_difference/mean": 0.021053414791822433, "step": 2290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.0, "completions/max_terminated_length": 313.0, "completions/mean_length": 218.1375, "completions/mean_terminated_length": 218.1375, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.0973895706236363, "epoch": 0.21145536453066102, "frac_reward_zero_std": 0.0, "grad_norm": 5.394501686096191, "kl": 0.03340508458204568, "learning_rate": 4.9942904657725044e-06, "loss": 0.0018, "num_tokens": 10004855.0, "reward": -0.175, "reward_std": 0.5268197923898696, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5268198102712631, "sampling/importance_sampling_ratio/max": 1.2829238653182984, "sampling/importance_sampling_ratio/mean": 1.0002897024154662, "sampling/importance_sampling_ratio/min": 0.7752999007701874, "sampling/sampling_logp_difference/max": 0.2683062791824341, "sampling/sampling_logp_difference/mean": 0.02157579008489847, "step": 2300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.6, "completions/max_terminated_length": 296.6, "completions/mean_length": 217.2, "completions/mean_terminated_length": 217.2, "completions/min_length": 158.1, "completions/min_terminated_length": 158.1, "entropy": 1.1110787563025952, "epoch": 0.21237473568079435, "frac_reward_zero_std": 0.2, "grad_norm": 5.3878493309021, "kl": 0.03351037814863957, "learning_rate": 4.9941180496518265e-06, "loss": 0.0021, "num_tokens": 10052295.0, "reward": -0.05, "reward_std": 0.3909654438495636, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3909654557704926, "sampling/importance_sampling_ratio/max": 1.2621555209159852, "sampling/importance_sampling_ratio/mean": 1.0002065360546113, "sampling/importance_sampling_ratio/min": 0.7889642417430878, "sampling/sampling_logp_difference/max": 0.2540652811527252, "sampling/sampling_logp_difference/mean": 0.02103464026004076, "step": 2310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 343.0, "completions/max_terminated_length": 343.0, "completions/mean_length": 232.7875, "completions/mean_terminated_length": 232.7875, "completions/min_length": 151.2, "completions/min_terminated_length": 151.2, "entropy": 1.1603176042437553, "epoch": 0.21329410683092764, "frac_reward_zero_std": 0.2, "grad_norm": 5.886568546295166, "kl": 0.034893020521849394, "learning_rate": 4.993943071871984e-06, "loss": 0.0019, "num_tokens": 10098510.0, "reward": -0.225, "reward_std": 0.44006140530109406, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.440061417222023, "sampling/importance_sampling_ratio/max": 1.3078218579292298, "sampling/importance_sampling_ratio/mean": 1.0001706719398498, "sampling/importance_sampling_ratio/min": 0.7689285039901733, "sampling/sampling_logp_difference/max": 0.3046458959579468, "sampling/sampling_logp_difference/mean": 0.02199532762169838, "step": 2320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 219.8375, "completions/mean_terminated_length": 219.8375, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0886065609753133, "epoch": 0.21421347798106097, "frac_reward_zero_std": 0.1, "grad_norm": 5.624151706695557, "kl": 0.05127502836985513, "learning_rate": 4.993765532612692e-06, "loss": 0.0023, "num_tokens": 10145217.0, "reward": -0.1375, "reward_std": 0.4553533583879471, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.455353370308876, "sampling/importance_sampling_ratio/max": 1.26222323179245, "sampling/importance_sampling_ratio/mean": 0.9998669266700745, "sampling/importance_sampling_ratio/min": 0.7888380587100983, "sampling/sampling_logp_difference/max": 0.2595196783542633, "sampling/sampling_logp_difference/mean": 0.020886969193816184, "step": 2330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 211.6375, "completions/mean_terminated_length": 211.6375, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.0902759991586208, "epoch": 0.21513284913119426, "frac_reward_zero_std": 0.2, "grad_norm": 6.470648288726807, "kl": 0.06268089526565745, "learning_rate": 4.993585432056299e-06, "loss": 0.0029, "num_tokens": 10187236.0, "reward": -0.2, "reward_std": 0.41113159954547884, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.41113162934780123, "sampling/importance_sampling_ratio/max": 1.2463592410087585, "sampling/importance_sampling_ratio/mean": 0.9999272048473358, "sampling/importance_sampling_ratio/min": 0.771414703130722, "sampling/sampling_logp_difference/max": 0.2705600321292877, "sampling/sampling_logp_difference/mean": 0.021528172120451928, "step": 2340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.9, "completions/max_terminated_length": 334.9, "completions/mean_length": 239.75, "completions/mean_terminated_length": 239.75, "completions/min_length": 164.6, "completions/min_terminated_length": 164.6, "entropy": 1.0869650952517986, "epoch": 0.21605222028132756, "frac_reward_zero_std": 0.2, "grad_norm": 5.322874069213867, "kl": 0.05598060296906624, "learning_rate": 4.993402770387782e-06, "loss": 0.0029, "num_tokens": 10230712.0, "reward": 0.0125, "reward_std": 0.3156497091054916, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.3156497091054916, "sampling/importance_sampling_ratio/max": 1.235500407218933, "sampling/importance_sampling_ratio/mean": 0.9997797667980194, "sampling/importance_sampling_ratio/min": 0.7792201340198517, "sampling/sampling_logp_difference/max": 0.254269540309906, "sampling/sampling_logp_difference/mean": 0.02162648644298315, "step": 2350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.3, "completions/max_terminated_length": 257.3, "completions/mean_length": 196.925, "completions/mean_terminated_length": 196.925, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.1058829128742218, "epoch": 0.2169715914314609, "frac_reward_zero_std": 0.4, "grad_norm": 0.16930444538593292, "kl": 0.06482969542848878, "learning_rate": 4.993217547794748e-06, "loss": 0.0032, "num_tokens": 10274154.0, "reward": 0.11875, "reward_std": 0.3333532750606537, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.3333532869815826, "sampling/importance_sampling_ratio/max": 1.2637481212615966, "sampling/importance_sampling_ratio/mean": 0.9993589758872986, "sampling/importance_sampling_ratio/min": 0.7624344527721405, "sampling/sampling_logp_difference/max": 0.2947443604469299, "sampling/sampling_logp_difference/mean": 0.021724693663418292, "step": 2360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.6, "completions/max_terminated_length": 316.6, "completions/mean_length": 204.8625, "completions/mean_terminated_length": 204.8625, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.0692421056330204, "epoch": 0.21789096258159418, "frac_reward_zero_std": 0.4, "grad_norm": 0.15967492759227753, "kl": 0.20456191007688176, "learning_rate": 4.993029764467438e-06, "loss": 0.01, "num_tokens": 10317031.0, "reward": -0.275, "reward_std": 0.25040294229984283, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.2504029482603073, "sampling/importance_sampling_ratio/max": 1.2576494455337524, "sampling/importance_sampling_ratio/mean": 0.9995150744915009, "sampling/importance_sampling_ratio/min": 0.795537668466568, "sampling/sampling_logp_difference/max": 0.2508530020713806, "sampling/sampling_logp_difference/mean": 0.020893950760364533, "step": 2370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 199.5125, "completions/mean_terminated_length": 199.5125, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0404366724193097, "epoch": 0.2188103337317275, "frac_reward_zero_std": 0.3, "grad_norm": 6.369706153869629, "kl": 0.05878325866069645, "learning_rate": 4.992839420598719e-06, "loss": 0.0026, "num_tokens": 10362424.0, "reward": 0.1625, "reward_std": 0.3896386116743088, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.3896386355161667, "sampling/importance_sampling_ratio/max": 1.327103841304779, "sampling/importance_sampling_ratio/mean": 1.0000702798366548, "sampling/importance_sampling_ratio/min": 0.7723352551460266, "sampling/sampling_logp_difference/max": 0.3010377049446106, "sampling/sampling_logp_difference/mean": 0.020696754939854144, "step": 2380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.8, "completions/max_terminated_length": 308.8, "completions/mean_length": 218.1375, "completions/mean_terminated_length": 218.1375, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.0392695881426335, "epoch": 0.2197297048818608, "frac_reward_zero_std": 0.5, "grad_norm": 5.291379928588867, "kl": 0.05432071414543316, "learning_rate": 4.992646516384089e-06, "loss": 0.0028, "num_tokens": 10406099.0, "reward": 0.0125, "reward_std": 0.2478467583656311, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.24784677624702453, "sampling/importance_sampling_ratio/max": 1.319090974330902, "sampling/importance_sampling_ratio/mean": 0.9999670863151551, "sampling/importance_sampling_ratio/min": 0.7603765904903412, "sampling/sampling_logp_difference/max": 0.33032592535018923, "sampling/sampling_logp_difference/mean": 0.02081177644431591, "step": 2390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.05, "completions/max_length": 313.5, "completions/max_terminated_length": 292.0, "completions/mean_length": 213.6125, "completions/mean_terminated_length": 198.75, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.0136086121201515, "epoch": 0.22064907603199413, "frac_reward_zero_std": 0.1, "grad_norm": 7.112517356872559, "kl": 0.05415805063676089, "learning_rate": 4.992451052021676e-06, "loss": 0.0027, "num_tokens": 10449860.0, "reward": 0.025, "reward_std": 0.42108623385429383, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.42108623385429383, "sampling/importance_sampling_ratio/max": 1.2796338558197022, "sampling/importance_sampling_ratio/mean": 0.9999727964401245, "sampling/importance_sampling_ratio/min": 0.7964086532592773, "sampling/sampling_logp_difference/max": 0.26001877784729005, "sampling/sampling_logp_difference/mean": 0.020032955892384054, "step": 2400 }, { "epoch": 0.22064907603199413, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 284.1535776614311, "eval_completions/max_terminated_length": 282.23385689354274, "eval_completions/mean_length": 202.07482547993018, "eval_completions/mean_terminated_length": 201.57536587540392, "eval_completions/min_length": 132.80104712041884, "eval_completions/min_terminated_length": 132.80104712041884, "eval_entropy": 1.0633187783430176, "eval_frac_reward_zero_std": 0.3333333333333333, "eval_kl": 0.051979973414070894, "eval_loss": 0.0027242586947977543, "eval_num_tokens": 10449860.0, "eval_reward": 0.03861256544502618, "eval_reward_std": 0.32740326946110415, "eval_rewards/reward_fn/mean": 0.03861256544502618, "eval_rewards/reward_fn/std": 0.3274032780949358, "eval_runtime": 1668.9665, "eval_samples_per_second": 0.343, "eval_sampling/importance_sampling_ratio/max": 1.2727945083931063, "eval_sampling/importance_sampling_ratio/mean": 0.9999052890516285, "eval_sampling/importance_sampling_ratio/min": 0.771780295323509, "eval_sampling/sampling_logp_difference/max": 0.2922986398920755, "eval_sampling/sampling_logp_difference/mean": 0.021046556556763137, "eval_steps_per_second": 0.043, "step": 2400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.9, "completions/max_terminated_length": 314.9, "completions/mean_length": 208.475, "completions/mean_terminated_length": 208.475, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.005654465407133, "epoch": 0.22156844718212743, "frac_reward_zero_std": 0.0, "grad_norm": 7.824005603790283, "kl": 0.06712795995990746, "learning_rate": 4.992253027712238e-06, "loss": 0.0036, "num_tokens": 10494194.0, "reward": -0.15625, "reward_std": 0.4667517691850662, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.46675178706645964, "sampling/importance_sampling_ratio/max": 1.3075680613517762, "sampling/importance_sampling_ratio/mean": 0.9999204099178314, "sampling/importance_sampling_ratio/min": 0.7279807150363922, "sampling/sampling_logp_difference/max": 0.33991732001304625, "sampling/sampling_logp_difference/mean": 0.020904942788183688, "step": 2410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.1, "completions/max_terminated_length": 252.1, "completions/mean_length": 169.075, "completions/mean_terminated_length": 169.075, "completions/min_length": 105.1, "completions/min_terminated_length": 105.1, "entropy": 1.0016811445355416, "epoch": 0.22248781833226072, "frac_reward_zero_std": 0.2, "grad_norm": 0.1650761067867279, "kl": 0.04423586474149488, "learning_rate": 4.992052443659163e-06, "loss": 0.0029, "num_tokens": 10531160.0, "reward": 0.08125, "reward_std": 0.44778977036476136, "rewards/reward_fn/mean": 0.08125, "rewards/reward_fn/std": 0.44778978228569033, "sampling/importance_sampling_ratio/max": 1.2289716601371765, "sampling/importance_sampling_ratio/mean": 1.0003277361392975, "sampling/importance_sampling_ratio/min": 0.7768762826919555, "sampling/sampling_logp_difference/max": 0.2542852520942688, "sampling/sampling_logp_difference/mean": 0.020382095873355866, "step": 2420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 241.8, "completions/max_terminated_length": 218.0, "completions/mean_length": 164.775, "completions/mean_terminated_length": 160.76071472167968, "completions/min_length": 105.7, "completions/min_terminated_length": 105.7, "entropy": 1.0449124954640865, "epoch": 0.22340718948239405, "frac_reward_zero_std": 0.3, "grad_norm": 6.653650283813477, "kl": 0.04148949554655701, "learning_rate": 4.9918493000684664e-06, "loss": 0.0021, "num_tokens": 10573358.0, "reward": 0.16875, "reward_std": 0.40078147053718566, "rewards/reward_fn/mean": 0.16875, "rewards/reward_fn/std": 0.4007814884185791, "sampling/importance_sampling_ratio/max": 1.247040319442749, "sampling/importance_sampling_ratio/mean": 0.999780535697937, "sampling/importance_sampling_ratio/min": 0.774491935968399, "sampling/sampling_logp_difference/max": 0.26907032132148745, "sampling/sampling_logp_difference/mean": 0.021498653292655944, "step": 2430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 200.8125, "completions/mean_terminated_length": 200.8125, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.1084297522902489, "epoch": 0.22432656063252734, "frac_reward_zero_std": 0.0, "grad_norm": 6.192923545837402, "kl": 0.04346291564288549, "learning_rate": 4.991643597148792e-06, "loss": 0.0026, "num_tokens": 10617647.0, "reward": -0.2125, "reward_std": 0.5326397061347962, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.5326397299766541, "sampling/importance_sampling_ratio/max": 1.2513515949249268, "sampling/importance_sampling_ratio/mean": 0.9998542249202729, "sampling/importance_sampling_ratio/min": 0.7990241110324859, "sampling/sampling_logp_difference/max": 0.24619550704956056, "sampling/sampling_logp_difference/mean": 0.022279351390898226, "step": 2440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.2, "completions/max_terminated_length": 315.2, "completions/mean_length": 210.6125, "completions/mean_terminated_length": 210.6125, "completions/min_length": 120.6, "completions/min_terminated_length": 120.6, "entropy": 0.9969607658684254, "epoch": 0.22524593178266067, "frac_reward_zero_std": 0.2, "grad_norm": 0.2507394552230835, "kl": 0.03999340662267059, "learning_rate": 4.991435335111413e-06, "loss": 0.0018, "num_tokens": 10661576.0, "reward": -0.3125, "reward_std": 0.42823243141174316, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.42823246121406555, "sampling/importance_sampling_ratio/max": 1.3170373082160949, "sampling/importance_sampling_ratio/mean": 1.0001026749610902, "sampling/importance_sampling_ratio/min": 0.7803112447261811, "sampling/sampling_logp_difference/max": 0.3029950797557831, "sampling/sampling_logp_difference/mean": 0.02015311736613512, "step": 2450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.2, "completions/max_terminated_length": 258.2, "completions/mean_length": 191.875, "completions/mean_terminated_length": 191.875, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 1.0560759492218494, "epoch": 0.22616530293279397, "frac_reward_zero_std": 0.3, "grad_norm": 8.39122486114502, "kl": 0.037859664915595205, "learning_rate": 4.991224514170232e-06, "loss": 0.0016, "num_tokens": 10702694.0, "reward": 0.075, "reward_std": 0.38138206005096437, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.38138206005096437, "sampling/importance_sampling_ratio/max": 1.2704686760902404, "sampling/importance_sampling_ratio/mean": 0.999655568599701, "sampling/importance_sampling_ratio/min": 0.7703635215759277, "sampling/sampling_logp_difference/max": 0.2721349120140076, "sampling/sampling_logp_difference/mean": 0.020930757001042365, "step": 2460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.0, "completions/max_terminated_length": 288.0, "completions/mean_length": 189.975, "completions/mean_terminated_length": 189.975, "completions/min_length": 110.1, "completions/min_terminated_length": 110.1, "entropy": 1.0435327395796776, "epoch": 0.2270846740829273, "frac_reward_zero_std": 0.4, "grad_norm": 6.204001426696777, "kl": 0.046926383359823375, "learning_rate": 4.9910111345417785e-06, "loss": 0.0024, "num_tokens": 10744932.0, "reward": 0.13125, "reward_std": 0.2790661096572876, "rewards/reward_fn/mean": 0.13125, "rewards/reward_fn/std": 0.2790661096572876, "sampling/importance_sampling_ratio/max": 1.2310782432556153, "sampling/importance_sampling_ratio/mean": 1.0001035153865814, "sampling/importance_sampling_ratio/min": 0.7943857610225677, "sampling/sampling_logp_difference/max": 0.2332480013370514, "sampling/sampling_logp_difference/mean": 0.020773524418473244, "step": 2470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.5, "completions/max_terminated_length": 262.5, "completions/mean_length": 183.1875, "completions/mean_terminated_length": 183.1875, "completions/min_length": 120.3, "completions/min_terminated_length": 120.3, "entropy": 1.022875565290451, "epoch": 0.2280040452330606, "frac_reward_zero_std": 0.2, "grad_norm": 6.991796970367432, "kl": 0.0434352322074119, "learning_rate": 4.990795196445212e-06, "loss": 0.0015, "num_tokens": 10783203.0, "reward": 0.025, "reward_std": 0.45646098256111145, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.45646100044250487, "sampling/importance_sampling_ratio/max": 1.2591790556907654, "sampling/importance_sampling_ratio/mean": 1.000213861465454, "sampling/importance_sampling_ratio/min": 0.7987667083740234, "sampling/sampling_logp_difference/max": 0.2563947081565857, "sampling/sampling_logp_difference/mean": 0.02045534607023001, "step": 2480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 187.45, "completions/mean_terminated_length": 187.45, "completions/min_length": 113.9, "completions/min_terminated_length": 113.9, "entropy": 1.0187353186309338, "epoch": 0.22892341638319388, "frac_reward_zero_std": 0.3, "grad_norm": 5.596317768096924, "kl": 0.031543022918049245, "learning_rate": 4.990576700102316e-06, "loss": 0.0012, "num_tokens": 10822711.0, "reward": 0.1, "reward_std": 0.4136129081249237, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.41361292004585265, "sampling/importance_sampling_ratio/max": 1.2634188771247863, "sampling/importance_sampling_ratio/mean": 1.0004124641418457, "sampling/importance_sampling_ratio/min": 0.7752062618732453, "sampling/sampling_logp_difference/max": 0.2689652442932129, "sampling/sampling_logp_difference/mean": 0.020628484524786474, "step": 2490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.7, "completions/max_terminated_length": 273.7, "completions/mean_length": 188.8375, "completions/mean_terminated_length": 188.8375, "completions/min_length": 125.1, "completions/min_terminated_length": 125.1, "entropy": 0.9977653473615646, "epoch": 0.2298427875333272, "frac_reward_zero_std": 0.2, "grad_norm": 6.077992916107178, "kl": 0.04374045781441964, "learning_rate": 4.990355645737506e-06, "loss": 0.002, "num_tokens": 10863234.0, "reward": 0.1625, "reward_std": 0.4556564152240753, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.45565642714500426, "sampling/importance_sampling_ratio/max": 1.2421785593032837, "sampling/importance_sampling_ratio/mean": 1.0000094532966615, "sampling/importance_sampling_ratio/min": 0.798731905221939, "sampling/sampling_logp_difference/max": 0.24611550569534302, "sampling/sampling_logp_difference/mean": 0.019924255460500716, "step": 2500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.4, "completions/max_terminated_length": 291.4, "completions/mean_length": 209.8125, "completions/mean_terminated_length": 209.8125, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.073886164277792, "epoch": 0.2307621586834605, "frac_reward_zero_std": 0.2, "grad_norm": 6.000349521636963, "kl": 0.03493021497270092, "learning_rate": 4.99013203357782e-06, "loss": 0.0018, "num_tokens": 10906875.0, "reward": -0.1375, "reward_std": 0.4305500477552414, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.43055006563663484, "sampling/importance_sampling_ratio/max": 1.2677898645401, "sampling/importance_sampling_ratio/mean": 0.9999549925327301, "sampling/importance_sampling_ratio/min": 0.7775612354278565, "sampling/sampling_logp_difference/max": 0.27662190794944763, "sampling/sampling_logp_difference/mean": 0.02086197044700384, "step": 2510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.4, "completions/max_terminated_length": 313.4, "completions/mean_length": 218.3125, "completions/mean_terminated_length": 218.3125, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.1404215224087237, "epoch": 0.23168152983359383, "frac_reward_zero_std": 0.3, "grad_norm": 4.994783878326416, "kl": 0.030836883222218604, "learning_rate": 4.989905863852928e-06, "loss": 0.0011, "num_tokens": 10948908.0, "reward": -0.2125, "reward_std": 0.3935216277837753, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.3935216277837753, "sampling/importance_sampling_ratio/max": 1.2588043093681336, "sampling/importance_sampling_ratio/mean": 1.0003569304943085, "sampling/importance_sampling_ratio/min": 0.7848890483379364, "sampling/sampling_logp_difference/max": 0.2565630555152893, "sampling/sampling_logp_difference/mean": 0.021893820166587828, "step": 2520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.7, "completions/max_terminated_length": 265.7, "completions/mean_length": 200.6875, "completions/mean_terminated_length": 200.6875, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.1183612763881683, "epoch": 0.23260090098372713, "frac_reward_zero_std": 0.2, "grad_norm": 7.919711589813232, "kl": 0.03401488094823435, "learning_rate": 4.989677136795123e-06, "loss": 0.0018, "num_tokens": 10989299.0, "reward": -0.04375, "reward_std": 0.44345162659883497, "rewards/reward_fn/mean": -0.04375, "rewards/reward_fn/std": 0.4434516325592995, "sampling/importance_sampling_ratio/max": 1.2850476384162903, "sampling/importance_sampling_ratio/mean": 0.9999486744403839, "sampling/importance_sampling_ratio/min": 0.7547445714473724, "sampling/sampling_logp_difference/max": 0.3379413396120071, "sampling/sampling_logp_difference/mean": 0.02173040695488453, "step": 2530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 218.45, "completions/mean_terminated_length": 218.45, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.1781387448310852, "epoch": 0.23352027213386045, "frac_reward_zero_std": 0.1, "grad_norm": 5.417177200317383, "kl": 0.03935691156657413, "learning_rate": 4.989445852639328e-06, "loss": 0.0018, "num_tokens": 11036847.0, "reward": -0.175, "reward_std": 0.5068073570728302, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.5068073749542237, "sampling/importance_sampling_ratio/max": 1.3013556718826294, "sampling/importance_sampling_ratio/mean": 0.999855923652649, "sampling/importance_sampling_ratio/min": 0.7862894415855408, "sampling/sampling_logp_difference/max": 0.27718343734741213, "sampling/sampling_logp_difference/mean": 0.021141526848077775, "step": 2540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.3, "completions/max_terminated_length": 288.3, "completions/mean_length": 195.7875, "completions/mean_terminated_length": 195.7875, "completions/min_length": 112.5, "completions/min_terminated_length": 112.5, "entropy": 1.145099437236786, "epoch": 0.23443964328399375, "frac_reward_zero_std": 0.2, "grad_norm": 6.554821491241455, "kl": 0.050792729353997854, "learning_rate": 4.989212011623087e-06, "loss": 0.003, "num_tokens": 11080366.0, "reward": 0.2375, "reward_std": 0.3682431042194366, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.36824311017990113, "sampling/importance_sampling_ratio/max": 1.3067343711853028, "sampling/importance_sampling_ratio/mean": 1.0003229796886444, "sampling/importance_sampling_ratio/min": 0.7865084171295166, "sampling/sampling_logp_difference/max": 0.2849071264266968, "sampling/sampling_logp_difference/mean": 0.02231056559830904, "step": 2550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.7, "completions/max_terminated_length": 304.7, "completions/mean_length": 204.5875, "completions/mean_terminated_length": 204.5875, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.118402624130249, "epoch": 0.23535901443412705, "frac_reward_zero_std": 0.2, "grad_norm": 5.418643474578857, "kl": 0.033206678985152394, "learning_rate": 4.988975613986575e-06, "loss": 0.0015, "num_tokens": 11124397.0, "reward": 0.2625, "reward_std": 0.4032860606908798, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.4032860606908798, "sampling/importance_sampling_ratio/max": 1.2581639528274535, "sampling/importance_sampling_ratio/mean": 1.0003222167491912, "sampling/importance_sampling_ratio/min": 0.7984795153141022, "sampling/sampling_logp_difference/max": 0.24500706195831298, "sampling/sampling_logp_difference/mean": 0.02116144709289074, "step": 2560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.1, "completions/max_terminated_length": 294.1, "completions/mean_length": 216.1625, "completions/mean_terminated_length": 216.1625, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0753696747124195, "epoch": 0.23627838558426037, "frac_reward_zero_std": 0.1, "grad_norm": 5.658927917480469, "kl": 0.0454733666498214, "learning_rate": 4.988736659972593e-06, "loss": 0.0026, "num_tokens": 11170010.0, "reward": -0.16875, "reward_std": 0.4471833348274231, "rewards/reward_fn/mean": -0.16875, "rewards/reward_fn/std": 0.44718335270881654, "sampling/importance_sampling_ratio/max": 1.2791348099708557, "sampling/importance_sampling_ratio/mean": 1.0001384019851685, "sampling/importance_sampling_ratio/min": 0.7986905455589295, "sampling/sampling_logp_difference/max": 0.24947422742843628, "sampling/sampling_logp_difference/mean": 0.020509620755910875, "step": 2570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 200.8125, "completions/mean_terminated_length": 200.8125, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.018815215677023, "epoch": 0.23719775673439367, "frac_reward_zero_std": 0.3, "grad_norm": 0.1251971423625946, "kl": 0.03866477177361958, "learning_rate": 4.988495149826563e-06, "loss": 0.0021, "num_tokens": 11215067.0, "reward": 0.3375, "reward_std": 0.40160019397735597, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.40160020589828493, "sampling/importance_sampling_ratio/max": 1.300172507762909, "sampling/importance_sampling_ratio/mean": 0.9998606741428375, "sampling/importance_sampling_ratio/min": 0.7573921799659729, "sampling/sampling_logp_difference/max": 0.3145493656396866, "sampling/sampling_logp_difference/mean": 0.02019682712852955, "step": 2580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.2, "completions/max_terminated_length": 284.2, "completions/mean_length": 189.3375, "completions/mean_terminated_length": 189.3375, "completions/min_length": 121.9, "completions/min_terminated_length": 121.9, "entropy": 1.0066481456160545, "epoch": 0.238117127884527, "frac_reward_zero_std": 0.1, "grad_norm": 5.907307147979736, "kl": 0.04344964168849401, "learning_rate": 4.988251083796539e-06, "loss": 0.0024, "num_tokens": 11257534.0, "reward": -0.0125, "reward_std": 0.4938108593225479, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4938108831644058, "sampling/importance_sampling_ratio/max": 1.2580394625663758, "sampling/importance_sampling_ratio/mean": 1.0001824855804444, "sampling/importance_sampling_ratio/min": 0.7979860544204712, "sampling/sampling_logp_difference/max": 0.24451079964637756, "sampling/sampling_logp_difference/mean": 0.020432468689978122, "step": 2590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 197.4875, "completions/mean_terminated_length": 197.4875, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.098967546224594, "epoch": 0.2390364990346603, "frac_reward_zero_std": 0.6, "grad_norm": 7.8013458251953125, "kl": 0.038584265240933745, "learning_rate": 4.988004462133192e-06, "loss": 0.0021, "num_tokens": 11300565.0, "reward": 0.3, "reward_std": 0.2140781342983246, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.21407814025878907, "sampling/importance_sampling_ratio/max": 1.2968656063079833, "sampling/importance_sampling_ratio/mean": 0.9996149718761445, "sampling/importance_sampling_ratio/min": 0.7863863348960877, "sampling/sampling_logp_difference/max": 0.2896435022354126, "sampling/sampling_logp_difference/mean": 0.02172896321862936, "step": 2600 }, { "epoch": 0.2390364990346603, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 280.4066317626527, "eval_completions/max_terminated_length": 279.44153577661433, "eval_completions/mean_length": 198.2931937172775, "eval_completions/mean_terminated_length": 197.99836912038646, "eval_completions/min_length": 126.51657940663176, "eval_completions/min_terminated_length": 126.51657940663176, "eval_entropy": 1.105177756067346, "eval_frac_reward_zero_std": 0.32460732984293195, "eval_kl": 0.048586749424589036, "eval_loss": 0.0025696163065731525, "eval_num_tokens": 11300565.0, "eval_reward": 0.06140924956369982, "eval_reward_std": 0.33572474153254045, "eval_rewards/reward_fn/mean": 0.06140924956369982, "eval_rewards/reward_fn/std": 0.3357247505824603, "eval_runtime": 1649.9815, "eval_samples_per_second": 0.347, "eval_sampling/importance_sampling_ratio/max": 1.2648652752448544, "eval_sampling/importance_sampling_ratio/mean": 0.9999690453211466, "eval_sampling/importance_sampling_ratio/min": 0.7784229363030372, "eval_sampling/sampling_logp_difference/max": 0.27980934148475556, "eval_sampling/sampling_logp_difference/mean": 0.021116749835355015, "eval_steps_per_second": 0.044, "step": 2600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.2, "completions/max_terminated_length": 268.2, "completions/mean_length": 185.9625, "completions/mean_terminated_length": 185.9625, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 0.964335186779499, "epoch": 0.2399558701847936, "frac_reward_zero_std": 0.6, "grad_norm": 0.13531634211540222, "kl": 0.04779042224399745, "learning_rate": 4.9877552850898255e-06, "loss": 0.0023, "num_tokens": 11337506.0, "reward": 0.2625, "reward_std": 0.15235702097415924, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.15235702097415924, "sampling/importance_sampling_ratio/max": 1.2524919152259826, "sampling/importance_sampling_ratio/mean": 0.9994821786880493, "sampling/importance_sampling_ratio/min": 0.7655371785163879, "sampling/sampling_logp_difference/max": 0.28164924383163453, "sampling/sampling_logp_difference/mean": 0.020220484212040903, "step": 2610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.4, "completions/max_terminated_length": 294.4, "completions/mean_length": 211.9625, "completions/mean_terminated_length": 211.9625, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0349913381040097, "epoch": 0.2408752413349269, "frac_reward_zero_std": 0.1, "grad_norm": 5.048066139221191, "kl": 0.04933973160805181, "learning_rate": 4.987503552922363e-06, "loss": 0.0022, "num_tokens": 11380551.0, "reward": 0.11875, "reward_std": 0.4711076617240906, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.47110766768455503, "sampling/importance_sampling_ratio/max": 1.2970452785491944, "sampling/importance_sampling_ratio/mean": 1.000468409061432, "sampling/importance_sampling_ratio/min": 0.7365161001682281, "sampling/sampling_logp_difference/max": 0.33642680644989015, "sampling/sampling_logp_difference/mean": 0.020787146501243116, "step": 2620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.5, "completions/max_terminated_length": 256.5, "completions/mean_length": 195.4625, "completions/mean_terminated_length": 195.4625, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.0658831246197225, "epoch": 0.2417946124850602, "frac_reward_zero_std": 0.4, "grad_norm": 6.394301414489746, "kl": 0.04424796831444837, "learning_rate": 4.987249265889355e-06, "loss": 0.0022, "num_tokens": 11423716.0, "reward": 0.0625, "reward_std": 0.34450737535953524, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.34450739324092866, "sampling/importance_sampling_ratio/max": 1.2958876371383667, "sampling/importance_sampling_ratio/mean": 1.0003436505794525, "sampling/importance_sampling_ratio/min": 0.7792130589485169, "sampling/sampling_logp_difference/max": 0.27105374336242677, "sampling/sampling_logp_difference/mean": 0.021434491500258446, "step": 2630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.0, "completions/max_terminated_length": 289.0, "completions/mean_length": 203.8875, "completions/mean_terminated_length": 203.8875, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0949511714279652, "epoch": 0.24271398363519353, "frac_reward_zero_std": 0.5, "grad_norm": 0.1297372430562973, "kl": 0.03642780394293368, "learning_rate": 4.9869924242519715e-06, "loss": 0.0017, "num_tokens": 11466379.0, "reward": 0.0125, "reward_std": 0.24982124865055083, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.24982124865055083, "sampling/importance_sampling_ratio/max": 1.2476243257522583, "sampling/importance_sampling_ratio/mean": 0.9994765996932984, "sampling/importance_sampling_ratio/min": 0.7869486212730408, "sampling/sampling_logp_difference/max": 0.2617160797119141, "sampling/sampling_logp_difference/mean": 0.021958728693425654, "step": 2640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.9, "completions/max_terminated_length": 292.9, "completions/mean_length": 197.0875, "completions/mean_terminated_length": 197.0875, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.151274362206459, "epoch": 0.24363335478532683, "frac_reward_zero_std": 0.2, "grad_norm": 5.6219801902771, "kl": 0.038672633573878554, "learning_rate": 4.986733028274013e-06, "loss": 0.002, "num_tokens": 11509154.0, "reward": 0.0625, "reward_std": 0.46739681661128996, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.4673968285322189, "sampling/importance_sampling_ratio/max": 1.2946714997291564, "sampling/importance_sampling_ratio/mean": 1.0000390529632568, "sampling/importance_sampling_ratio/min": 0.8016185164451599, "sampling/sampling_logp_difference/max": 0.2692372381687164, "sampling/sampling_logp_difference/mean": 0.022010532766580583, "step": 2650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.4, "completions/max_terminated_length": 281.4, "completions/mean_length": 197.125, "completions/mean_terminated_length": 197.125, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.1257120065391064, "epoch": 0.24455272593546015, "frac_reward_zero_std": 0.5, "grad_norm": 7.526282787322998, "kl": 0.0321127021801658, "learning_rate": 4.986471078221898e-06, "loss": 0.0017, "num_tokens": 11555284.0, "reward": -0.0875, "reward_std": 0.2658330500125885, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.26583306193351747, "sampling/importance_sampling_ratio/max": 1.226505410671234, "sampling/importance_sampling_ratio/mean": 0.9999152839183807, "sampling/importance_sampling_ratio/min": 0.77623330950737, "sampling/sampling_logp_difference/max": 0.2550153940916061, "sampling/sampling_logp_difference/mean": 0.02201693318784237, "step": 2660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 198.05, "completions/mean_terminated_length": 198.05, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0493835730478167, "epoch": 0.24547209708559345, "frac_reward_zero_std": 0.2, "grad_norm": 0.10485733300447464, "kl": 0.0424958658462856, "learning_rate": 4.986206574364671e-06, "loss": 0.002, "num_tokens": 11600976.0, "reward": 0.2, "reward_std": 0.5227612406015396, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.5227612584829331, "sampling/importance_sampling_ratio/max": 1.309563684463501, "sampling/importance_sampling_ratio/mean": 1.0002075254917144, "sampling/importance_sampling_ratio/min": 0.7929507851600647, "sampling/sampling_logp_difference/max": 0.28748188018798826, "sampling/sampling_logp_difference/mean": 0.02090072529390454, "step": 2670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.2, "completions/max_terminated_length": 271.2, "completions/mean_length": 192.9125, "completions/mean_terminated_length": 192.9125, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.05034648925066, "epoch": 0.24639146823572677, "frac_reward_zero_std": 0.3, "grad_norm": 7.577430248260498, "kl": 0.03439225670299493, "learning_rate": 4.985939516973996e-06, "loss": 0.0016, "num_tokens": 11641457.0, "reward": 0.13125, "reward_std": 0.46862156093120577, "rewards/reward_fn/mean": 0.13125, "rewards/reward_fn/std": 0.46862159073352816, "sampling/importance_sampling_ratio/max": 1.2239811897277832, "sampling/importance_sampling_ratio/mean": 1.0003397941589356, "sampling/importance_sampling_ratio/min": 0.7566246837377548, "sampling/sampling_logp_difference/max": 0.29382072687149047, "sampling/sampling_logp_difference/mean": 0.021501031704246998, "step": 2680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 240.6, "completions/max_terminated_length": 240.6, "completions/mean_length": 174.7125, "completions/mean_terminated_length": 174.7125, "completions/min_length": 107.1, "completions/min_terminated_length": 107.1, "entropy": 0.9959322221577167, "epoch": 0.24731083938586007, "frac_reward_zero_std": 0.6, "grad_norm": 9.488479614257812, "kl": 0.030505432945210485, "learning_rate": 4.985669906324167e-06, "loss": 0.0015, "num_tokens": 11681402.0, "reward": 0.2625, "reward_std": 0.28032026886940004, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.280320280790329, "sampling/importance_sampling_ratio/max": 1.275475299358368, "sampling/importance_sampling_ratio/mean": 1.0008409023284912, "sampling/importance_sampling_ratio/min": 0.7753716886043549, "sampling/sampling_logp_difference/max": 0.27277711033821106, "sampling/sampling_logp_difference/mean": 0.020704777911305428, "step": 2690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 214.4375, "completions/mean_terminated_length": 214.4375, "completions/min_length": 152.3, "completions/min_terminated_length": 152.3, "entropy": 1.0832776449620725, "epoch": 0.24823021053599337, "frac_reward_zero_std": 0.3, "grad_norm": 0.1169515922665596, "kl": 0.04267606807989068, "learning_rate": 4.985397742692093e-06, "loss": 0.002, "num_tokens": 11725829.0, "reward": -0.45, "reward_std": 0.31817767322063445, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.31817767918109896, "sampling/importance_sampling_ratio/max": 1.30653977394104, "sampling/importance_sampling_ratio/mean": 1.0001599073410035, "sampling/importance_sampling_ratio/min": 0.7692269086837769, "sampling/sampling_logp_difference/max": 0.3095731496810913, "sampling/sampling_logp_difference/mean": 0.021753770671784876, "step": 2700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.4, "completions/max_terminated_length": 259.4, "completions/mean_length": 197.3625, "completions/mean_terminated_length": 197.3625, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.0232362508773805, "epoch": 0.2491495816861267, "frac_reward_zero_std": 0.2, "grad_norm": 5.008146286010742, "kl": 0.05452837008051574, "learning_rate": 4.985123026357308e-06, "loss": 0.0027, "num_tokens": 11766626.0, "reward": 0.2, "reward_std": 0.408385643362999, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.40838564932346344, "sampling/importance_sampling_ratio/max": 1.2262384057044984, "sampling/importance_sampling_ratio/mean": 1.0001239717006682, "sampling/importance_sampling_ratio/min": 0.7899364292621612, "sampling/sampling_logp_difference/max": 0.24692018032073976, "sampling/sampling_logp_difference/mean": 0.020400654152035715, "step": 2710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.1, "completions/max_terminated_length": 258.1, "completions/mean_length": 202.7625, "completions/mean_terminated_length": 202.7625, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 0.9118117436766624, "epoch": 0.25006895283626, "frac_reward_zero_std": 0.3, "grad_norm": 6.27532434463501, "kl": 0.039663255779305476, "learning_rate": 4.9848457576019685e-06, "loss": 0.0023, "num_tokens": 11809999.0, "reward": 0.125, "reward_std": 0.439727446436882, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4397274643182755, "sampling/importance_sampling_ratio/max": 1.238251769542694, "sampling/importance_sampling_ratio/mean": 0.9999612867832184, "sampling/importance_sampling_ratio/min": 0.7949703752994537, "sampling/sampling_logp_difference/max": 0.25368083715438844, "sampling/sampling_logp_difference/mean": 0.0191750209312886, "step": 2720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.1, "completions/max_terminated_length": 300.1, "completions/mean_length": 207.4125, "completions/mean_terminated_length": 207.4125, "completions/min_length": 134.9, "completions/min_terminated_length": 134.9, "entropy": 1.0888357855379582, "epoch": 0.2509883239863933, "frac_reward_zero_std": 0.4, "grad_norm": 6.360905647277832, "kl": 0.03997602123417891, "learning_rate": 4.98456593671085e-06, "loss": 0.0022, "num_tokens": 11852880.0, "reward": 0.075, "reward_std": 0.3181816428899765, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.31818165481090543, "sampling/importance_sampling_ratio/max": 1.2487877368927003, "sampling/importance_sampling_ratio/mean": 1.0000269055366515, "sampling/importance_sampling_ratio/min": 0.7777351140975952, "sampling/sampling_logp_difference/max": 0.2791467189788818, "sampling/sampling_logp_difference/mean": 0.021814374439418315, "step": 2730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 226.6, "completions/mean_terminated_length": 226.6, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.1566815495491027, "epoch": 0.2519076951365266, "frac_reward_zero_std": 0.3, "grad_norm": 6.370908737182617, "kl": 0.049444525118451564, "learning_rate": 4.9842835639713525e-06, "loss": 0.0025, "num_tokens": 11895296.0, "reward": 0.0875, "reward_std": 0.3311904340982437, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.3311904460191727, "sampling/importance_sampling_ratio/max": 1.2461877465248108, "sampling/importance_sampling_ratio/mean": 1.000012892484665, "sampling/importance_sampling_ratio/min": 0.7453417599201202, "sampling/sampling_logp_difference/max": 0.30544832944869993, "sampling/sampling_logp_difference/mean": 0.02173358667641878, "step": 2740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 215.5125, "completions/mean_terminated_length": 215.5125, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.1005959428846837, "epoch": 0.2528270662866599, "frac_reward_zero_std": 0.1, "grad_norm": 6.106991291046143, "kl": 0.030825991154415533, "learning_rate": 4.983998639673496e-06, "loss": 0.0012, "num_tokens": 11942673.0, "reward": -0.0625, "reward_std": 0.4530158966779709, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.4530159145593643, "sampling/importance_sampling_ratio/max": 1.277392816543579, "sampling/importance_sampling_ratio/mean": 1.0000280022621155, "sampling/importance_sampling_ratio/min": 0.7524430990219116, "sampling/sampling_logp_difference/max": 0.3119939923286438, "sampling/sampling_logp_difference/mean": 0.02174394391477108, "step": 2750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.1, "completions/max_terminated_length": 295.1, "completions/mean_length": 201.2875, "completions/mean_terminated_length": 201.2875, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0241153724491596, "epoch": 0.25374643743679326, "frac_reward_zero_std": 0.3, "grad_norm": 5.417367458343506, "kl": 0.02174271009862423, "learning_rate": 4.983711164109919e-06, "loss": 0.0008, "num_tokens": 11987288.0, "reward": 0.025, "reward_std": 0.38901832699775696, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.38901835680007935, "sampling/importance_sampling_ratio/max": 1.2437778472900392, "sampling/importance_sampling_ratio/mean": 0.999558013677597, "sampling/importance_sampling_ratio/min": 0.7623007476329804, "sampling/sampling_logp_difference/max": 0.2828593969345093, "sampling/sampling_logp_difference/mean": 0.020254597812891007, "step": 2760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 218.3, "completions/mean_terminated_length": 218.3, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.0376436173915864, "epoch": 0.25466580858692656, "frac_reward_zero_std": 0.3, "grad_norm": 6.323155879974365, "kl": 0.032405230321455744, "learning_rate": 4.983421137575882e-06, "loss": 0.0016, "num_tokens": 12032168.0, "reward": 0.1875, "reward_std": 0.36794218719005584, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.36794219315052035, "sampling/importance_sampling_ratio/max": 1.2814663410186768, "sampling/importance_sampling_ratio/mean": 1.0002431273460388, "sampling/importance_sampling_ratio/min": 0.7896285057067871, "sampling/sampling_logp_difference/max": 0.26454647183418273, "sampling/sampling_logp_difference/mean": 0.020564932003617288, "step": 2770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.6, "completions/max_terminated_length": 264.6, "completions/mean_length": 185.6875, "completions/mean_terminated_length": 185.6875, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.0407466940581798, "epoch": 0.25558517973705985, "frac_reward_zero_std": 0.5, "grad_norm": 0.12284121662378311, "kl": 0.030147026717895643, "learning_rate": 4.983128560369267e-06, "loss": 0.0015, "num_tokens": 12071335.0, "reward": 0.5875, "reward_std": 0.29839127957820893, "rewards/reward_fn/mean": 0.5875, "rewards/reward_fn/std": 0.29839127957820893, "sampling/importance_sampling_ratio/max": 1.2712011456489563, "sampling/importance_sampling_ratio/mean": 0.9999915659427643, "sampling/importance_sampling_ratio/min": 0.8019644439220428, "sampling/sampling_logp_difference/max": 0.2486160159111023, "sampling/sampling_logp_difference/mean": 0.020536273159086703, "step": 2780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.5, "completions/max_terminated_length": 262.5, "completions/mean_length": 182.2, "completions/mean_terminated_length": 182.2, "completions/min_length": 119.1, "completions/min_terminated_length": 119.1, "entropy": 0.982857346534729, "epoch": 0.25650455088719315, "frac_reward_zero_std": 0.2, "grad_norm": 7.815053939819336, "kl": 0.04034502828726545, "learning_rate": 4.982833432790573e-06, "loss": 0.0021, "num_tokens": 12108215.0, "reward": 0.075, "reward_std": 0.4472267836332321, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.44722679555416106, "sampling/importance_sampling_ratio/max": 1.2445849061012269, "sampling/importance_sampling_ratio/mean": 1.000073504447937, "sampling/importance_sampling_ratio/min": 0.7712437093257904, "sampling/sampling_logp_difference/max": 0.2701414227485657, "sampling/sampling_logp_difference/mean": 0.020210597664117813, "step": 2790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.7, "completions/max_terminated_length": 291.7, "completions/mean_length": 220.0875, "completions/mean_terminated_length": 220.0875, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.1246454559266568, "epoch": 0.25742392203732645, "frac_reward_zero_std": 0.0, "grad_norm": 5.9768147468566895, "kl": 0.031025142641738057, "learning_rate": 4.982535755142921e-06, "loss": 0.0019, "num_tokens": 12151054.0, "reward": -0.0625, "reward_std": 0.5694230645895004, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.5694230824708939, "sampling/importance_sampling_ratio/max": 1.3540500164031983, "sampling/importance_sampling_ratio/mean": 1.00035600066185, "sampling/importance_sampling_ratio/min": 0.7829943418502807, "sampling/sampling_logp_difference/max": 0.3097230434417725, "sampling/sampling_logp_difference/mean": 0.02193455081433058, "step": 2800 }, { "epoch": 0.25742392203732645, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 288.56893542757416, "eval_completions/max_terminated_length": 286.413612565445, "eval_completions/mean_length": 204.54886561954623, "eval_completions/mean_terminated_length": 204.09514478114264, "eval_completions/min_length": 137.4502617801047, "eval_completions/min_terminated_length": 137.4502617801047, "eval_entropy": 1.1007631256734305, "eval_frac_reward_zero_std": 0.27399650959860383, "eval_kl": 0.041012291155306926, "eval_loss": 0.0021502634044736624, "eval_num_tokens": 12151054.0, "eval_reward": 0.052683246073298426, "eval_reward_std": 0.38608684082617933, "eval_rewards/reward_fn/mean": 0.052683246073298426, "eval_rewards/reward_fn/std": 0.3860868508122978, "eval_runtime": 1689.5098, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2644031249623857, "eval_sampling/importance_sampling_ratio/mean": 1.000042509034042, "eval_sampling/importance_sampling_ratio/min": 0.7802122043897552, "eval_sampling/sampling_logp_difference/max": 0.27356586354357204, "eval_sampling/sampling_logp_difference/mean": 0.021417336515679203, "eval_steps_per_second": 0.043, "step": 2800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 239.4, "completions/max_terminated_length": 239.4, "completions/mean_length": 179.15, "completions/mean_terminated_length": 179.15, "completions/min_length": 120.7, "completions/min_terminated_length": 120.7, "entropy": 1.0156222365796566, "epoch": 0.2583432931874598, "frac_reward_zero_std": 0.4, "grad_norm": 5.617149353027344, "kl": 0.04966756717767566, "learning_rate": 4.9822355277320475e-06, "loss": 0.0024, "num_tokens": 12191274.0, "reward": 0.325, "reward_std": 0.30269542038440705, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.302695432305336, "sampling/importance_sampling_ratio/max": 1.2694220304489137, "sampling/importance_sampling_ratio/mean": 1.0000813603401184, "sampling/importance_sampling_ratio/min": 0.7904092967510223, "sampling/sampling_logp_difference/max": 0.2642872095108032, "sampling/sampling_logp_difference/mean": 0.02089224271476269, "step": 2810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.5, "completions/max_terminated_length": 278.5, "completions/mean_length": 192.5875, "completions/mean_terminated_length": 192.5875, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.040990813449025, "epoch": 0.2592626643375931, "frac_reward_zero_std": 0.1, "grad_norm": 2.842435359954834, "kl": 0.04421546008961741, "learning_rate": 4.981932750866313e-06, "loss": 0.0026, "num_tokens": 12233257.0, "reward": 0.125, "reward_std": 0.45548935830593107, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.45548937022686004, "sampling/importance_sampling_ratio/max": 1.2264577865600585, "sampling/importance_sampling_ratio/mean": 1.0000287413597106, "sampling/importance_sampling_ratio/min": 0.7567507982254028, "sampling/sampling_logp_difference/max": 0.28429675102233887, "sampling/sampling_logp_difference/mean": 0.021971827000379564, "step": 2820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 206.8625, "completions/mean_terminated_length": 206.8625, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.071357686072588, "epoch": 0.2601820354877264, "frac_reward_zero_std": 0.2, "grad_norm": 6.80636739730835, "kl": 0.03778555438038893, "learning_rate": 4.981627424856691e-06, "loss": 0.0023, "num_tokens": 12276054.0, "reward": 0.1375, "reward_std": 0.3947398692369461, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.39473988115787506, "sampling/importance_sampling_ratio/max": 1.2608086943626404, "sampling/importance_sampling_ratio/mean": 0.9997180044651032, "sampling/importance_sampling_ratio/min": 0.7502443373203278, "sampling/sampling_logp_difference/max": 0.3030838966369629, "sampling/sampling_logp_difference/mean": 0.021685385145246983, "step": 2830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.5, "completions/max_terminated_length": 320.5, "completions/mean_length": 218.575, "completions/mean_terminated_length": 218.575, "completions/min_length": 152.7, "completions/min_terminated_length": 152.7, "entropy": 1.0965552292764187, "epoch": 0.2611014066378597, "frac_reward_zero_std": 0.4, "grad_norm": 0.10269385576248169, "kl": 0.050288640061626214, "learning_rate": 4.981319550016777e-06, "loss": 0.0028, "num_tokens": 12324596.0, "reward": 0.05, "reward_std": 0.28629584312438966, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.2862958490848541, "sampling/importance_sampling_ratio/max": 1.244599413871765, "sampling/importance_sampling_ratio/mean": 0.9998269200325012, "sampling/importance_sampling_ratio/min": 0.7803301751613617, "sampling/sampling_logp_difference/max": 0.2602036714553833, "sampling/sampling_logp_difference/mean": 0.021378837898373602, "step": 2840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.1, "completions/max_terminated_length": 333.1, "completions/mean_length": 221.8625, "completions/mean_terminated_length": 221.8625, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.1194487735629082, "epoch": 0.262020777787993, "frac_reward_zero_std": 0.3, "grad_norm": 5.97575044631958, "kl": 0.05989346381975338, "learning_rate": 4.9810091266627835e-06, "loss": 0.0031, "num_tokens": 12369417.0, "reward": -0.1875, "reward_std": 0.3257186770439148, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.32571869492530825, "sampling/importance_sampling_ratio/max": 1.2479507446289062, "sampling/importance_sampling_ratio/mean": 0.999977195262909, "sampling/importance_sampling_ratio/min": 0.75471031665802, "sampling/sampling_logp_difference/max": 0.2949321448802948, "sampling/sampling_logp_difference/mean": 0.021055320836603643, "step": 2850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.5, "completions/max_terminated_length": 278.5, "completions/mean_length": 198.2375, "completions/mean_terminated_length": 198.2375, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.0531558103859424, "epoch": 0.26294014893812634, "frac_reward_zero_std": 0.2, "grad_norm": 5.708948135375977, "kl": 0.036559860524721446, "learning_rate": 4.980696155113539e-06, "loss": 0.0025, "num_tokens": 12412564.0, "reward": -0.1, "reward_std": 0.3446744382381439, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3446744501590729, "sampling/importance_sampling_ratio/max": 1.2376705050468444, "sampling/importance_sampling_ratio/mean": 0.9998081743717193, "sampling/importance_sampling_ratio/min": 0.7684375524520874, "sampling/sampling_logp_difference/max": 0.27625452280044555, "sampling/sampling_logp_difference/mean": 0.02165361903607845, "step": 2860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.4, "completions/max_terminated_length": 259.4, "completions/mean_length": 197.525, "completions/mean_terminated_length": 197.525, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.1611466467380525, "epoch": 0.26385952008825964, "frac_reward_zero_std": 0.3, "grad_norm": 0.18947243690490723, "kl": 0.03421938415849581, "learning_rate": 4.980380635690492e-06, "loss": 0.0022, "num_tokens": 12456510.0, "reward": 0.15, "reward_std": 0.35464063584804534, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3546406477689743, "sampling/importance_sampling_ratio/max": 1.3299927473068238, "sampling/importance_sampling_ratio/mean": 1.0002754747867584, "sampling/importance_sampling_ratio/min": 0.7863232970237732, "sampling/sampling_logp_difference/max": 0.30860941410064696, "sampling/sampling_logp_difference/mean": 0.022235395200550557, "step": 2870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.2, "completions/max_terminated_length": 264.2, "completions/mean_length": 184.4875, "completions/mean_terminated_length": 184.4875, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 1.1276733413338662, "epoch": 0.26477889123839293, "frac_reward_zero_std": 0.3, "grad_norm": 6.813807010650635, "kl": 0.055635694501688704, "learning_rate": 4.980062568717704e-06, "loss": 0.0028, "num_tokens": 12495869.0, "reward": 0.2125, "reward_std": 0.3435225158929825, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.34352252185344695, "sampling/importance_sampling_ratio/max": 1.2383883357048036, "sampling/importance_sampling_ratio/mean": 0.9991217076778411, "sampling/importance_sampling_ratio/min": 0.7905645251274109, "sampling/sampling_logp_difference/max": 0.2475505828857422, "sampling/sampling_logp_difference/mean": 0.0216397263109684, "step": 2880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 291.6, "completions/max_terminated_length": 282.5, "completions/mean_length": 204.55, "completions/mean_terminated_length": 201.44642944335936, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0076976522803307, "epoch": 0.26569826238852623, "frac_reward_zero_std": 0.4, "grad_norm": 0.1592673659324646, "kl": 0.04144300357438624, "learning_rate": 4.979741954521857e-06, "loss": 0.0021, "num_tokens": 12537721.0, "reward": 0.26875, "reward_std": 0.355224609375, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.35522462129592897, "sampling/importance_sampling_ratio/max": 1.2800498366355897, "sampling/importance_sampling_ratio/mean": 0.9999342679977417, "sampling/importance_sampling_ratio/min": 0.7533438205718994, "sampling/sampling_logp_difference/max": 0.3110794723033905, "sampling/sampling_logp_difference/mean": 0.021210421435534955, "step": 2890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.1, "completions/max_terminated_length": 279.1, "completions/mean_length": 184.8875, "completions/mean_terminated_length": 184.8875, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 0.9692872066050768, "epoch": 0.2666176335386596, "frac_reward_zero_std": 0.1, "grad_norm": 5.114948272705078, "kl": 0.05371689720777795, "learning_rate": 4.979418793432248e-06, "loss": 0.0026, "num_tokens": 12581248.0, "reward": 0.0125, "reward_std": 0.5004733771085739, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.5004734069108963, "sampling/importance_sampling_ratio/max": 1.2769236803054809, "sampling/importance_sampling_ratio/mean": 1.0001907169818878, "sampling/importance_sampling_ratio/min": 0.7716196656227112, "sampling/sampling_logp_difference/max": 0.2743303418159485, "sampling/sampling_logp_difference/mean": 0.020811568945646286, "step": 2900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.4, "completions/max_terminated_length": 296.4, "completions/mean_length": 213.3, "completions/mean_terminated_length": 213.3, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.0639724999666214, "epoch": 0.2675370046887929, "frac_reward_zero_std": 0.5, "grad_norm": 6.853530406951904, "kl": 0.05208175663137808, "learning_rate": 4.979093085780788e-06, "loss": 0.0024, "num_tokens": 12627784.0, "reward": -0.0625, "reward_std": 0.20580926835536956, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.20580926835536956, "sampling/importance_sampling_ratio/max": 1.2603452920913696, "sampling/importance_sampling_ratio/mean": 1.0005508840084076, "sampling/importance_sampling_ratio/min": 0.7826572835445404, "sampling/sampling_logp_difference/max": 0.2577659010887146, "sampling/sampling_logp_difference/mean": 0.02154218889772892, "step": 2910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 214.8375, "completions/mean_terminated_length": 214.8375, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.049601886421442, "epoch": 0.2684563758389262, "frac_reward_zero_std": 0.4, "grad_norm": 0.14672520756721497, "kl": 0.058840165205765516, "learning_rate": 4.978764831902005e-06, "loss": 0.0031, "num_tokens": 12672515.0, "reward": 0.075, "reward_std": 0.31564186215400697, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.31564187407493594, "sampling/importance_sampling_ratio/max": 1.269123363494873, "sampling/importance_sampling_ratio/mean": 0.9999415457248688, "sampling/importance_sampling_ratio/min": 0.7534095704555511, "sampling/sampling_logp_difference/max": 0.2956400036811829, "sampling/sampling_logp_difference/mean": 0.020669311843812467, "step": 2920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.9, "completions/max_terminated_length": 245.9, "completions/mean_length": 177.8125, "completions/mean_terminated_length": 177.8125, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 1.0855563215911388, "epoch": 0.2693757469890595, "frac_reward_zero_std": 0.1, "grad_norm": 7.849121570587158, "kl": 0.0681488914298825, "learning_rate": 4.978434032133042e-06, "loss": 0.0035, "num_tokens": 12709956.0, "reward": -0.025, "reward_std": 0.42207511365413664, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4220751315355301, "sampling/importance_sampling_ratio/max": 1.2360840797424317, "sampling/importance_sampling_ratio/mean": 0.9993714809417724, "sampling/importance_sampling_ratio/min": 0.7885816752910614, "sampling/sampling_logp_difference/max": 0.25503069162368774, "sampling/sampling_logp_difference/mean": 0.021899352408945562, "step": 2930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.8, "completions/max_terminated_length": 294.8, "completions/mean_length": 222.85, "completions/mean_terminated_length": 222.85, "completions/min_length": 156.4, "completions/min_terminated_length": 156.4, "entropy": 1.036701576411724, "epoch": 0.27029511813919277, "frac_reward_zero_std": 0.2, "grad_norm": 4.780606269836426, "kl": 0.047939900774508715, "learning_rate": 4.978100686813658e-06, "loss": 0.0025, "num_tokens": 12753416.0, "reward": 0.10625, "reward_std": 0.39183932840824126, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.3918393403291702, "sampling/importance_sampling_ratio/max": 1.2341225862503051, "sampling/importance_sampling_ratio/mean": 0.9999453306198121, "sampling/importance_sampling_ratio/min": 0.7725021332502365, "sampling/sampling_logp_difference/max": 0.281782329082489, "sampling/sampling_logp_difference/mean": 0.020637968368828297, "step": 2940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.9, "completions/max_terminated_length": 310.9, "completions/mean_length": 213.575, "completions/mean_terminated_length": 213.575, "completions/min_length": 141.6, "completions/min_terminated_length": 141.6, "entropy": 1.2040755182504654, "epoch": 0.2712144892893261, "frac_reward_zero_std": 0.5, "grad_norm": 5.997399806976318, "kl": 0.06263853986747563, "learning_rate": 4.977764796286224e-06, "loss": 0.0032, "num_tokens": 12798078.0, "reward": -0.025, "reward_std": 0.22051151394844054, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2205115258693695, "sampling/importance_sampling_ratio/max": 1.2764033079147339, "sampling/importance_sampling_ratio/mean": 0.9998996794223786, "sampling/importance_sampling_ratio/min": 0.7966888904571533, "sampling/sampling_logp_difference/max": 0.2518589496612549, "sampling/sampling_logp_difference/mean": 0.022214872390031816, "step": 2950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 201.2875, "completions/mean_terminated_length": 201.2875, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.107544044405222, "epoch": 0.2721338604394594, "frac_reward_zero_std": 0.5, "grad_norm": 0.1241946667432785, "kl": 0.057853935711318624, "learning_rate": 4.977426360895727e-06, "loss": 0.0031, "num_tokens": 12840629.0, "reward": -0.1125, "reward_std": 0.2612558871507645, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.26125589907169344, "sampling/importance_sampling_ratio/max": 1.2495791912078857, "sampling/importance_sampling_ratio/mean": 1.0001029014587401, "sampling/importance_sampling_ratio/min": 0.7849660515785217, "sampling/sampling_logp_difference/max": 0.2641411066055298, "sampling/sampling_logp_difference/mean": 0.0222238564863801, "step": 2960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.4, "completions/max_terminated_length": 305.8, "completions/mean_length": 215.55, "completions/mean_terminated_length": 211.94464416503905, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 0.9620839280076325, "epoch": 0.2730532315895927, "frac_reward_zero_std": 0.4, "grad_norm": 4.880609512329102, "kl": 0.038240350448177196, "learning_rate": 4.977085380989768e-06, "loss": 0.0018, "num_tokens": 12883513.0, "reward": 0.26875, "reward_std": 0.3974449634552002, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.39744496941566465, "sampling/importance_sampling_ratio/max": 1.2809149622917175, "sampling/importance_sampling_ratio/mean": 1.0003820061683655, "sampling/importance_sampling_ratio/min": 0.7656985700130463, "sampling/sampling_logp_difference/max": 0.3023351788520813, "sampling/sampling_logp_difference/mean": 0.019760796707123517, "step": 2970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 207.2875, "completions/mean_terminated_length": 207.2875, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.024321373552084, "epoch": 0.273972602739726, "frac_reward_zero_std": 0.3, "grad_norm": 5.848912715911865, "kl": 0.045333441102411595, "learning_rate": 4.9767418569185585e-06, "loss": 0.0021, "num_tokens": 12924784.0, "reward": -0.025, "reward_std": 0.4164423167705536, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.41644232273101806, "sampling/importance_sampling_ratio/max": 1.2930202960968018, "sampling/importance_sampling_ratio/mean": 0.9998917162418366, "sampling/importance_sampling_ratio/min": 0.7350365877151489, "sampling/sampling_logp_difference/max": 0.3134430170059204, "sampling/sampling_logp_difference/mean": 0.0207748519256711, "step": 2980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.4, "completions/max_terminated_length": 332.4, "completions/mean_length": 235.3, "completions/mean_terminated_length": 235.3, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.1033256374299527, "epoch": 0.2748919738898593, "frac_reward_zero_std": 0.4, "grad_norm": 4.912093639373779, "kl": 0.04564914793591015, "learning_rate": 4.9763957890349284e-06, "loss": 0.0022, "num_tokens": 12971480.0, "reward": 0.075, "reward_std": 0.33515144884586334, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.33515146672725676, "sampling/importance_sampling_ratio/max": 1.2637401580810548, "sampling/importance_sampling_ratio/mean": 1.0002837836742402, "sampling/importance_sampling_ratio/min": 0.7673021078109741, "sampling/sampling_logp_difference/max": 0.2827598750591278, "sampling/sampling_logp_difference/mean": 0.02086910270154476, "step": 2990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 338.4, "completions/max_terminated_length": 313.9, "completions/mean_length": 213.5375, "completions/mean_terminated_length": 208.3446472167969, "completions/min_length": 122.1, "completions/min_terminated_length": 122.1, "entropy": 1.1471606727689505, "epoch": 0.27581134503999266, "frac_reward_zero_std": 0.3, "grad_norm": 4.2565531730651855, "kl": 0.030792669008951633, "learning_rate": 4.976047177694315e-06, "loss": 0.0019, "num_tokens": 13017027.0, "reward": -0.0375, "reward_std": 0.37291836440563203, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.3729183703660965, "sampling/importance_sampling_ratio/max": 1.255772602558136, "sampling/importance_sampling_ratio/mean": 0.9997790336608887, "sampling/importance_sampling_ratio/min": 0.7934587657451629, "sampling/sampling_logp_difference/max": 0.26108033657073976, "sampling/sampling_logp_difference/mean": 0.021706547029316424, "step": 3000 }, { "epoch": 0.27581134503999266, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002181500872600349, "eval_completions/max_length": 295.0732984293194, "eval_completions/max_terminated_length": 293.2216404886562, "eval_completions/mean_length": 202.85929319371726, "eval_completions/mean_terminated_length": 202.31337797953822, "eval_completions/min_length": 126.57766143106457, "eval_completions/min_terminated_length": 126.57766143106457, "eval_entropy": 1.0980144182632938, "eval_frac_reward_zero_std": 0.24083769633507854, "eval_kl": 0.03658795784053638, "eval_loss": 0.0018208450637757778, "eval_num_tokens": 13017027.0, "eval_reward": 0.037303664921465966, "eval_reward_std": 0.4122541890094418, "eval_rewards/reward_fn/mean": 0.037303664921465966, "eval_rewards/reward_fn/std": 0.41225420149208986, "eval_runtime": 1721.6143, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2836325915488065, "eval_sampling/importance_sampling_ratio/mean": 1.0000328719095826, "eval_sampling/importance_sampling_ratio/min": 0.7668360717737653, "eval_sampling/sampling_logp_difference/max": 0.2988978398615985, "eval_sampling/sampling_logp_difference/mean": 0.021316910227581885, "eval_steps_per_second": 0.042, "step": 3000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.4, "completions/max_terminated_length": 270.4, "completions/mean_length": 192.3875, "completions/mean_terminated_length": 192.3875, "completions/min_length": 114.7, "completions/min_terminated_length": 114.7, "entropy": 1.0674329996109009, "epoch": 0.27673071619012596, "frac_reward_zero_std": 0.0, "grad_norm": 6.307253837585449, "kl": 0.039002185128629206, "learning_rate": 4.975696023254771e-06, "loss": 0.0014, "num_tokens": 13055258.0, "reward": 0.0875, "reward_std": 0.5295886754989624, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.5295886993408203, "sampling/importance_sampling_ratio/max": 1.3257169485092164, "sampling/importance_sampling_ratio/mean": 0.9996242046356201, "sampling/importance_sampling_ratio/min": 0.7877074241638183, "sampling/sampling_logp_difference/max": 0.31526724100112913, "sampling/sampling_logp_difference/mean": 0.021550440788269044, "step": 3010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 342.9, "completions/max_terminated_length": 310.7, "completions/mean_length": 215.6, "completions/mean_terminated_length": 209.03036041259764, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.0915420100092887, "epoch": 0.27765008734025926, "frac_reward_zero_std": 0.1, "grad_norm": 6.749044895172119, "kl": 0.05413867571041919, "learning_rate": 4.97534232607696e-06, "loss": 0.0022, "num_tokens": 13098818.0, "reward": 0.0625, "reward_std": 0.5031409710645676, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.5031409829854965, "sampling/importance_sampling_ratio/max": 1.2760480880737304, "sampling/importance_sampling_ratio/mean": 0.9998669803142548, "sampling/importance_sampling_ratio/min": 0.7631634891033172, "sampling/sampling_logp_difference/max": 0.29209101796150205, "sampling/sampling_logp_difference/mean": 0.021610433980822564, "step": 3020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.0, "completions/max_terminated_length": 278.0, "completions/mean_length": 195.575, "completions/mean_terminated_length": 195.575, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.048918904364109, "epoch": 0.27856945849039255, "frac_reward_zero_std": 0.3, "grad_norm": 5.53798246383667, "kl": 0.0417088690912351, "learning_rate": 4.974986086524156e-06, "loss": 0.0021, "num_tokens": 13139744.0, "reward": -0.14375, "reward_std": 0.46859510242938995, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.4685951083898544, "sampling/importance_sampling_ratio/max": 1.2597753643989562, "sampling/importance_sampling_ratio/mean": 0.9999534487724304, "sampling/importance_sampling_ratio/min": 0.7457925081253052, "sampling/sampling_logp_difference/max": 0.31349319219589233, "sampling/sampling_logp_difference/mean": 0.020610079169273376, "step": 3030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 199.525, "completions/mean_terminated_length": 199.525, "completions/min_length": 117.5, "completions/min_terminated_length": 117.5, "entropy": 1.0540482595562934, "epoch": 0.2794888296405259, "frac_reward_zero_std": 0.1, "grad_norm": 6.633767604827881, "kl": 0.033208812162047255, "learning_rate": 4.974627304962247e-06, "loss": 0.0019, "num_tokens": 13182074.0, "reward": 0.34375, "reward_std": 0.5549404412508011, "rewards/reward_fn/mean": 0.34375, "rewards/reward_fn/std": 0.5549404472112656, "sampling/importance_sampling_ratio/max": 1.2974908113479615, "sampling/importance_sampling_ratio/mean": 0.999974274635315, "sampling/importance_sampling_ratio/min": 0.8051841020584106, "sampling/sampling_logp_difference/max": 0.2678371489048004, "sampling/sampling_logp_difference/mean": 0.020697659999132156, "step": 3040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.4, "completions/max_terminated_length": 264.4, "completions/mean_length": 179.4375, "completions/mean_terminated_length": 179.4375, "completions/min_length": 98.9, "completions/min_terminated_length": 98.9, "entropy": 1.0010355595499276, "epoch": 0.2804082007906592, "frac_reward_zero_std": 0.3, "grad_norm": 0.19045215845108032, "kl": 0.028951244370546192, "learning_rate": 4.97426598175973e-06, "loss": 0.0015, "num_tokens": 13222797.0, "reward": -0.1125, "reward_std": 0.3980518102645874, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.39805181622505187, "sampling/importance_sampling_ratio/max": 1.2700633764266969, "sampling/importance_sampling_ratio/mean": 1.0003682076931, "sampling/importance_sampling_ratio/min": 0.7980851233005524, "sampling/sampling_logp_difference/max": 0.2582335829734802, "sampling/sampling_logp_difference/mean": 0.019789416529238223, "step": 3050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.8, "completions/max_terminated_length": 285.8, "completions/mean_length": 199.2375, "completions/mean_terminated_length": 199.2375, "completions/min_length": 111.8, "completions/min_terminated_length": 111.8, "entropy": 1.1236065208911896, "epoch": 0.2813275719407925, "frac_reward_zero_std": 0.2, "grad_norm": 5.808686256408691, "kl": 0.04446217557415366, "learning_rate": 4.973902117287714e-06, "loss": 0.0025, "num_tokens": 13267168.0, "reward": -0.1125, "reward_std": 0.48544313609600065, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.48544314205646516, "sampling/importance_sampling_ratio/max": 1.2551715612411498, "sampling/importance_sampling_ratio/mean": 0.9999238669872283, "sampling/importance_sampling_ratio/min": 0.7100574348121882, "sampling/sampling_logp_difference/max": 0.5332852244377136, "sampling/sampling_logp_difference/mean": 0.022360038198530673, "step": 3060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.5, "completions/max_terminated_length": 288.5, "completions/mean_length": 190.3375, "completions/mean_terminated_length": 190.3375, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.0981203615665436, "epoch": 0.2822469430909258, "frac_reward_zero_std": 0.4, "grad_norm": 0.11504124104976654, "kl": 0.02490336262853816, "learning_rate": 4.9735357119199154e-06, "loss": 0.0012, "num_tokens": 13310067.0, "reward": 0.2125, "reward_std": 0.2711646556854248, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.2711646556854248, "sampling/importance_sampling_ratio/max": 1.2721507787704467, "sampling/importance_sampling_ratio/mean": 0.9998740136623383, "sampling/importance_sampling_ratio/min": 0.8018085181713104, "sampling/sampling_logp_difference/max": 0.25268272757530214, "sampling/sampling_logp_difference/mean": 0.021480437740683554, "step": 3070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.1, "completions/max_terminated_length": 285.1, "completions/mean_length": 204.1875, "completions/mean_terminated_length": 204.1875, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.120399809628725, "epoch": 0.2831663142410591, "frac_reward_zero_std": 0.3, "grad_norm": 6.795775413513184, "kl": 0.029315559909446166, "learning_rate": 4.9731667660326625e-06, "loss": 0.0016, "num_tokens": 13351354.0, "reward": -0.2, "reward_std": 0.45687578022480013, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.456875804066658, "sampling/importance_sampling_ratio/max": 1.2691314101219178, "sampling/importance_sampling_ratio/mean": 0.9995928943157196, "sampling/importance_sampling_ratio/min": 0.7629804670810699, "sampling/sampling_logp_difference/max": 0.2955487906932831, "sampling/sampling_logp_difference/mean": 0.021765482425689698, "step": 3080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.7, "completions/max_terminated_length": 282.7, "completions/mean_length": 210.95, "completions/mean_terminated_length": 210.95, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0443960294127463, "epoch": 0.28408568539119244, "frac_reward_zero_std": 0.4, "grad_norm": 5.066106796264648, "kl": 0.026497692969860508, "learning_rate": 4.972795280004893e-06, "loss": 0.0012, "num_tokens": 13395502.0, "reward": -0.05, "reward_std": 0.33574098944664, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.33574100136756896, "sampling/importance_sampling_ratio/max": 1.2967255115509033, "sampling/importance_sampling_ratio/mean": 1.0002476513385772, "sampling/importance_sampling_ratio/min": 0.7796294748783111, "sampling/sampling_logp_difference/max": 0.29486062526702883, "sampling/sampling_logp_difference/mean": 0.021191072650253773, "step": 3090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 210.4625, "completions/mean_terminated_length": 210.4625, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.1657846421003342, "epoch": 0.28500505654132574, "frac_reward_zero_std": 0.0, "grad_norm": 5.7392778396606445, "kl": 0.02931773483287543, "learning_rate": 4.972421254218151e-06, "loss": 0.0015, "num_tokens": 13438315.0, "reward": 0.025, "reward_std": 0.5231043308973312, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5231043308973312, "sampling/importance_sampling_ratio/max": 1.228700840473175, "sampling/importance_sampling_ratio/mean": 0.9998594582080841, "sampling/importance_sampling_ratio/min": 0.787255734205246, "sampling/sampling_logp_difference/max": 0.2500169575214386, "sampling/sampling_logp_difference/mean": 0.02262396290898323, "step": 3100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.4, "completions/max_terminated_length": 281.4, "completions/mean_length": 202.0125, "completions/mean_terminated_length": 198.0250015258789, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.0418601922690869, "epoch": 0.28592442769145904, "frac_reward_zero_std": 0.4, "grad_norm": 4.446115016937256, "kl": 0.042371876200195405, "learning_rate": 4.972044689056593e-06, "loss": 0.0023, "num_tokens": 13482820.0, "reward": 0.28125, "reward_std": 0.2689625948667526, "rewards/reward_fn/mean": 0.28125, "rewards/reward_fn/std": 0.2689626067876816, "sampling/importance_sampling_ratio/max": 1.2813395977020263, "sampling/importance_sampling_ratio/mean": 0.9995700478553772, "sampling/importance_sampling_ratio/min": 0.7875003457069397, "sampling/sampling_logp_difference/max": 0.2666512727737427, "sampling/sampling_logp_difference/mean": 0.020998050272464753, "step": 3110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 203.325, "completions/mean_terminated_length": 203.325, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 0.9960630960762501, "epoch": 0.28684379884159233, "frac_reward_zero_std": 0.0, "grad_norm": 5.162259101867676, "kl": 0.047720687976107, "learning_rate": 4.97166558490698e-06, "loss": 0.0023, "num_tokens": 13524614.0, "reward": 0.0125, "reward_std": 0.5707158267498016, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.5707158446311951, "sampling/importance_sampling_ratio/max": 1.2818321585655212, "sampling/importance_sampling_ratio/mean": 1.000101637840271, "sampling/importance_sampling_ratio/min": 0.7687285840511322, "sampling/sampling_logp_difference/max": 0.28715428709983826, "sampling/sampling_logp_difference/mean": 0.02036844249814749, "step": 3120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.8, "completions/max_terminated_length": 246.8, "completions/mean_length": 180.5, "completions/mean_terminated_length": 180.5, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 0.9635615319013595, "epoch": 0.2877631699917257, "frac_reward_zero_std": 0.4, "grad_norm": 4.49933385848999, "kl": 0.04054884716751985, "learning_rate": 4.971283942158684e-06, "loss": 0.0022, "num_tokens": 13564638.0, "reward": -0.05, "reward_std": 0.2799107313156128, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.2799107372760773, "sampling/importance_sampling_ratio/max": 1.2627691984176637, "sampling/importance_sampling_ratio/mean": 0.999916386604309, "sampling/importance_sampling_ratio/min": 0.778308093547821, "sampling/sampling_logp_difference/max": 0.2602915823459625, "sampling/sampling_logp_difference/mean": 0.020179171673953535, "step": 3130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.7, "completions/max_terminated_length": 271.7, "completions/mean_length": 181.7125, "completions/mean_terminated_length": 181.7125, "completions/min_length": 115.4, "completions/min_terminated_length": 115.4, "entropy": 1.0361908085644245, "epoch": 0.288682541141859, "frac_reward_zero_std": 0.2, "grad_norm": 0.2952147126197815, "kl": 0.038313745649065825, "learning_rate": 4.9708997612036805e-06, "loss": 0.0017, "num_tokens": 13608391.0, "reward": -0.025, "reward_std": 0.4446160405874252, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4446160525083542, "sampling/importance_sampling_ratio/max": 1.2630624651908875, "sampling/importance_sampling_ratio/mean": 0.9997836589813233, "sampling/importance_sampling_ratio/min": 0.7413627564907074, "sampling/sampling_logp_difference/max": 0.30697921514511106, "sampling/sampling_logp_difference/mean": 0.021165824867784976, "step": 3140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 248.1, "completions/max_terminated_length": 248.1, "completions/mean_length": 178.725, "completions/mean_terminated_length": 178.725, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 0.9629081703722477, "epoch": 0.2896019122919923, "frac_reward_zero_std": 0.4, "grad_norm": 0.16850629448890686, "kl": 0.04100114912143908, "learning_rate": 4.970513042436556e-06, "loss": 0.0022, "num_tokens": 13648793.0, "reward": 0.2625, "reward_std": 0.29114731550216677, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.2911473214626312, "sampling/importance_sampling_ratio/max": 1.2726138830184937, "sampling/importance_sampling_ratio/mean": 1.000109314918518, "sampling/importance_sampling_ratio/min": 0.7598125219345093, "sampling/sampling_logp_difference/max": 0.3038113474845886, "sampling/sampling_logp_difference/mean": 0.020257325284183024, "step": 3150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.2, "completions/max_terminated_length": 294.2, "completions/mean_length": 205.925, "completions/mean_terminated_length": 205.925, "completions/min_length": 124.6, "completions/min_terminated_length": 124.6, "entropy": 1.0496474407613277, "epoch": 0.2905212834421256, "frac_reward_zero_std": 0.1, "grad_norm": 4.621676445007324, "kl": 0.04431773464893922, "learning_rate": 4.970123786254502e-06, "loss": 0.0023, "num_tokens": 13691659.0, "reward": -0.0125, "reward_std": 0.4355591177940369, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4355591356754303, "sampling/importance_sampling_ratio/max": 1.2872658133506776, "sampling/importance_sampling_ratio/mean": 0.9999986410140991, "sampling/importance_sampling_ratio/min": 0.742449676990509, "sampling/sampling_logp_difference/max": 0.3295546770095825, "sampling/sampling_logp_difference/mean": 0.0217421879991889, "step": 3160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 294.2, "completions/max_terminated_length": 281.4, "completions/mean_length": 199.75, "completions/mean_terminated_length": 196.63035888671874, "completions/min_length": 112.5, "completions/min_terminated_length": 112.5, "entropy": 1.0945949390530587, "epoch": 0.2914406545922589, "frac_reward_zero_std": 0.3, "grad_norm": 6.85806131362915, "kl": 0.03886735549895093, "learning_rate": 4.969731993057313e-06, "loss": 0.0017, "num_tokens": 13738151.0, "reward": 0.10625, "reward_std": 0.3863959848880768, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.38639599084854126, "sampling/importance_sampling_ratio/max": 1.2561033248901368, "sampling/importance_sampling_ratio/mean": 1.0004417181015015, "sampling/importance_sampling_ratio/min": 0.7989996373653412, "sampling/sampling_logp_difference/max": 0.2525927424430847, "sampling/sampling_logp_difference/mean": 0.0211037527769804, "step": 3170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.8, "completions/max_terminated_length": 277.8, "completions/mean_length": 179.85, "completions/mean_terminated_length": 179.85, "completions/min_length": 105.0, "completions/min_terminated_length": 105.0, "entropy": 1.0940855354070664, "epoch": 0.2923600257423922, "frac_reward_zero_std": 0.5, "grad_norm": 0.25357934832572937, "kl": 0.04477006859378889, "learning_rate": 4.969337663247395e-06, "loss": 0.0021, "num_tokens": 13781563.0, "reward": 0.1875, "reward_std": 0.2609549701213837, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.2609549820423126, "sampling/importance_sampling_ratio/max": 1.3523677229881286, "sampling/importance_sampling_ratio/mean": 0.9998609185218811, "sampling/importance_sampling_ratio/min": 0.734242606163025, "sampling/sampling_logp_difference/max": 0.3794539928436279, "sampling/sampling_logp_difference/mean": 0.021928014792501928, "step": 3180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 211.325, "completions/mean_terminated_length": 211.325, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0738450296223163, "epoch": 0.2932793968925255, "frac_reward_zero_std": 0.4, "grad_norm": 6.378332614898682, "kl": 0.0345492938242387, "learning_rate": 4.968940797229756e-06, "loss": 0.0017, "num_tokens": 13825949.0, "reward": -0.25, "reward_std": 0.339043465256691, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.33904347717761996, "sampling/importance_sampling_ratio/max": 1.266939675807953, "sampling/importance_sampling_ratio/mean": 1.0004842698574066, "sampling/importance_sampling_ratio/min": 0.7825249791145324, "sampling/sampling_logp_difference/max": 0.2704328656196594, "sampling/sampling_logp_difference/mean": 0.020584259554743765, "step": 3190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 202.4125, "completions/mean_terminated_length": 202.4125, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.1267855055630207, "epoch": 0.2941987680426588, "frac_reward_zero_std": 0.3, "grad_norm": 5.992135524749756, "kl": 0.04341563874622807, "learning_rate": 4.9685413954120064e-06, "loss": 0.0022, "num_tokens": 13871142.0, "reward": -0.0875, "reward_std": 0.4157505214214325, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.41575053334236145, "sampling/importance_sampling_ratio/max": 1.3103078126907348, "sampling/importance_sampling_ratio/mean": 1.0004344403743743, "sampling/importance_sampling_ratio/min": 0.7893327891826629, "sampling/sampling_logp_difference/max": 0.2822605848312378, "sampling/sampling_logp_difference/mean": 0.022518909722566604, "step": 3200 }, { "epoch": 0.2941987680426588, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 284.97731239092496, "eval_completions/max_terminated_length": 283.5340314136126, "eval_completions/mean_length": 195.61649214659687, "eval_completions/mean_terminated_length": 195.230699069837, "eval_completions/min_length": 119.19371727748691, "eval_completions/min_terminated_length": 119.19371727748691, "eval_entropy": 1.1038762665439859, "eval_frac_reward_zero_std": 0.24083769633507854, "eval_kl": 0.038884694642512485, "eval_loss": 0.0020698360167443752, "eval_num_tokens": 13871142.0, "eval_reward": 0.03130453752181501, "eval_reward_std": 0.41291597225487126, "eval_rewards/reward_fn/mean": 0.03130453752181501, "eval_rewards/reward_fn/std": 0.4129159840093649, "eval_runtime": 1673.7544, "eval_samples_per_second": 0.342, "eval_sampling/importance_sampling_ratio/max": 1.261679679311383, "eval_sampling/importance_sampling_ratio/mean": 1.0000500616602872, "eval_sampling/importance_sampling_ratio/min": 0.7770163390856971, "eval_sampling/sampling_logp_difference/max": 0.277808698424078, "eval_sampling/sampling_logp_difference/mean": 0.021343533936270347, "eval_steps_per_second": 0.043, "step": 3200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.4, "completions/max_terminated_length": 271.4, "completions/mean_length": 179.8875, "completions/mean_terminated_length": 179.8875, "completions/min_length": 103.5, "completions/min_terminated_length": 103.5, "entropy": 1.0777474775910378, "epoch": 0.2951181391927921, "frac_reward_zero_std": 0.1, "grad_norm": 6.2479634284973145, "kl": 0.0417698580364231, "learning_rate": 4.9681394582043676e-06, "loss": 0.0015, "num_tokens": 13911357.0, "reward": -0.0625, "reward_std": 0.4260198652744293, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.4260198771953583, "sampling/importance_sampling_ratio/max": 1.2817686200141907, "sampling/importance_sampling_ratio/mean": 1.000111722946167, "sampling/importance_sampling_ratio/min": 0.771096158027649, "sampling/sampling_logp_difference/max": 0.30051048994064333, "sampling/sampling_logp_difference/mean": 0.021910884976387025, "step": 3210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 199.6, "completions/mean_terminated_length": 199.6, "completions/min_length": 117.7, "completions/min_terminated_length": 117.7, "entropy": 1.09399051964283, "epoch": 0.2960375103429254, "frac_reward_zero_std": 0.3, "grad_norm": 6.04278564453125, "kl": 0.03642093924572691, "learning_rate": 4.967734986019659e-06, "loss": 0.0015, "num_tokens": 13954469.0, "reward": 0.06875, "reward_std": 0.42089247405529023, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.42089249193668365, "sampling/importance_sampling_ratio/max": 1.2835207343101502, "sampling/importance_sampling_ratio/mean": 1.0002387642860413, "sampling/importance_sampling_ratio/min": 0.7825363755226136, "sampling/sampling_logp_difference/max": 0.2811251699924469, "sampling/sampling_logp_difference/mean": 0.02142675668001175, "step": 3220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.5, "completions/max_terminated_length": 254.5, "completions/mean_length": 181.3625, "completions/mean_terminated_length": 181.3625, "completions/min_length": 112.7, "completions/min_terminated_length": 112.7, "entropy": 1.1572017669677734, "epoch": 0.29695688149305877, "frac_reward_zero_std": 0.3, "grad_norm": 0.134357288479805, "kl": 0.12948516857577488, "learning_rate": 4.967327979273307e-06, "loss": 0.0061, "num_tokens": 13997762.0, "reward": 0.1875, "reward_std": 0.32480532228946685, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3248053342103958, "sampling/importance_sampling_ratio/max": 1.249702513217926, "sampling/importance_sampling_ratio/mean": 1.0000651717185973, "sampling/importance_sampling_ratio/min": 0.7590961575508117, "sampling/sampling_logp_difference/max": 0.33977370262145995, "sampling/sampling_logp_difference/mean": 0.022720706462860108, "step": 3230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.9, "completions/max_terminated_length": 250.9, "completions/mean_length": 176.85, "completions/mean_terminated_length": 176.85, "completions/min_length": 107.8, "completions/min_terminated_length": 107.8, "entropy": 1.0830637857317924, "epoch": 0.29787625264319206, "frac_reward_zero_std": 0.4, "grad_norm": 7.255032539367676, "kl": 0.045315441233105956, "learning_rate": 4.966918438383338e-06, "loss": 0.0016, "num_tokens": 14035646.0, "reward": 0.075, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2868318915367127, "sampling/importance_sampling_ratio/mean": 1.0002416014671325, "sampling/importance_sampling_ratio/min": 0.8168776869773865, "sampling/sampling_logp_difference/max": 0.25513908863067625, "sampling/sampling_logp_difference/mean": 0.021351755037903784, "step": 3240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 194.025, "completions/mean_terminated_length": 194.025, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.028859481215477, "epoch": 0.29879562379332536, "frac_reward_zero_std": 0.1, "grad_norm": 6.21858549118042, "kl": 0.04447895947378129, "learning_rate": 4.966506363770387e-06, "loss": 0.002, "num_tokens": 14075936.0, "reward": 0.1125, "reward_std": 0.4489682465791702, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.4489682585000992, "sampling/importance_sampling_ratio/max": 1.2909308552742005, "sampling/importance_sampling_ratio/mean": 1.000076675415039, "sampling/importance_sampling_ratio/min": 0.7841595768928528, "sampling/sampling_logp_difference/max": 0.28194546699523926, "sampling/sampling_logp_difference/mean": 0.02058293465524912, "step": 3250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 197.5125, "completions/mean_terminated_length": 197.5125, "completions/min_length": 116.4, "completions/min_terminated_length": 116.4, "entropy": 1.0998571321368218, "epoch": 0.29971499494345866, "frac_reward_zero_std": 0.2, "grad_norm": 7.343313694000244, "kl": 0.04879973778151907, "learning_rate": 4.966091755857685e-06, "loss": 0.0022, "num_tokens": 14118889.0, "reward": -0.1, "reward_std": 0.4374816298484802, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4374816358089447, "sampling/importance_sampling_ratio/max": 1.28458514213562, "sampling/importance_sampling_ratio/mean": 1.0001663267612457, "sampling/importance_sampling_ratio/min": 0.7976416409015655, "sampling/sampling_logp_difference/max": 0.271599555015564, "sampling/sampling_logp_difference/mean": 0.020944624766707422, "step": 3260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.6, "completions/max_terminated_length": 334.6, "completions/mean_length": 228.75, "completions/mean_terminated_length": 228.75, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.0862649217247964, "epoch": 0.300634366093592, "frac_reward_zero_std": 0.2, "grad_norm": 7.297386646270752, "kl": 0.0435910573345609, "learning_rate": 4.965674615071069e-06, "loss": 0.0021, "num_tokens": 14164997.0, "reward": -0.1875, "reward_std": 0.4481093943119049, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.44810941219329836, "sampling/importance_sampling_ratio/max": 1.3015381574630738, "sampling/importance_sampling_ratio/mean": 0.9994955420494079, "sampling/importance_sampling_ratio/min": 0.7816940486431122, "sampling/sampling_logp_difference/max": 0.29544293880462646, "sampling/sampling_logp_difference/mean": 0.020667477510869503, "step": 3270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 275.2, "completions/max_terminated_length": 265.8, "completions/mean_length": 194.2875, "completions/mean_terminated_length": 191.51071472167968, "completions/min_length": 122.0, "completions/min_terminated_length": 122.0, "entropy": 1.1601298116147518, "epoch": 0.3015537372437253, "frac_reward_zero_std": 0.3, "grad_norm": 8.687579154968262, "kl": 0.04985051549156196, "learning_rate": 4.965254941838974e-06, "loss": 0.0028, "num_tokens": 14206012.0, "reward": -0.04375, "reward_std": 0.40193670988082886, "rewards/reward_fn/mean": -0.04375, "rewards/reward_fn/std": 0.4019367277622223, "sampling/importance_sampling_ratio/max": 1.2199535489082336, "sampling/importance_sampling_ratio/mean": 1.0005960702896117, "sampling/importance_sampling_ratio/min": 0.7992093741893769, "sampling/sampling_logp_difference/max": 0.22871973514556884, "sampling/sampling_logp_difference/mean": 0.02164380457252264, "step": 3280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.3, "completions/max_terminated_length": 324.3, "completions/mean_length": 216.7625, "completions/mean_terminated_length": 216.7625, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.1029285594820977, "epoch": 0.3024731083938586, "frac_reward_zero_std": 0.3, "grad_norm": 0.16830675303936005, "kl": 0.0405906010302715, "learning_rate": 4.964832736592441e-06, "loss": 0.0013, "num_tokens": 14254009.0, "reward": 0.3625, "reward_std": 0.3782377243041992, "rewards/reward_fn/mean": 0.3625, "rewards/reward_fn/std": 0.3782377302646637, "sampling/importance_sampling_ratio/max": 1.2514019846916198, "sampling/importance_sampling_ratio/mean": 0.9997725665569306, "sampling/importance_sampling_ratio/min": 0.7823267221450806, "sampling/sampling_logp_difference/max": 0.2652637720108032, "sampling/sampling_logp_difference/mean": 0.0212613919749856, "step": 3290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.6, "completions/max_terminated_length": 294.6, "completions/mean_length": 202.2875, "completions/mean_terminated_length": 202.2875, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.1082360297441483, "epoch": 0.3033924795439919, "frac_reward_zero_std": 0.2, "grad_norm": 6.044140815734863, "kl": 0.04713789201341569, "learning_rate": 4.964407999765105e-06, "loss": 0.0026, "num_tokens": 14292768.0, "reward": -0.1, "reward_std": 0.3900520890951157, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.39005209505558014, "sampling/importance_sampling_ratio/max": 1.3164944648742676, "sampling/importance_sampling_ratio/mean": 0.9996932804584503, "sampling/importance_sampling_ratio/min": 0.7677010238170624, "sampling/sampling_logp_difference/max": 0.28633167743682864, "sampling/sampling_logp_difference/mean": 0.021988416276872157, "step": 3300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 301.9, "completions/max_terminated_length": 300.0, "completions/mean_length": 220.2, "completions/mean_terminated_length": 218.13392944335936, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.112907313555479, "epoch": 0.3043118506941252, "frac_reward_zero_std": 0.3, "grad_norm": 5.377566814422607, "kl": 0.03129078163765371, "learning_rate": 4.963980731793208e-06, "loss": 0.0013, "num_tokens": 14335296.0, "reward": 0.11875, "reward_std": 0.3474827975034714, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.34748280346393584, "sampling/importance_sampling_ratio/max": 1.313843059539795, "sampling/importance_sampling_ratio/mean": 1.0002396404743195, "sampling/importance_sampling_ratio/min": 0.7749020040035248, "sampling/sampling_logp_difference/max": 0.28961069583892823, "sampling/sampling_logp_difference/mean": 0.02126773204654455, "step": 3310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.9, "completions/max_terminated_length": 282.9, "completions/mean_length": 209.5, "completions/mean_terminated_length": 209.5, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0537394411861896, "epoch": 0.30523122184425855, "frac_reward_zero_std": 0.1, "grad_norm": 4.839500904083252, "kl": 0.05472843888564967, "learning_rate": 4.963550933115587e-06, "loss": 0.0038, "num_tokens": 14380368.0, "reward": 0.05, "reward_std": 0.3745658665895462, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.37456587255001067, "sampling/importance_sampling_ratio/max": 1.2789030194282531, "sampling/importance_sampling_ratio/mean": 1.000424474477768, "sampling/importance_sampling_ratio/min": 0.7834398627281189, "sampling/sampling_logp_difference/max": 0.2687701523303986, "sampling/sampling_logp_difference/mean": 0.02134343031793833, "step": 3320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 313.7, "completions/max_terminated_length": 309.6, "completions/mean_length": 226.8875, "completions/mean_terminated_length": 219.82250061035157, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.1447204016149044, "epoch": 0.30615059299439185, "frac_reward_zero_std": 0.5, "grad_norm": 4.403074264526367, "kl": 0.04042053988669068, "learning_rate": 4.96311860417368e-06, "loss": 0.002, "num_tokens": 14425839.0, "reward": 0.14375, "reward_std": 0.24405362606048583, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.2440536379814148, "sampling/importance_sampling_ratio/max": 1.2183462262153626, "sampling/importance_sampling_ratio/mean": 1.0000001311302185, "sampling/importance_sampling_ratio/min": 0.7743454873561859, "sampling/sampling_logp_difference/max": 0.25811060667037966, "sampling/sampling_logp_difference/mean": 0.021984154544770717, "step": 3330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 349.0, "completions/max_terminated_length": 340.3, "completions/mean_length": 245.4375, "completions/mean_terminated_length": 241.0625, "completions/min_length": 163.3, "completions/min_terminated_length": 163.3, "entropy": 1.0474849876016379, "epoch": 0.30706996414452514, "frac_reward_zero_std": 0.0, "grad_norm": 5.259395599365234, "kl": 0.045495229022344576, "learning_rate": 4.962683745411524e-06, "loss": 0.0024, "num_tokens": 14471722.0, "reward": -0.25, "reward_std": 0.48375589549541476, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4837559133768082, "sampling/importance_sampling_ratio/max": 1.261279320716858, "sampling/importance_sampling_ratio/mean": 1.0002941191196442, "sampling/importance_sampling_ratio/min": 0.7963183403015137, "sampling/sampling_logp_difference/max": 0.2499399185180664, "sampling/sampling_logp_difference/mean": 0.02021382739767432, "step": 3340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 341.9, "completions/max_terminated_length": 324.4, "completions/mean_length": 231.725, "completions/mean_terminated_length": 228.94285888671874, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.1465473890304565, "epoch": 0.30798933529465844, "frac_reward_zero_std": 0.4, "grad_norm": 6.311001300811768, "kl": 0.040943041816353797, "learning_rate": 4.9622463572757525e-06, "loss": 0.0019, "num_tokens": 14518420.0, "reward": -0.24375, "reward_std": 0.3198355257511139, "rewards/reward_fn/mean": -0.24375, "rewards/reward_fn/std": 0.31983553767204287, "sampling/importance_sampling_ratio/max": 1.2758788466453552, "sampling/importance_sampling_ratio/mean": 0.9996203184127808, "sampling/importance_sampling_ratio/min": 0.7949548125267029, "sampling/sampling_logp_difference/max": 0.2642476558685303, "sampling/sampling_logp_difference/mean": 0.02134274486452341, "step": 3350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 201.7125, "completions/mean_terminated_length": 201.7125, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.0588207058608532, "epoch": 0.30890870644479174, "frac_reward_zero_std": 0.2, "grad_norm": 8.081878662109375, "kl": 0.040357350173871966, "learning_rate": 4.961806440215599e-06, "loss": 0.0021, "num_tokens": 14563069.0, "reward": 0.2125, "reward_std": 0.33921052813529967, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.3392105340957642, "sampling/importance_sampling_ratio/max": 1.2543137669563293, "sampling/importance_sampling_ratio/mean": 1.0002131581306457, "sampling/importance_sampling_ratio/min": 0.8036286413669587, "sampling/sampling_logp_difference/max": 0.248527455329895, "sampling/sampling_logp_difference/mean": 0.02072956208139658, "step": 3360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.4, "completions/max_terminated_length": 293.4, "completions/mean_length": 222.625, "completions/mean_terminated_length": 222.625, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.1128535896539689, "epoch": 0.3098280775949251, "frac_reward_zero_std": 0.3, "grad_norm": 6.671053886413574, "kl": 0.04595191022381186, "learning_rate": 4.9613639946828925e-06, "loss": 0.0022, "num_tokens": 14607599.0, "reward": -0.1625, "reward_std": 0.35445033609867094, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.3544503539800644, "sampling/importance_sampling_ratio/max": 1.2700241565704347, "sampling/importance_sampling_ratio/mean": 0.9999836146831512, "sampling/importance_sampling_ratio/min": 0.7773639738559723, "sampling/sampling_logp_difference/max": 0.27117196321487425, "sampling/sampling_logp_difference/mean": 0.021323941834270953, "step": 3370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.1, "completions/max_terminated_length": 260.1, "completions/mean_length": 180.65, "completions/mean_terminated_length": 180.65, "completions/min_length": 108.2, "completions/min_terminated_length": 108.2, "entropy": 1.037803926691413, "epoch": 0.3107474487450584, "frac_reward_zero_std": 0.0, "grad_norm": 5.516391277313232, "kl": 0.057165876278304495, "learning_rate": 4.960919021132062e-06, "loss": 0.0026, "num_tokens": 14646387.0, "reward": 0.0875, "reward_std": 0.5316522598266602, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.5316522598266602, "sampling/importance_sampling_ratio/max": 1.3149107336997985, "sampling/importance_sampling_ratio/mean": 1.0002249658107758, "sampling/importance_sampling_ratio/min": 0.7886558175086975, "sampling/sampling_logp_difference/max": 0.29990922212600707, "sampling/sampling_logp_difference/mean": 0.020648540277034045, "step": 3380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 206.2125, "completions/mean_terminated_length": 206.2125, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0441970452666283, "epoch": 0.3116668198951917, "frac_reward_zero_std": 0.1, "grad_norm": 8.926807403564453, "kl": 0.04298793796915561, "learning_rate": 4.96047152002013e-06, "loss": 0.0022, "num_tokens": 14692068.0, "reward": -0.1, "reward_std": 0.40020377933979034, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.40020379722118377, "sampling/importance_sampling_ratio/max": 1.2929712414741517, "sampling/importance_sampling_ratio/mean": 1.0000506401062013, "sampling/importance_sampling_ratio/min": 0.7757519900798797, "sampling/sampling_logp_difference/max": 0.2902611494064331, "sampling/sampling_logp_difference/mean": 0.02051110677421093, "step": 3390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.2, "completions/max_terminated_length": 270.2, "completions/mean_length": 188.9375, "completions/mean_terminated_length": 188.9375, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0867524527013301, "epoch": 0.312586191045325, "frac_reward_zero_std": 0.3, "grad_norm": 5.35761022567749, "kl": 0.07332004045601934, "learning_rate": 4.960021491806716e-06, "loss": 0.004, "num_tokens": 14731671.0, "reward": -0.2625, "reward_std": 0.3093190997838974, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.30931911170482634, "sampling/importance_sampling_ratio/max": 1.2320275068283082, "sampling/importance_sampling_ratio/mean": 0.999734616279602, "sampling/importance_sampling_ratio/min": 0.7832122325897217, "sampling/sampling_logp_difference/max": 0.25267844200134276, "sampling/sampling_logp_difference/mean": 0.02092149220407009, "step": 3400 }, { "epoch": 0.312586191045325, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0002181500872600349, "eval_completions/max_length": 261.0314136125655, "eval_completions/max_terminated_length": 260.97731239092496, "eval_completions/mean_length": 184.41884816753927, "eval_completions/mean_terminated_length": 184.37250687897102, "eval_completions/min_length": 121.42931937172774, "eval_completions/min_terminated_length": 121.42931937172774, "eval_entropy": 1.1124632294577455, "eval_frac_reward_zero_std": 0.38219895287958117, "eval_kl": 0.06015212383173263, "eval_loss": 0.0031121568754315376, "eval_num_tokens": 14731671.0, "eval_reward": 0.051919720767888304, "eval_reward_std": 0.3012696447380758, "eval_rewards/reward_fn/mean": 0.051919720767888304, "eval_rewards/reward_fn/std": 0.3012696541000619, "eval_runtime": 1552.7607, "eval_samples_per_second": 0.369, "eval_sampling/importance_sampling_ratio/max": 1.2540524825583785, "eval_sampling/importance_sampling_ratio/mean": 0.9999092207737202, "eval_sampling/importance_sampling_ratio/min": 0.7823862497927214, "eval_sampling/sampling_logp_difference/max": 0.2652849298081057, "eval_sampling/sampling_logp_difference/mean": 0.021112393709728543, "eval_steps_per_second": 0.046, "step": 3400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.1, "completions/max_terminated_length": 290.1, "completions/mean_length": 195.4125, "completions/mean_terminated_length": 195.4125, "completions/min_length": 128.0, "completions/min_terminated_length": 128.0, "entropy": 1.2019731745123863, "epoch": 0.31350556219545833, "frac_reward_zero_std": 0.3, "grad_norm": 0.16295911371707916, "kl": 0.0738321740878746, "learning_rate": 4.959568936954036e-06, "loss": 0.0033, "num_tokens": 14774072.0, "reward": -0.1125, "reward_std": 0.3311904340982437, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.3311904460191727, "sampling/importance_sampling_ratio/max": 1.2507876396179198, "sampling/importance_sampling_ratio/mean": 0.9995803654193878, "sampling/importance_sampling_ratio/min": 0.7372380375862122, "sampling/sampling_logp_difference/max": 0.32430399060249326, "sampling/sampling_logp_difference/mean": 0.022767294198274612, "step": 3410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 203.0125, "completions/mean_terminated_length": 203.0125, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.130712130665779, "epoch": 0.31442493334559163, "frac_reward_zero_std": 0.4, "grad_norm": 0.17660915851593018, "kl": 0.05297590153058991, "learning_rate": 4.959113855926898e-06, "loss": 0.0026, "num_tokens": 14817921.0, "reward": 0.05, "reward_std": 0.28659676015377045, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.2865967661142349, "sampling/importance_sampling_ratio/max": 1.2884628772735596, "sampling/importance_sampling_ratio/mean": 1.0003902316093445, "sampling/importance_sampling_ratio/min": 0.752024120092392, "sampling/sampling_logp_difference/max": 0.3201058626174927, "sampling/sampling_logp_difference/mean": 0.0222373865544796, "step": 3420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.1, "completions/max_terminated_length": 281.1, "completions/mean_length": 196.7875, "completions/mean_terminated_length": 196.7875, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 0.9948059290647506, "epoch": 0.3153443044957249, "frac_reward_zero_std": 0.4, "grad_norm": 5.125006675720215, "kl": 0.0645312755019404, "learning_rate": 4.958656249192707e-06, "loss": 0.0029, "num_tokens": 14858296.0, "reward": -0.0375, "reward_std": 0.2558668524026871, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.25586686432361605, "sampling/importance_sampling_ratio/max": 1.2303800344467164, "sampling/importance_sampling_ratio/mean": 0.9998889267444611, "sampling/importance_sampling_ratio/min": 0.7863169848918915, "sampling/sampling_logp_difference/max": 0.24640003442764283, "sampling/sampling_logp_difference/mean": 0.020010964013636112, "step": 3430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.5, "completions/max_terminated_length": 260.5, "completions/mean_length": 189.7875, "completions/mean_terminated_length": 189.7875, "completions/min_length": 125.4, "completions/min_terminated_length": 125.4, "entropy": 1.0337785191833972, "epoch": 0.3162636756458582, "frac_reward_zero_std": 0.4, "grad_norm": 7.000353813171387, "kl": 0.056080954388016836, "learning_rate": 4.958196117221464e-06, "loss": 0.0025, "num_tokens": 14899375.0, "reward": 0.15, "reward_std": 0.3408251374959946, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.34082514345645903, "sampling/importance_sampling_ratio/max": 1.251307439804077, "sampling/importance_sampling_ratio/mean": 0.9999927282333374, "sampling/importance_sampling_ratio/min": 0.7130638420581817, "sampling/sampling_logp_difference/max": 0.3540029048919678, "sampling/sampling_logp_difference/mean": 0.021173002570867537, "step": 3440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.0, "completions/max_terminated_length": 253.0, "completions/mean_length": 187.1, "completions/mean_terminated_length": 187.1, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.08740364164114, "epoch": 0.3171830467959915, "frac_reward_zero_std": 0.6, "grad_norm": 0.1681165099143982, "kl": 0.04227160386508331, "learning_rate": 4.95773346048576e-06, "loss": 0.002, "num_tokens": 14942111.0, "reward": 0.1125, "reward_std": 0.19231741726398469, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.19231742918491362, "sampling/importance_sampling_ratio/max": 1.2743193864822389, "sampling/importance_sampling_ratio/mean": 0.9997759819030761, "sampling/importance_sampling_ratio/min": 0.79029461145401, "sampling/sampling_logp_difference/max": 0.27064874172210696, "sampling/sampling_logp_difference/mean": 0.02145589329302311, "step": 3450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 192.6625, "completions/mean_terminated_length": 192.6625, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.1556284435093402, "epoch": 0.31810241794612487, "frac_reward_zero_std": 0.6, "grad_norm": 15.46902084350586, "kl": 0.0817931110446807, "learning_rate": 4.95726827946078e-06, "loss": 0.0043, "num_tokens": 14984036.0, "reward": 0.15, "reward_std": 0.15782093107700348, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.15782093703746797, "sampling/importance_sampling_ratio/max": 1.2285220623016357, "sampling/importance_sampling_ratio/mean": 0.9999253928661347, "sampling/importance_sampling_ratio/min": 0.7690546929836273, "sampling/sampling_logp_difference/max": 0.27185516357421874, "sampling/sampling_logp_difference/mean": 0.02233248669654131, "step": 3460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.0, "completions/max_terminated_length": 278.0, "completions/mean_length": 210.0125, "completions/mean_terminated_length": 210.0125, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.1114678218960763, "epoch": 0.31902178909625817, "frac_reward_zero_std": 0.3, "grad_norm": 0.17906618118286133, "kl": 0.05498292844276875, "learning_rate": 4.956800574624303e-06, "loss": 0.0028, "num_tokens": 15026229.0, "reward": 0.275, "reward_std": 0.2748226135969162, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.2748226195573807, "sampling/importance_sampling_ratio/max": 1.2572020888328552, "sampling/importance_sampling_ratio/mean": 0.9999335944652558, "sampling/importance_sampling_ratio/min": 0.7917510688304901, "sampling/sampling_logp_difference/max": 0.2547708719968796, "sampling/sampling_logp_difference/mean": 0.02142923753708601, "step": 3470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.1, "completions/max_terminated_length": 302.1, "completions/mean_length": 202.375, "completions/mean_terminated_length": 202.375, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0744490265846252, "epoch": 0.31994116024639147, "frac_reward_zero_std": 0.1, "grad_norm": 6.356229305267334, "kl": 0.08345635380828753, "learning_rate": 4.9563303464567e-06, "loss": 0.0037, "num_tokens": 15065843.0, "reward": -0.025, "reward_std": 0.514201533794403, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5142015397548676, "sampling/importance_sampling_ratio/max": 1.2476582050323486, "sampling/importance_sampling_ratio/mean": 1.0003630936145782, "sampling/importance_sampling_ratio/min": 0.7805294215679168, "sampling/sampling_logp_difference/max": 0.26747645139694215, "sampling/sampling_logp_difference/mean": 0.021488935872912406, "step": 3480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 202.15, "completions/mean_terminated_length": 202.15, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.1452050752937795, "epoch": 0.32086053139652476, "frac_reward_zero_std": 0.2, "grad_norm": 5.393003463745117, "kl": 0.04011841090978123, "learning_rate": 4.955857595440931e-06, "loss": 0.0025, "num_tokens": 15111639.0, "reward": 0.2625, "reward_std": 0.38434176445007323, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.3843417763710022, "sampling/importance_sampling_ratio/max": 1.2546111941337585, "sampling/importance_sampling_ratio/mean": 0.9998025953769684, "sampling/importance_sampling_ratio/min": 0.7932824790477753, "sampling/sampling_logp_difference/max": 0.2517578363418579, "sampling/sampling_logp_difference/mean": 0.021969419531524183, "step": 3490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.4, "completions/max_terminated_length": 298.4, "completions/mean_length": 220.9, "completions/mean_terminated_length": 220.9, "completions/min_length": 145.8, "completions/min_terminated_length": 145.8, "entropy": 1.1159426502883434, "epoch": 0.32177990254665806, "frac_reward_zero_std": 0.2, "grad_norm": 5.783603191375732, "kl": 0.0334494337788783, "learning_rate": 4.955382322062551e-06, "loss": 0.0018, "num_tokens": 15157527.0, "reward": 0.0625, "reward_std": 0.384642681479454, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.384642693400383, "sampling/importance_sampling_ratio/max": 1.239232099056244, "sampling/importance_sampling_ratio/mean": 1.0003385365009307, "sampling/importance_sampling_ratio/min": 0.7998551845550537, "sampling/sampling_logp_difference/max": 0.2339189827442169, "sampling/sampling_logp_difference/mean": 0.021455386839807033, "step": 3500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.0, "completions/max_terminated_length": 270.0, "completions/mean_length": 193.4375, "completions/mean_terminated_length": 193.4375, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.0763651072978973, "epoch": 0.3226992736967914, "frac_reward_zero_std": 0.1, "grad_norm": 8.56969165802002, "kl": 0.044204343925230205, "learning_rate": 4.954904526809704e-06, "loss": 0.0015, "num_tokens": 15197930.0, "reward": 0.0625, "reward_std": 0.4350122928619385, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.43501230478286745, "sampling/importance_sampling_ratio/max": 1.3894447207450866, "sampling/importance_sampling_ratio/mean": 0.9999247670173645, "sampling/importance_sampling_ratio/min": 0.7801200091838837, "sampling/sampling_logp_difference/max": 0.3426011085510254, "sampling/sampling_logp_difference/mean": 0.02111690603196621, "step": 3510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.9, "completions/max_terminated_length": 265.9, "completions/mean_length": 188.6125, "completions/mean_terminated_length": 188.6125, "completions/min_length": 118.4, "completions/min_terminated_length": 118.4, "entropy": 1.0959118843078612, "epoch": 0.3236186448469247, "frac_reward_zero_std": 0.3, "grad_norm": 6.62729024887085, "kl": 0.027812705538235605, "learning_rate": 4.954424210173122e-06, "loss": 0.0015, "num_tokens": 15239739.0, "reward": 0.275, "reward_std": 0.413811257481575, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.41381126940250396, "sampling/importance_sampling_ratio/max": 1.319244658946991, "sampling/importance_sampling_ratio/mean": 1.000618177652359, "sampling/importance_sampling_ratio/min": 0.7903915464878082, "sampling/sampling_logp_difference/max": 0.2810153245925903, "sampling/sampling_logp_difference/mean": 0.021199222840368747, "step": 3520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 295.9, "completions/max_terminated_length": 283.7, "completions/mean_length": 200.2125, "completions/mean_terminated_length": 196.90714416503906, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0972279332578183, "epoch": 0.324538015997058, "frac_reward_zero_std": 0.3, "grad_norm": 5.982865810394287, "kl": 0.03530151855084114, "learning_rate": 4.95394137264613e-06, "loss": 0.0019, "num_tokens": 15280564.0, "reward": 0.19375, "reward_std": 0.4013991504907608, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.40139917433261874, "sampling/importance_sampling_ratio/max": 1.2826046228408814, "sampling/importance_sampling_ratio/mean": 0.9995478928089142, "sampling/importance_sampling_ratio/min": 0.7916634500026702, "sampling/sampling_logp_difference/max": 0.2662949085235596, "sampling/sampling_logp_difference/mean": 0.020691893808543683, "step": 3530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 183.35, "completions/mean_terminated_length": 183.35, "completions/min_length": 121.6, "completions/min_terminated_length": 121.6, "entropy": 1.1036290675401688, "epoch": 0.3254573871471913, "frac_reward_zero_std": 0.3, "grad_norm": 0.13474500179290771, "kl": 0.03013908474240452, "learning_rate": 4.953456014724643e-06, "loss": 0.0017, "num_tokens": 15317968.0, "reward": 0.0, "reward_std": 0.3964372009038925, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.396437206864357, "sampling/importance_sampling_ratio/max": 1.3327782154083252, "sampling/importance_sampling_ratio/mean": 1.000079196691513, "sampling/importance_sampling_ratio/min": 0.7919772505760193, "sampling/sampling_logp_difference/max": 0.3088480710983276, "sampling/sampling_logp_difference/mean": 0.021486950293183326, "step": 3540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.8, "completions/max_terminated_length": 272.8, "completions/mean_length": 189.6, "completions/mean_terminated_length": 189.6, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.114840267598629, "epoch": 0.32637675829732465, "frac_reward_zero_std": 0.5, "grad_norm": 0.2900848984718323, "kl": 0.025565692828968166, "learning_rate": 4.952968136907158e-06, "loss": 0.0012, "num_tokens": 15358128.0, "reward": -0.05, "reward_std": 0.3709822952747345, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.370982301235199, "sampling/importance_sampling_ratio/max": 1.2497904300689697, "sampling/importance_sampling_ratio/mean": 0.9997704565525055, "sampling/importance_sampling_ratio/min": 0.7842993438243866, "sampling/sampling_logp_difference/max": 0.2657679557800293, "sampling/sampling_logp_difference/mean": 0.021084331348538397, "step": 3550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.0, "completions/max_terminated_length": 311.3, "completions/mean_length": 228.05, "completions/mean_terminated_length": 225.3107147216797, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.124162356555462, "epoch": 0.32729612944745795, "frac_reward_zero_std": 0.2, "grad_norm": 0.1541113704442978, "kl": 0.03783324921969324, "learning_rate": 4.952477739694768e-06, "loss": 0.0015, "num_tokens": 15405028.0, "reward": -0.03125, "reward_std": 0.3908640146255493, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.3908640265464783, "sampling/importance_sampling_ratio/max": 1.2465819478034974, "sampling/importance_sampling_ratio/mean": 1.0000207543373107, "sampling/importance_sampling_ratio/min": 0.796151626110077, "sampling/sampling_logp_difference/max": 0.24124348163604736, "sampling/sampling_logp_difference/mean": 0.021043410524725914, "step": 3560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.8, "completions/max_terminated_length": 279.8, "completions/mean_length": 212.4125, "completions/mean_terminated_length": 212.4125, "completions/min_length": 145.3, "completions/min_terminated_length": 145.3, "entropy": 1.2658915683627128, "epoch": 0.32821550059759125, "frac_reward_zero_std": 0.3, "grad_norm": 5.551015377044678, "kl": 0.0411328959162347, "learning_rate": 4.951984823591148e-06, "loss": 0.0018, "num_tokens": 15449293.0, "reward": -0.1125, "reward_std": 0.4665554463863373, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.46655545234680174, "sampling/importance_sampling_ratio/max": 1.2340397000312806, "sampling/importance_sampling_ratio/mean": 1.0001348793506621, "sampling/importance_sampling_ratio/min": 0.7853639185428619, "sampling/sampling_logp_difference/max": 0.25348862409591677, "sampling/sampling_logp_difference/mean": 0.02258206345140934, "step": 3570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.2, "completions/max_terminated_length": 259.2, "completions/mean_length": 191.6875, "completions/mean_terminated_length": 191.6875, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.1672445468604564, "epoch": 0.32913487174772454, "frac_reward_zero_std": 0.3, "grad_norm": 6.156494617462158, "kl": 0.043730844254605473, "learning_rate": 4.951489389102564e-06, "loss": 0.002, "num_tokens": 15490532.0, "reward": -0.0625, "reward_std": 0.3647657185792923, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.36476574242115023, "sampling/importance_sampling_ratio/max": 1.2308570504188538, "sampling/importance_sampling_ratio/mean": 0.9994194984436036, "sampling/importance_sampling_ratio/min": 0.7789407908916474, "sampling/sampling_logp_difference/max": 0.2654020071029663, "sampling/sampling_logp_difference/mean": 0.022163640707731247, "step": 3580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 205.275, "completions/mean_terminated_length": 205.275, "completions/min_length": 130.7, "completions/min_terminated_length": 130.7, "entropy": 1.1414539583027363, "epoch": 0.33005424289785784, "frac_reward_zero_std": 0.1, "grad_norm": 5.749905109405518, "kl": 0.039948835014365615, "learning_rate": 4.950991436737868e-06, "loss": 0.0023, "num_tokens": 15534834.0, "reward": 0.1125, "reward_std": 0.43887105882167815, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.4388710707426071, "sampling/importance_sampling_ratio/max": 1.2631184101104735, "sampling/importance_sampling_ratio/mean": 0.9997998237609863, "sampling/importance_sampling_ratio/min": 0.7739748060703278, "sampling/sampling_logp_difference/max": 0.29264225959777834, "sampling/sampling_logp_difference/mean": 0.02140676397830248, "step": 3590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 195.7625, "completions/mean_terminated_length": 195.7625, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.1062183894217015, "epoch": 0.3309736140479912, "frac_reward_zero_std": 0.3, "grad_norm": 7.103593826293945, "kl": 0.04652380687184632, "learning_rate": 4.9504909670084934e-06, "loss": 0.0019, "num_tokens": 15578039.0, "reward": -0.25, "reward_std": 0.3500634729862213, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3500634849071503, "sampling/importance_sampling_ratio/max": 1.2819628834724426, "sampling/importance_sampling_ratio/mean": 1.0000762343406677, "sampling/importance_sampling_ratio/min": 0.7846461653709411, "sampling/sampling_logp_difference/max": 0.2850764274597168, "sampling/sampling_logp_difference/mean": 0.0208723871037364, "step": 3600 }, { "epoch": 0.3309736140479912, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 294.4938917975567, "eval_completions/max_terminated_length": 291.40488656195464, "eval_completions/mean_length": 202.26657940663176, "eval_completions/mean_terminated_length": 201.4967902751077, "eval_completions/min_length": 127.19371727748691, "eval_completions/min_terminated_length": 127.19371727748691, "eval_entropy": 1.1608602334899636, "eval_frac_reward_zero_std": 0.28272251308900526, "eval_kl": 0.04071375128876476, "eval_loss": 0.0020598764531314373, "eval_num_tokens": 15578039.0, "eval_reward": 0.04253926701570681, "eval_reward_std": 0.3759350982636057, "eval_rewards/reward_fn/mean": 0.04253926701570681, "eval_rewards/reward_fn/std": 0.3759351092899449, "eval_runtime": 1719.186, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2685776919802654, "eval_sampling/importance_sampling_ratio/mean": 0.9999945341604543, "eval_sampling/importance_sampling_ratio/min": 0.7839648288529997, "eval_sampling/sampling_logp_difference/max": 0.2756668614779467, "eval_sampling/sampling_logp_difference/mean": 0.02123409967355911, "eval_steps_per_second": 0.042, "step": 3600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 333.3, "completions/max_terminated_length": 310.8, "completions/mean_length": 217.575, "completions/mean_terminated_length": 214.05535888671875, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.180570238083601, "epoch": 0.3318929851981245, "frac_reward_zero_std": 0.4, "grad_norm": 0.21361000835895538, "kl": 0.031616039772052316, "learning_rate": 4.949987980428466e-06, "loss": 0.0015, "num_tokens": 15626413.0, "reward": 0.01875, "reward_std": 0.33154732286930083, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.3315473347902298, "sampling/importance_sampling_ratio/max": 1.2511038780212402, "sampling/importance_sampling_ratio/mean": 0.9998937308788299, "sampling/importance_sampling_ratio/min": 0.8101923346519471, "sampling/sampling_logp_difference/max": 0.2298378348350525, "sampling/sampling_logp_difference/mean": 0.021659868955612182, "step": 3610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.3, "completions/max_terminated_length": 314.3, "completions/mean_length": 217.55, "completions/mean_terminated_length": 217.55, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.1649263449013234, "epoch": 0.3328123563482578, "frac_reward_zero_std": 0.4, "grad_norm": 6.513465404510498, "kl": 0.04388266638852656, "learning_rate": 4.949482477514391e-06, "loss": 0.0021, "num_tokens": 15670633.0, "reward": -0.1375, "reward_std": 0.3458986967802048, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.34589870274066925, "sampling/importance_sampling_ratio/max": 1.33402818441391, "sampling/importance_sampling_ratio/mean": 0.9993565559387207, "sampling/importance_sampling_ratio/min": 0.7917412281036377, "sampling/sampling_logp_difference/max": 0.3039770245552063, "sampling/sampling_logp_difference/mean": 0.021887423284351826, "step": 3620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 303.4, "completions/max_terminated_length": 301.5, "completions/mean_length": 215.1875, "completions/mean_terminated_length": 213.78571472167968, "completions/min_length": 129.0, "completions/min_terminated_length": 129.0, "entropy": 1.0215728491544724, "epoch": 0.3337317274983911, "frac_reward_zero_std": 0.1, "grad_norm": 5.613890647888184, "kl": 0.040379841323010625, "learning_rate": 4.948974458785463e-06, "loss": 0.0017, "num_tokens": 15711720.0, "reward": 0.09375, "reward_std": 0.4271946668624878, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.42719467878341677, "sampling/importance_sampling_ratio/max": 1.297000002861023, "sampling/importance_sampling_ratio/mean": 0.9994298100471497, "sampling/importance_sampling_ratio/min": 0.7874313831329346, "sampling/sampling_logp_difference/max": 0.2804200530052185, "sampling/sampling_logp_difference/mean": 0.020793511904776095, "step": 3630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.9, "completions/max_terminated_length": 268.9, "completions/mean_length": 190.925, "completions/mean_terminated_length": 190.925, "completions/min_length": 120.7, "completions/min_terminated_length": 120.7, "entropy": 1.1066973432898521, "epoch": 0.33465109864852444, "frac_reward_zero_std": 0.4, "grad_norm": 0.15176279842853546, "kl": 0.03049780459841713, "learning_rate": 4.9484639247634555e-06, "loss": 0.0012, "num_tokens": 15755402.0, "reward": 0.2625, "reward_std": 0.2394672751426697, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.23946728110313414, "sampling/importance_sampling_ratio/max": 1.2649291038513184, "sampling/importance_sampling_ratio/mean": 0.9996630251407623, "sampling/importance_sampling_ratio/min": 0.7570954263210297, "sampling/sampling_logp_difference/max": 0.3304596126079559, "sampling/sampling_logp_difference/mean": 0.022077969275414944, "step": 3640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.9, "completions/max_terminated_length": 288.9, "completions/mean_length": 210.2875, "completions/mean_terminated_length": 210.2875, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.1345528542995453, "epoch": 0.33557046979865773, "frac_reward_zero_std": 0.5, "grad_norm": 0.1285932958126068, "kl": 0.044535727973561735, "learning_rate": 4.947950875972729e-06, "loss": 0.0023, "num_tokens": 15801385.0, "reward": 0.0375, "reward_std": 0.2540946453809738, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.2540946573019028, "sampling/importance_sampling_ratio/max": 1.2794812321662903, "sampling/importance_sampling_ratio/mean": 1.0000095248222352, "sampling/importance_sampling_ratio/min": 0.7962130904197693, "sampling/sampling_logp_difference/max": 0.265420126914978, "sampling/sampling_logp_difference/mean": 0.021642114501446486, "step": 3650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 217.9625, "completions/mean_terminated_length": 217.9625, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.087868282943964, "epoch": 0.33648984094879103, "frac_reward_zero_std": 0.3, "grad_norm": 5.727581977844238, "kl": 0.04330416562734172, "learning_rate": 4.947435312940226e-06, "loss": 0.0021, "num_tokens": 15845302.0, "reward": 0.1375, "reward_std": 0.3432215988636017, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.34322160482406616, "sampling/importance_sampling_ratio/max": 1.2704196572303772, "sampling/importance_sampling_ratio/mean": 0.9999796926975251, "sampling/importance_sampling_ratio/min": 0.7525640308856965, "sampling/sampling_logp_difference/max": 0.3059571862220764, "sampling/sampling_logp_difference/mean": 0.021398631483316423, "step": 3660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.1, "completions/max_terminated_length": 256.1, "completions/mean_length": 189.8875, "completions/mean_terminated_length": 189.8875, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 1.0530556544661522, "epoch": 0.3374092120989243, "frac_reward_zero_std": 0.5, "grad_norm": 6.61439323425293, "kl": 0.02843841178691946, "learning_rate": 4.94691723619547e-06, "loss": 0.0018, "num_tokens": 15887277.0, "reward": 0.475, "reward_std": 0.267578649520874, "rewards/reward_fn/mean": 0.475, "rewards/reward_fn/std": 0.267578661441803, "sampling/importance_sampling_ratio/max": 1.2762251019477844, "sampling/importance_sampling_ratio/mean": 1.0003567576408385, "sampling/importance_sampling_ratio/min": 0.7683699250221252, "sampling/sampling_logp_difference/max": 0.30725838541984557, "sampling/sampling_logp_difference/mean": 0.020744536258280277, "step": 3670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 299.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 200.05, "completions/mean_terminated_length": 196.35714416503907, "completions/min_length": 117.8, "completions/min_terminated_length": 117.8, "entropy": 1.0470870293676853, "epoch": 0.3383285832490576, "frac_reward_zero_std": 0.3, "grad_norm": 5.2421112060546875, "kl": 0.027060857601463796, "learning_rate": 4.946396646270569e-06, "loss": 0.0017, "num_tokens": 15934937.0, "reward": 0.19375, "reward_std": 0.4837629318237305, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.48376294374465945, "sampling/importance_sampling_ratio/max": 1.2478211402893067, "sampling/importance_sampling_ratio/mean": 1.0000157594680785, "sampling/importance_sampling_ratio/min": 0.7879285931587219, "sampling/sampling_logp_difference/max": 0.24876997470855713, "sampling/sampling_logp_difference/mean": 0.020888124220073225, "step": 3680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.7, "completions/max_terminated_length": 255.7, "completions/mean_length": 180.6625, "completions/mean_terminated_length": 180.6625, "completions/min_length": 113.2, "completions/min_terminated_length": 113.2, "entropy": 1.0996396481990813, "epoch": 0.339247954399191, "frac_reward_zero_std": 0.2, "grad_norm": 7.609167575836182, "kl": 0.027532691974192856, "learning_rate": 4.945873543700209e-06, "loss": 0.0014, "num_tokens": 15975270.0, "reward": 0.325, "reward_std": 0.49643046259880064, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.49643046855926515, "sampling/importance_sampling_ratio/max": 1.2901219129562378, "sampling/importance_sampling_ratio/mean": 0.9997725486755371, "sampling/importance_sampling_ratio/min": 0.7695867896080018, "sampling/sampling_logp_difference/max": 0.3122891902923584, "sampling/sampling_logp_difference/mean": 0.021773027069866658, "step": 3690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.1, "completions/max_terminated_length": 287.1, "completions/mean_length": 189.5375, "completions/mean_terminated_length": 189.5375, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.1054870411753654, "epoch": 0.3401673255493243, "frac_reward_zero_std": 0.3, "grad_norm": 5.653794288635254, "kl": 0.034996135684195906, "learning_rate": 4.945347929021659e-06, "loss": 0.0018, "num_tokens": 16018065.0, "reward": 0.3, "reward_std": 0.445802640914917, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.445802640914917, "sampling/importance_sampling_ratio/max": 1.2407708287239074, "sampling/importance_sampling_ratio/mean": 1.000261563062668, "sampling/importance_sampling_ratio/min": 0.7980013370513916, "sampling/sampling_logp_difference/max": 0.2325606346130371, "sampling/sampling_logp_difference/mean": 0.021481046080589296, "step": 3700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.1, "completions/max_terminated_length": 245.1, "completions/mean_length": 182.3625, "completions/mean_terminated_length": 182.3625, "completions/min_length": 118.2, "completions/min_terminated_length": 118.2, "entropy": 1.0582818120718003, "epoch": 0.34108669669945757, "frac_reward_zero_std": 0.6, "grad_norm": 0.14568209648132324, "kl": 0.027012444072170183, "learning_rate": 4.944819802774767e-06, "loss": 0.001, "num_tokens": 16056830.0, "reward": -0.0125, "reward_std": 0.1974882572889328, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.19748826324939728, "sampling/importance_sampling_ratio/max": 1.2445526838302612, "sampling/importance_sampling_ratio/mean": 1.0001866579055787, "sampling/importance_sampling_ratio/min": 0.7962485194206238, "sampling/sampling_logp_difference/max": 0.24086624383926392, "sampling/sampling_logp_difference/mean": 0.021006628498435022, "step": 3710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.1, "completions/max_terminated_length": 308.1, "completions/mean_length": 203.5625, "completions/mean_terminated_length": 203.5625, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.1114559836685658, "epoch": 0.34200606784959087, "frac_reward_zero_std": 0.7, "grad_norm": 4.144114971160889, "kl": 0.021790811145910992, "learning_rate": 4.944289165501961e-06, "loss": 0.001, "num_tokens": 16099795.0, "reward": -0.0625, "reward_std": 0.14511305689811707, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.14511306285858155, "sampling/importance_sampling_ratio/max": 1.281035840511322, "sampling/importance_sampling_ratio/mean": 1.000019919872284, "sampling/importance_sampling_ratio/min": 0.7919380784034729, "sampling/sampling_logp_difference/max": 0.27017725706100465, "sampling/sampling_logp_difference/mean": 0.021005408093333244, "step": 3720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 206.225, "completions/mean_terminated_length": 206.225, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.0347593866288662, "epoch": 0.34292543899972416, "frac_reward_zero_std": 0.2, "grad_norm": 6.044461727142334, "kl": 0.0385551703220699, "learning_rate": 4.9437560177482494e-06, "loss": 0.0019, "num_tokens": 16140285.0, "reward": -0.0125, "reward_std": 0.47564709186553955, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4756471037864685, "sampling/importance_sampling_ratio/max": 1.3385959029197694, "sampling/importance_sampling_ratio/mean": 0.9997911810874939, "sampling/importance_sampling_ratio/min": 0.7981269598007202, "sampling/sampling_logp_difference/max": 0.2989547193050385, "sampling/sampling_logp_difference/mean": 0.020690604858100415, "step": 3730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.6, "completions/max_terminated_length": 306.6, "completions/mean_length": 212.2, "completions/mean_terminated_length": 212.2, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.1073269113898276, "epoch": 0.3438448101498575, "frac_reward_zero_std": 0.1, "grad_norm": 5.492613315582275, "kl": 0.03056586690363474, "learning_rate": 4.943220360061215e-06, "loss": 0.0013, "num_tokens": 16185741.0, "reward": 0.0625, "reward_std": 0.49708993136882784, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.49708996117115023, "sampling/importance_sampling_ratio/max": 1.2766258239746093, "sampling/importance_sampling_ratio/mean": 0.9999300539493561, "sampling/importance_sampling_ratio/min": 0.7869200527667999, "sampling/sampling_logp_difference/max": 0.26742135286331176, "sampling/sampling_logp_difference/mean": 0.020869196392595768, "step": 3740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.0, "completions/max_terminated_length": 302.0, "completions/mean_length": 212.825, "completions/mean_terminated_length": 212.825, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 1.147009377926588, "epoch": 0.3447641812999908, "frac_reward_zero_std": 0.2, "grad_norm": 0.11257202178239822, "kl": 0.039408953103702515, "learning_rate": 4.942682192991024e-06, "loss": 0.0024, "num_tokens": 16228631.0, "reward": -0.15, "reward_std": 0.42775094211101533, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4277509480714798, "sampling/importance_sampling_ratio/max": 1.3021114349365235, "sampling/importance_sampling_ratio/mean": 1.0002467393875123, "sampling/importance_sampling_ratio/min": 0.7513287425041199, "sampling/sampling_logp_difference/max": 0.3089804947376251, "sampling/sampling_logp_difference/mean": 0.022145429626107216, "step": 3750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.4, "completions/max_terminated_length": 270.4, "completions/mean_length": 209.025, "completions/mean_terminated_length": 209.025, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.04922189489007, "epoch": 0.3456835524501241, "frac_reward_zero_std": 0.0, "grad_norm": 6.559751987457275, "kl": 0.04193950870539993, "learning_rate": 4.942141517090415e-06, "loss": 0.0021, "num_tokens": 16272425.0, "reward": 0.0, "reward_std": 0.4817752480506897, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.48177525997161863, "sampling/importance_sampling_ratio/max": 1.2497560858726502, "sampling/importance_sampling_ratio/mean": 0.9995654940605163, "sampling/importance_sampling_ratio/min": 0.7870202004909516, "sampling/sampling_logp_difference/max": 0.2617586374282837, "sampling/sampling_logp_difference/mean": 0.020650873146951198, "step": 3760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.4, "completions/max_terminated_length": 258.4, "completions/mean_length": 193.4, "completions/mean_terminated_length": 193.4, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.0805205300450325, "epoch": 0.3466029236002574, "frac_reward_zero_std": 0.4, "grad_norm": 7.2005414962768555, "kl": 0.03449253172148019, "learning_rate": 4.941598332914707e-06, "loss": 0.0015, "num_tokens": 16315489.0, "reward": 0.1, "reward_std": 0.30355427265167234, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.30355427861213685, "sampling/importance_sampling_ratio/max": 1.2482670187950133, "sampling/importance_sampling_ratio/mean": 1.0004632771015167, "sampling/importance_sampling_ratio/min": 0.7893210172653198, "sampling/sampling_logp_difference/max": 0.25456771850585935, "sampling/sampling_logp_difference/mean": 0.021257933415472507, "step": 3770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.1, "completions/max_terminated_length": 261.1, "completions/mean_length": 197.025, "completions/mean_terminated_length": 197.025, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.0283584214746953, "epoch": 0.34752229475039076, "frac_reward_zero_std": 0.3, "grad_norm": 6.11715030670166, "kl": 0.03571356469765306, "learning_rate": 4.941052641021793e-06, "loss": 0.0021, "num_tokens": 16361083.0, "reward": 0.0125, "reward_std": 0.3774814397096634, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.3774814516305923, "sampling/importance_sampling_ratio/max": 1.2746479749679565, "sampling/importance_sampling_ratio/mean": 1.0005091607570649, "sampling/importance_sampling_ratio/min": 0.7983085095882416, "sampling/sampling_logp_difference/max": 0.2533171534538269, "sampling/sampling_logp_difference/mean": 0.020525658130645753, "step": 3780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 293.7, "completions/max_terminated_length": 267.5, "completions/mean_length": 197.3, "completions/mean_terminated_length": 193.1714294433594, "completions/min_length": 115.9, "completions/min_terminated_length": 115.9, "entropy": 1.024768951535225, "epoch": 0.34844166590052406, "frac_reward_zero_std": 0.3, "grad_norm": 0.14785781502723694, "kl": 0.038201427122112365, "learning_rate": 4.940504441972142e-06, "loss": 0.0018, "num_tokens": 16403099.0, "reward": 0.0, "reward_std": 0.3418251842260361, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.34182519614696505, "sampling/importance_sampling_ratio/max": 1.2535872340202332, "sampling/importance_sampling_ratio/mean": 1.0001335263252258, "sampling/importance_sampling_ratio/min": 0.7969481885433197, "sampling/sampling_logp_difference/max": 0.25146701335906985, "sampling/sampling_logp_difference/mean": 0.020142976008355618, "step": 3790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.6, "completions/max_terminated_length": 249.6, "completions/mean_length": 184.9, "completions/mean_terminated_length": 184.9, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0642747826874257, "epoch": 0.34936103705065735, "frac_reward_zero_std": 0.4, "grad_norm": 6.6343207359313965, "kl": 0.040106067794840786, "learning_rate": 4.939953736328799e-06, "loss": 0.0019, "num_tokens": 16442803.0, "reward": 0.1625, "reward_std": 0.2722664296627045, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.2722664475440979, "sampling/importance_sampling_ratio/max": 1.2576985359191895, "sampling/importance_sampling_ratio/mean": 0.9996882736682892, "sampling/importance_sampling_ratio/min": 0.7921940565109253, "sampling/sampling_logp_difference/max": 0.24677510261535646, "sampling/sampling_logp_difference/mean": 0.021370902471244336, "step": 3800 }, { "epoch": 0.34936103705065735, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 288.4013961605585, "eval_completions/max_terminated_length": 285.5183246073298, "eval_completions/mean_length": 202.68215532286212, "eval_completions/mean_terminated_length": 202.03300950081976, "eval_completions/min_length": 131.21640488656195, "eval_completions/min_terminated_length": 131.21640488656195, "eval_entropy": 1.0889998854870988, "eval_frac_reward_zero_std": 0.2914485165794066, "eval_kl": 0.03765272300033773, "eval_loss": 0.0019535778556019068, "eval_num_tokens": 16442803.0, "eval_reward": 0.04308464223385689, "eval_reward_std": 0.37965829766649645, "eval_rewards/reward_fn/mean": 0.04308464223385689, "eval_rewards/reward_fn/std": 0.37965830796468114, "eval_runtime": 1689.934, "eval_samples_per_second": 0.339, "eval_sampling/importance_sampling_ratio/max": 1.2654195166799203, "eval_sampling/importance_sampling_ratio/mean": 0.9999659575300899, "eval_sampling/importance_sampling_ratio/min": 0.7853988346629118, "eval_sampling/sampling_logp_difference/max": 0.26999440357531135, "eval_sampling/sampling_logp_difference/mean": 0.020895753397133336, "eval_steps_per_second": 0.043, "step": 3800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.9, "completions/max_terminated_length": 287.9, "completions/mean_length": 195.1875, "completions/mean_terminated_length": 195.1875, "completions/min_length": 113.8, "completions/min_terminated_length": 113.8, "entropy": 1.0213661059737205, "epoch": 0.35028040820079065, "frac_reward_zero_std": 0.0, "grad_norm": 7.19238805770874, "kl": 0.04155476473970339, "learning_rate": 4.9394005246573826e-06, "loss": 0.002, "num_tokens": 16485538.0, "reward": -0.0125, "reward_std": 0.5410771846771241, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.541077196598053, "sampling/importance_sampling_ratio/max": 1.2793991088867187, "sampling/importance_sampling_ratio/mean": 1.0003066837787629, "sampling/importance_sampling_ratio/min": 0.8043634951114654, "sampling/sampling_logp_difference/max": 0.24925187826156617, "sampling/sampling_logp_difference/mean": 0.02093137949705124, "step": 3810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.3, "completions/max_terminated_length": 298.3, "completions/mean_length": 211.2375, "completions/mean_terminated_length": 211.2375, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.07375026345253, "epoch": 0.35119977935092395, "frac_reward_zero_std": 0.3, "grad_norm": 0.1306552141904831, "kl": 0.030958269751863554, "learning_rate": 4.938844807526085e-06, "loss": 0.0014, "num_tokens": 16527917.0, "reward": 0.175, "reward_std": 0.40066249668598175, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4006625026464462, "sampling/importance_sampling_ratio/max": 1.2238177061080933, "sampling/importance_sampling_ratio/mean": 1.0003024578094482, "sampling/importance_sampling_ratio/min": 0.7825372815132141, "sampling/sampling_logp_difference/max": 0.24665732383728028, "sampling/sampling_logp_difference/mean": 0.02107987031340599, "step": 3820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.2, "completions/max_terminated_length": 295.2, "completions/mean_length": 217.875, "completions/mean_terminated_length": 217.875, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.0477053500711917, "epoch": 0.3521191505010573, "frac_reward_zero_std": 0.4, "grad_norm": 7.669813632965088, "kl": 0.03516795663163066, "learning_rate": 4.938286585505673e-06, "loss": 0.0016, "num_tokens": 16573587.0, "reward": -0.1, "reward_std": 0.3466688096523285, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3466688334941864, "sampling/importance_sampling_ratio/max": 1.2880855798721313, "sampling/importance_sampling_ratio/mean": 0.9997962713241577, "sampling/importance_sampling_ratio/min": 0.7429078757762909, "sampling/sampling_logp_difference/max": 0.32917438745498656, "sampling/sampling_logp_difference/mean": 0.020702942460775375, "step": 3830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.6, "completions/max_terminated_length": 256.6, "completions/mean_length": 191.6125, "completions/mean_terminated_length": 191.6125, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 1.0914020456373692, "epoch": 0.3530385216511906, "frac_reward_zero_std": 0.1, "grad_norm": 5.068206787109375, "kl": 0.04721885845065117, "learning_rate": 4.9377258591694854e-06, "loss": 0.0026, "num_tokens": 16615004.0, "reward": -0.1625, "reward_std": 0.47980677485466006, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.47980678677558897, "sampling/importance_sampling_ratio/max": 1.3143089056015014, "sampling/importance_sampling_ratio/mean": 1.0000809967517852, "sampling/importance_sampling_ratio/min": 0.771519011259079, "sampling/sampling_logp_difference/max": 0.30351110100746154, "sampling/sampling_logp_difference/mean": 0.02236689832061529, "step": 3840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.0, "completions/max_terminated_length": 276.0, "completions/mean_length": 205.375, "completions/mean_terminated_length": 205.375, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0853016667068005, "epoch": 0.3539578928013239, "frac_reward_zero_std": 0.1, "grad_norm": 0.1450071781873703, "kl": 0.03386314922827296, "learning_rate": 4.937162629093433e-06, "loss": 0.002, "num_tokens": 16657802.0, "reward": 0.05, "reward_std": 0.4728062242269516, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.4728062301874161, "sampling/importance_sampling_ratio/max": 1.2426780104637145, "sampling/importance_sampling_ratio/mean": 1.0004954636096954, "sampling/importance_sampling_ratio/min": 0.7994471788406372, "sampling/sampling_logp_difference/max": 0.24012725353240966, "sampling/sampling_logp_difference/mean": 0.021801168285310268, "step": 3850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.3, "completions/max_terminated_length": 309.3, "completions/mean_length": 219.8875, "completions/mean_terminated_length": 219.8875, "completions/min_length": 128.9, "completions/min_terminated_length": 128.9, "entropy": 1.1077279642224311, "epoch": 0.3548772639514572, "frac_reward_zero_std": 0.2, "grad_norm": 5.3538594245910645, "kl": 0.03275170275010168, "learning_rate": 4.936596895855998e-06, "loss": 0.0014, "num_tokens": 16702505.0, "reward": -0.35, "reward_std": 0.40359844267368317, "rewards/reward_fn/mean": -0.35, "rewards/reward_fn/std": 0.4035984486341476, "sampling/importance_sampling_ratio/max": 1.2595935821533204, "sampling/importance_sampling_ratio/mean": 0.9996255397796631, "sampling/importance_sampling_ratio/min": 0.7407901108264923, "sampling/sampling_logp_difference/max": 0.3108835220336914, "sampling/sampling_logp_difference/mean": 0.022558890655636786, "step": 3860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.0, "completions/max_terminated_length": 316.0, "completions/mean_length": 227.6375, "completions/mean_terminated_length": 227.6375, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0208419144153595, "epoch": 0.3557966351015905, "frac_reward_zero_std": 0.3, "grad_norm": 0.20402826368808746, "kl": 0.033885483851190654, "learning_rate": 4.9360286600382326e-06, "loss": 0.0015, "num_tokens": 16746148.0, "reward": 0.125, "reward_std": 0.4000727891921997, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.40007280707359316, "sampling/importance_sampling_ratio/max": 1.246892273426056, "sampling/importance_sampling_ratio/mean": 1.0000983595848083, "sampling/importance_sampling_ratio/min": 0.801948344707489, "sampling/sampling_logp_difference/max": 0.23523476123809814, "sampling/sampling_logp_difference/mean": 0.020918942429125308, "step": 3870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 232.6125, "completions/mean_terminated_length": 232.6125, "completions/min_length": 151.7, "completions/min_terminated_length": 151.7, "entropy": 1.0896278016269207, "epoch": 0.35671600625172384, "frac_reward_zero_std": 0.3, "grad_norm": 6.469940662384033, "kl": 0.029803099611308427, "learning_rate": 4.9354579222237624e-06, "loss": 0.0012, "num_tokens": 16793605.0, "reward": 0.0875, "reward_std": 0.3946571469306946, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.394657164812088, "sampling/importance_sampling_ratio/max": 1.283231246471405, "sampling/importance_sampling_ratio/mean": 0.9998651146888733, "sampling/importance_sampling_ratio/min": 0.7650677621364593, "sampling/sampling_logp_difference/max": 0.2906580984592438, "sampling/sampling_logp_difference/mean": 0.021508488245308398, "step": 3880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.8, "completions/max_terminated_length": 306.3, "completions/mean_length": 220.75, "completions/mean_terminated_length": 217.78035888671874, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1108997106552123, "epoch": 0.35763537740185714, "frac_reward_zero_std": 0.3, "grad_norm": 7.163994789123535, "kl": 0.03250877143582329, "learning_rate": 4.934884682998781e-06, "loss": 0.0014, "num_tokens": 16837097.0, "reward": 0.10625, "reward_std": 0.44252554178237913, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.44252554178237913, "sampling/importance_sampling_ratio/max": 1.3308586359024048, "sampling/importance_sampling_ratio/mean": 1.0002409398555756, "sampling/importance_sampling_ratio/min": 0.7470481276512146, "sampling/sampling_logp_difference/max": 0.32441638708114623, "sampling/sampling_logp_difference/mean": 0.021972625330090523, "step": 3890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 346.0, "completions/max_terminated_length": 331.4, "completions/mean_length": 232.9, "completions/mean_terminated_length": 229.64642944335938, "completions/min_length": 155.2, "completions/min_terminated_length": 155.2, "entropy": 1.0732648648321628, "epoch": 0.35855474855199043, "frac_reward_zero_std": 0.3, "grad_norm": 6.840244770050049, "kl": 0.02413992037763819, "learning_rate": 4.934308942952049e-06, "loss": 0.0015, "num_tokens": 16882065.0, "reward": 0.06875, "reward_std": 0.4280139446258545, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.42801395058631897, "sampling/importance_sampling_ratio/max": 1.2515279054641724, "sampling/importance_sampling_ratio/mean": 0.9998293399810791, "sampling/importance_sampling_ratio/min": 0.7841599225997925, "sampling/sampling_logp_difference/max": 0.25341911911964415, "sampling/sampling_logp_difference/mean": 0.02119055576622486, "step": 3900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.0, "completions/max_terminated_length": 276.0, "completions/mean_length": 202.475, "completions/mean_terminated_length": 202.475, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0567621059715748, "epoch": 0.35947411970212373, "frac_reward_zero_std": 0.6, "grad_norm": 5.396238803863525, "kl": 0.024067029828438537, "learning_rate": 4.933730702674899e-06, "loss": 0.001, "num_tokens": 16926479.0, "reward": -0.1875, "reward_std": 0.20580926835536956, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.20580926835536956, "sampling/importance_sampling_ratio/max": 1.2392575860023498, "sampling/importance_sampling_ratio/mean": 1.000095546245575, "sampling/importance_sampling_ratio/min": 0.7632444858551025, "sampling/sampling_logp_difference/max": 0.2875574231147766, "sampling/sampling_logp_difference/mean": 0.020880523882806302, "step": 3910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.7, "completions/max_terminated_length": 266.7, "completions/mean_length": 197.0875, "completions/mean_terminated_length": 197.0875, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.0886995639652013, "epoch": 0.3603934908522571, "frac_reward_zero_std": 0.2, "grad_norm": 4.988309383392334, "kl": 0.030136342253535987, "learning_rate": 4.933149962761231e-06, "loss": 0.0012, "num_tokens": 16968718.0, "reward": 0.45, "reward_std": 0.5064159631729126, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.506415969133377, "sampling/importance_sampling_ratio/max": 1.2369025349617004, "sampling/importance_sampling_ratio/mean": 1.000149667263031, "sampling/importance_sampling_ratio/min": 0.7756860613822937, "sampling/sampling_logp_difference/max": 0.26561460494995115, "sampling/sampling_logp_difference/mean": 0.02201034687459469, "step": 3920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 204.3375, "completions/mean_terminated_length": 204.3375, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.031305456161499, "epoch": 0.3613128620023904, "frac_reward_zero_std": 0.2, "grad_norm": 4.9556803703308105, "kl": 0.02554867728613317, "learning_rate": 4.93256672380751e-06, "loss": 0.002, "num_tokens": 17012417.0, "reward": -0.05, "reward_std": 0.5804221421480179, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.5804221600294113, "sampling/importance_sampling_ratio/max": 1.3038613438606261, "sampling/importance_sampling_ratio/mean": 1.0000344038009643, "sampling/importance_sampling_ratio/min": 0.7963452279567719, "sampling/sampling_logp_difference/max": 0.2828441858291626, "sampling/sampling_logp_difference/mean": 0.020876816473901272, "step": 3930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.7, "completions/max_terminated_length": 255.7, "completions/mean_length": 183.275, "completions/mean_terminated_length": 183.275, "completions/min_length": 111.5, "completions/min_terminated_length": 111.5, "entropy": 1.05137086994946, "epoch": 0.3622322331525237, "frac_reward_zero_std": 0.1, "grad_norm": 7.099802017211914, "kl": 0.03895709650823846, "learning_rate": 4.931980986412769e-06, "loss": 0.0018, "num_tokens": 17053023.0, "reward": 0.075, "reward_std": 0.5395444571971894, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5395444750785827, "sampling/importance_sampling_ratio/max": 1.2688334822654723, "sampling/importance_sampling_ratio/mean": 1.0002437829971313, "sampling/importance_sampling_ratio/min": 0.7317319363355637, "sampling/sampling_logp_difference/max": 0.3511121034622192, "sampling/sampling_logp_difference/mean": 0.021704531740397215, "step": 3940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.2, "completions/max_terminated_length": 294.2, "completions/mean_length": 219.8875, "completions/mean_terminated_length": 219.8875, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.1323226742446422, "epoch": 0.36315160430265697, "frac_reward_zero_std": 0.2, "grad_norm": 6.40850305557251, "kl": 0.028351474611554295, "learning_rate": 4.931392751178608e-06, "loss": 0.0016, "num_tokens": 17100030.0, "reward": -0.1875, "reward_std": 0.421695351600647, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.42169535756111143, "sampling/importance_sampling_ratio/max": 1.2507598400115967, "sampling/importance_sampling_ratio/mean": 0.9999391674995423, "sampling/importance_sampling_ratio/min": 0.7372199535369873, "sampling/sampling_logp_difference/max": 0.33733655512332916, "sampling/sampling_logp_difference/mean": 0.021574048697948454, "step": 3950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.0, "completions/max_terminated_length": 325.0, "completions/mean_length": 228.9375, "completions/mean_terminated_length": 228.9375, "completions/min_length": 151.1, "completions/min_terminated_length": 151.1, "entropy": 1.1639699287712575, "epoch": 0.36407097545279027, "frac_reward_zero_std": 0.3, "grad_norm": 0.5508328676223755, "kl": 0.02543405604083091, "learning_rate": 4.930802018709192e-06, "loss": 0.001, "num_tokens": 17144409.0, "reward": -0.11875, "reward_std": 0.4696974903345108, "rewards/reward_fn/mean": -0.11875, "rewards/reward_fn/std": 0.46969751417636874, "sampling/importance_sampling_ratio/max": 1.3100485801696777, "sampling/importance_sampling_ratio/mean": 1.0001014530658723, "sampling/importance_sampling_ratio/min": 0.7754747748374939, "sampling/sampling_logp_difference/max": 0.3061722874641418, "sampling/sampling_logp_difference/mean": 0.021867893263697626, "step": 3960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 205.8875, "completions/mean_terminated_length": 205.8875, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 1.0047426655888558, "epoch": 0.3649903466029236, "frac_reward_zero_std": 0.4, "grad_norm": 0.11208415031433105, "kl": 0.022424453136045484, "learning_rate": 4.9302087896112506e-06, "loss": 0.0011, "num_tokens": 17186288.0, "reward": 0.0875, "reward_std": 0.2857582807540894, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.28575828671455383, "sampling/importance_sampling_ratio/max": 1.2722222805023193, "sampling/importance_sampling_ratio/mean": 0.9997186779975891, "sampling/importance_sampling_ratio/min": 0.7932699680328369, "sampling/sampling_logp_difference/max": 0.2713023662567139, "sampling/sampling_logp_difference/mean": 0.020550216920673846, "step": 3970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 196.1125, "completions/mean_terminated_length": 196.1125, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0293453693389893, "epoch": 0.3659097177530569, "frac_reward_zero_std": 0.2, "grad_norm": 5.847952842712402, "kl": 0.028373274678597227, "learning_rate": 4.929613064494076e-06, "loss": 0.0018, "num_tokens": 17225521.0, "reward": -0.075, "reward_std": 0.4863525778055191, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.48635258376598356, "sampling/importance_sampling_ratio/max": 1.330295491218567, "sampling/importance_sampling_ratio/mean": 1.0002269804477693, "sampling/importance_sampling_ratio/min": 0.7863039195537567, "sampling/sampling_logp_difference/max": 0.3064713627099991, "sampling/sampling_logp_difference/mean": 0.020670720376074314, "step": 3980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.8, "completions/max_terminated_length": 288.8, "completions/mean_length": 202.875, "completions/mean_terminated_length": 202.875, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.024110032618046, "epoch": 0.3668290889031902, "frac_reward_zero_std": 0.4, "grad_norm": 5.5246806144714355, "kl": 0.022712926811072974, "learning_rate": 4.929014843969527e-06, "loss": 0.0012, "num_tokens": 17266399.0, "reward": 0.19375, "reward_std": 0.35878387838602066, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.35878387838602066, "sampling/importance_sampling_ratio/max": 1.2430195927619934, "sampling/importance_sampling_ratio/mean": 1.0006589531898498, "sampling/importance_sampling_ratio/min": 0.7966007173061371, "sampling/sampling_logp_difference/max": 0.2463855743408203, "sampling/sampling_logp_difference/mean": 0.02132015787065029, "step": 3990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.6, "completions/max_terminated_length": 272.6, "completions/mean_length": 190.9, "completions/mean_terminated_length": 190.9, "completions/min_length": 114.6, "completions/min_terminated_length": 114.6, "entropy": 1.0733345195651054, "epoch": 0.3677484600533235, "frac_reward_zero_std": 0.3, "grad_norm": 7.040745735168457, "kl": 0.023259436117950828, "learning_rate": 4.9284141286520245e-06, "loss": 0.0016, "num_tokens": 17308143.0, "reward": -0.1125, "reward_std": 0.4148233026266098, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.41482333838939667, "sampling/importance_sampling_ratio/max": 1.3117130279541016, "sampling/importance_sampling_ratio/mean": 1.000256496667862, "sampling/importance_sampling_ratio/min": 0.792364090681076, "sampling/sampling_logp_difference/max": 0.2844870686531067, "sampling/sampling_logp_difference/mean": 0.021840430423617364, "step": 4000 }, { "epoch": 0.3677484600533235, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 283.88132635253055, "eval_completions/max_terminated_length": 282.68411867364745, "eval_completions/mean_length": 201.9705497382199, "eval_completions/mean_terminated_length": 201.58239844623452, "eval_completions/min_length": 132.88132635253055, "eval_completions/min_terminated_length": 132.88132635253055, "eval_entropy": 1.0520488543348163, "eval_frac_reward_zero_std": 0.29668411867364747, "eval_kl": 0.030604112914472455, "eval_loss": 0.001616702415049076, "eval_num_tokens": 17308143.0, "eval_reward": 0.031195462478184992, "eval_reward_std": 0.40771024956337026, "eval_rewards/reward_fn/mean": 0.031195462478184992, "eval_rewards/reward_fn/std": 0.4077102613178639, "eval_runtime": 1666.3235, "eval_samples_per_second": 0.344, "eval_sampling/importance_sampling_ratio/max": 1.271846962848883, "eval_sampling/importance_sampling_ratio/mean": 0.9999645783014947, "eval_sampling/importance_sampling_ratio/min": 0.7824295284340311, "eval_sampling/sampling_logp_difference/max": 0.273646653660721, "eval_sampling/sampling_logp_difference/mean": 0.020698034389831945, "eval_steps_per_second": 0.043, "step": 4000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 280.5, "completions/max_terminated_length": 264.5, "completions/mean_length": 204.2375, "completions/mean_terminated_length": 201.2517883300781, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 0.9521341491490602, "epoch": 0.3686678312034568, "frac_reward_zero_std": 0.3, "grad_norm": 0.164662167429924, "kl": 0.02677487238543108, "learning_rate": 4.92781091915855e-06, "loss": 0.0017, "num_tokens": 17349866.0, "reward": 0.23125, "reward_std": 0.44940980672836306, "rewards/reward_fn/mean": 0.23125, "rewards/reward_fn/std": 0.4494098246097565, "sampling/importance_sampling_ratio/max": 1.2505493760108948, "sampling/importance_sampling_ratio/mean": 0.9997177422046661, "sampling/importance_sampling_ratio/min": 0.7825440526008606, "sampling/sampling_logp_difference/max": 0.25838118195533755, "sampling/sampling_logp_difference/mean": 0.019815102312713863, "step": 4010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.5, "completions/max_terminated_length": 323.5, "completions/mean_length": 230.4625, "completions/mean_terminated_length": 230.4625, "completions/min_length": 154.1, "completions/min_terminated_length": 154.1, "entropy": 1.0772514939308167, "epoch": 0.36958720235359016, "frac_reward_zero_std": 0.4, "grad_norm": 5.355861186981201, "kl": 0.030957560474053025, "learning_rate": 4.927205216108649e-06, "loss": 0.0013, "num_tokens": 17396319.0, "reward": 0.125, "reward_std": 0.35137554705142976, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3513755530118942, "sampling/importance_sampling_ratio/max": 1.2867622375488281, "sampling/importance_sampling_ratio/mean": 0.9999241530895233, "sampling/importance_sampling_ratio/min": 0.7290894627571106, "sampling/sampling_logp_difference/max": 0.33557829856872556, "sampling/sampling_logp_difference/mean": 0.021585827134549616, "step": 4020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 238.8875, "completions/mean_terminated_length": 238.8875, "completions/min_length": 167.3, "completions/min_terminated_length": 167.3, "entropy": 1.1207863569259644, "epoch": 0.37050657350372346, "frac_reward_zero_std": 0.5, "grad_norm": 6.0623884201049805, "kl": 0.022352238243911415, "learning_rate": 4.926597020124428e-06, "loss": 0.0012, "num_tokens": 17445030.0, "reward": -0.45, "reward_std": 0.28049272000789643, "rewards/reward_fn/mean": -0.45, "rewards/reward_fn/std": 0.28049272000789643, "sampling/importance_sampling_ratio/max": 1.2419703960418702, "sampling/importance_sampling_ratio/mean": 0.9994587421417236, "sampling/importance_sampling_ratio/min": 0.7675145030021667, "sampling/sampling_logp_difference/max": 0.27184714674949645, "sampling/sampling_logp_difference/mean": 0.021614385396242143, "step": 4030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.9, "completions/max_terminated_length": 295.8, "completions/mean_length": 217.925, "completions/mean_terminated_length": 215.00357360839843, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 0.9822823759168386, "epoch": 0.37142594465385675, "frac_reward_zero_std": 0.5, "grad_norm": 6.078151226043701, "kl": 0.023352814401732758, "learning_rate": 4.92598633183055e-06, "loss": 0.0011, "num_tokens": 17485512.0, "reward": -0.01875, "reward_std": 0.3300223082304001, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.3300223141908646, "sampling/importance_sampling_ratio/max": 1.262546670436859, "sampling/importance_sampling_ratio/mean": 0.9999734222888946, "sampling/importance_sampling_ratio/min": 0.7684449434280396, "sampling/sampling_logp_difference/max": 0.3010485053062439, "sampling/sampling_logp_difference/mean": 0.019567642360925674, "step": 4040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 327.6, "completions/max_terminated_length": 318.1, "completions/mean_length": 222.1375, "completions/mean_terminated_length": 220.26964416503907, "completions/min_length": 147.0, "completions/min_terminated_length": 147.0, "entropy": 1.093243619799614, "epoch": 0.37234531580399005, "frac_reward_zero_std": 0.2, "grad_norm": 9.030506134033203, "kl": 0.022978481504833326, "learning_rate": 4.925373151854244e-06, "loss": 0.0013, "num_tokens": 17528347.0, "reward": 0.3, "reward_std": 0.5721349716186523, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.5721349954605103, "sampling/importance_sampling_ratio/max": 1.2676013708114624, "sampling/importance_sampling_ratio/mean": 1.000027734041214, "sampling/importance_sampling_ratio/min": 0.7944826364517212, "sampling/sampling_logp_difference/max": 0.2545661449432373, "sampling/sampling_logp_difference/mean": 0.021137323044240475, "step": 4050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.1, "completions/max_terminated_length": 284.1, "completions/mean_length": 211.1625, "completions/mean_terminated_length": 211.1625, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.0667280025780201, "epoch": 0.3732646869541234, "frac_reward_zero_std": 0.3, "grad_norm": 0.14511102437973022, "kl": 0.031603105168323965, "learning_rate": 4.924757480825292e-06, "loss": 0.001, "num_tokens": 17569384.0, "reward": -0.1375, "reward_std": 0.36135117411613465, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.3613511800765991, "sampling/importance_sampling_ratio/max": 1.2820291757583617, "sampling/importance_sampling_ratio/mean": 0.9999892592430115, "sampling/importance_sampling_ratio/min": 0.7780540645122528, "sampling/sampling_logp_difference/max": 0.27045043706893923, "sampling/sampling_logp_difference/mean": 0.021450281143188477, "step": 4060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.3, "completions/max_terminated_length": 309.3, "completions/mean_length": 215.65, "completions/mean_terminated_length": 215.65, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.01245990768075, "epoch": 0.3741840581042567, "frac_reward_zero_std": 0.2, "grad_norm": 4.895285606384277, "kl": 0.030177100194850936, "learning_rate": 4.92413931937604e-06, "loss": 0.0012, "num_tokens": 17614092.0, "reward": -0.3125, "reward_std": 0.43944525718688965, "rewards/reward_fn/mean": -0.3125, "rewards/reward_fn/std": 0.43944527506828307, "sampling/importance_sampling_ratio/max": 1.2394626140594482, "sampling/importance_sampling_ratio/mean": 0.9996505796909332, "sampling/importance_sampling_ratio/min": 0.7653551936149597, "sampling/sampling_logp_difference/max": 0.27353883981704713, "sampling/sampling_logp_difference/mean": 0.019894703291356564, "step": 4070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 233.2125, "completions/mean_terminated_length": 233.2125, "completions/min_length": 157.3, "completions/min_terminated_length": 157.3, "entropy": 1.1959784880280495, "epoch": 0.37510342925439, "frac_reward_zero_std": 0.2, "grad_norm": 7.870891094207764, "kl": 0.03600883090402931, "learning_rate": 4.923518668141388e-06, "loss": 0.0017, "num_tokens": 17660477.0, "reward": -0.325, "reward_std": 0.4479423314332962, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.44794235527515414, "sampling/importance_sampling_ratio/max": 1.3080520272254943, "sampling/importance_sampling_ratio/mean": 1.0002026081085205, "sampling/importance_sampling_ratio/min": 0.7897085726261139, "sampling/sampling_logp_difference/max": 0.29439462423324586, "sampling/sampling_logp_difference/mean": 0.022370431944727896, "step": 4080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.2, "completions/max_terminated_length": 299.2, "completions/mean_length": 225.125, "completions/mean_terminated_length": 225.125, "completions/min_length": 158.9, "completions/min_terminated_length": 158.9, "entropy": 1.1307223968207836, "epoch": 0.3760228004045233, "frac_reward_zero_std": 0.2, "grad_norm": 4.4964213371276855, "kl": 0.029724794812500475, "learning_rate": 4.922895527758795e-06, "loss": 0.0019, "num_tokens": 17705735.0, "reward": 0.175, "reward_std": 0.43609668016433717, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.4360966980457306, "sampling/importance_sampling_ratio/max": 1.2420305728912353, "sampling/importance_sampling_ratio/mean": 1.0001307249069213, "sampling/importance_sampling_ratio/min": 0.7889919281005859, "sampling/sampling_logp_difference/max": 0.25029834508895876, "sampling/sampling_logp_difference/mean": 0.021047768741846086, "step": 4090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.2, "completions/max_terminated_length": 287.2, "completions/mean_length": 206.3875, "completions/mean_terminated_length": 206.3875, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.1340158827602864, "epoch": 0.3769421715546566, "frac_reward_zero_std": 0.2, "grad_norm": 5.3849077224731445, "kl": 0.038168801891151814, "learning_rate": 4.9222698988682775e-06, "loss": 0.0018, "num_tokens": 17748926.0, "reward": 0.25, "reward_std": 0.44958853125572207, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.4495885372161865, "sampling/importance_sampling_ratio/max": 1.2284218430519105, "sampling/importance_sampling_ratio/mean": 1.0005335867404939, "sampling/importance_sampling_ratio/min": 0.7881789267063141, "sampling/sampling_logp_difference/max": 0.2432108163833618, "sampling/sampling_logp_difference/mean": 0.021849458292126655, "step": 4100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 294.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 200.05, "completions/mean_terminated_length": 196.09285736083984, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.069764541834593, "epoch": 0.37786154270478994, "frac_reward_zero_std": 0.1, "grad_norm": 0.13810424506664276, "kl": 0.04081633408786729, "learning_rate": 4.921641782112404e-06, "loss": 0.0018, "num_tokens": 17791706.0, "reward": 0.15625, "reward_std": 0.4460426539182663, "rewards/reward_fn/mean": 0.15625, "rewards/reward_fn/std": 0.44604267179965973, "sampling/importance_sampling_ratio/max": 1.262696921825409, "sampling/importance_sampling_ratio/mean": 0.999956876039505, "sampling/importance_sampling_ratio/min": 0.7846767902374268, "sampling/sampling_logp_difference/max": 0.27911136150360105, "sampling/sampling_logp_difference/mean": 0.021073807775974274, "step": 4110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.3, "completions/max_terminated_length": 288.3, "completions/mean_length": 207.5375, "completions/mean_terminated_length": 207.5375, "completions/min_length": 148.2, "completions/min_terminated_length": 148.2, "entropy": 1.026875903457403, "epoch": 0.37878091385492324, "frac_reward_zero_std": 0.5, "grad_norm": 0.1792694628238678, "kl": 0.0306867724400945, "learning_rate": 4.921011178136302e-06, "loss": 0.0014, "num_tokens": 17834725.0, "reward": 0.5875, "reward_std": 0.24493903219699859, "rewards/reward_fn/mean": 0.5875, "rewards/reward_fn/std": 0.24493903219699859, "sampling/importance_sampling_ratio/max": 1.2497349858283997, "sampling/importance_sampling_ratio/mean": 1.000208204984665, "sampling/importance_sampling_ratio/min": 0.7569918751716613, "sampling/sampling_logp_difference/max": 0.2868160128593445, "sampling/sampling_logp_difference/mean": 0.020739179104566574, "step": 4120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.6, "completions/max_terminated_length": 323.6, "completions/mean_length": 232.8625, "completions/mean_terminated_length": 232.8625, "completions/min_length": 151.4, "completions/min_terminated_length": 151.4, "entropy": 1.086667349934578, "epoch": 0.37970028500505654, "frac_reward_zero_std": 0.4, "grad_norm": 0.09945480525493622, "kl": 0.03359981171088293, "learning_rate": 4.920378087587652e-06, "loss": 0.0017, "num_tokens": 17880450.0, "reward": -0.2, "reward_std": 0.3309396654367447, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3309396654367447, "sampling/importance_sampling_ratio/max": 1.2475878477096558, "sampling/importance_sampling_ratio/mean": 0.9995445549488068, "sampling/importance_sampling_ratio/min": 0.7682983756065369, "sampling/sampling_logp_difference/max": 0.2868174374103546, "sampling/sampling_logp_difference/mean": 0.021320104971528052, "step": 4130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.4, "completions/max_terminated_length": 338.4, "completions/mean_length": 229.925, "completions/mean_terminated_length": 229.925, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.115572091192007, "epoch": 0.38061965615518983, "frac_reward_zero_std": 0.0, "grad_norm": 5.636646747589111, "kl": 0.040472197683993724, "learning_rate": 4.91974251111669e-06, "loss": 0.0019, "num_tokens": 17924724.0, "reward": 0.0625, "reward_std": 0.5299923837184906, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.529992401599884, "sampling/importance_sampling_ratio/max": 1.2865466952323914, "sampling/importance_sampling_ratio/mean": 0.9998015522956848, "sampling/importance_sampling_ratio/min": 0.7645593822002411, "sampling/sampling_logp_difference/max": 0.2887804269790649, "sampling/sampling_logp_difference/mean": 0.021381690725684167, "step": 4140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 199.825, "completions/mean_terminated_length": 199.825, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.1013034857809543, "epoch": 0.3815390273053232, "frac_reward_zero_std": 0.5, "grad_norm": 5.736066818237305, "kl": 0.04292634167941287, "learning_rate": 4.919104449376202e-06, "loss": 0.0019, "num_tokens": 17966302.0, "reward": 0.275, "reward_std": 0.26036913990974425, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.26036914587020876, "sampling/importance_sampling_ratio/max": 1.2266042232513428, "sampling/importance_sampling_ratio/mean": 0.99991774559021, "sampling/importance_sampling_ratio/min": 0.7998579144477844, "sampling/sampling_logp_difference/max": 0.2290733575820923, "sampling/sampling_logp_difference/mean": 0.021233107335865496, "step": 4150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 214.05, "completions/mean_terminated_length": 214.05, "completions/min_length": 139.3, "completions/min_terminated_length": 139.3, "entropy": 1.0977688491344453, "epoch": 0.3824583984554565, "frac_reward_zero_std": 0.2, "grad_norm": 5.757287979125977, "kl": 0.03785655811661855, "learning_rate": 4.9184639030215295e-06, "loss": 0.0017, "num_tokens": 18011130.0, "reward": 0.1, "reward_std": 0.5383017629384994, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.5383017748594284, "sampling/importance_sampling_ratio/max": 1.2806958079338073, "sampling/importance_sampling_ratio/mean": 1.00012549161911, "sampling/importance_sampling_ratio/min": 0.7377934008836746, "sampling/sampling_logp_difference/max": 0.32518887519836426, "sampling/sampling_logp_difference/mean": 0.021232788637280463, "step": 4160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 195.5375, "completions/mean_terminated_length": 195.5375, "completions/min_length": 116.2, "completions/min_terminated_length": 116.2, "entropy": 1.1318380936980248, "epoch": 0.3833777696055898, "frac_reward_zero_std": 0.3, "grad_norm": 7.340631484985352, "kl": 0.04119705901248381, "learning_rate": 4.917820872710566e-06, "loss": 0.0024, "num_tokens": 18054781.0, "reward": -0.15, "reward_std": 0.32111361920833587, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3211136251688004, "sampling/importance_sampling_ratio/max": 1.2262367010116577, "sampling/importance_sampling_ratio/mean": 1.0000063061714173, "sampling/importance_sampling_ratio/min": 0.7953880190849304, "sampling/sampling_logp_difference/max": 0.23239219188690186, "sampling/sampling_logp_difference/mean": 0.021214174479246138, "step": 4170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.7, "completions/max_terminated_length": 265.7, "completions/mean_length": 186.3625, "completions/mean_terminated_length": 186.3625, "completions/min_length": 119.5, "completions/min_terminated_length": 119.5, "entropy": 1.0833267003297806, "epoch": 0.3842971407557231, "frac_reward_zero_std": 0.2, "grad_norm": 6.464057922363281, "kl": 0.05841591847129166, "learning_rate": 4.917175359103753e-06, "loss": 0.0029, "num_tokens": 18096610.0, "reward": -0.1125, "reward_std": 0.43262317180633547, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.4326231896877289, "sampling/importance_sampling_ratio/max": 1.2406950116157531, "sampling/importance_sampling_ratio/mean": 1.0000700950622559, "sampling/importance_sampling_ratio/min": 0.7931158900260925, "sampling/sampling_logp_difference/max": 0.24209141731262207, "sampling/sampling_logp_difference/mean": 0.02181189451366663, "step": 4180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.7, "completions/max_terminated_length": 260.7, "completions/mean_length": 185.875, "completions/mean_terminated_length": 185.875, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.0153455756604672, "epoch": 0.3852165119058564, "frac_reward_zero_std": 0.1, "grad_norm": 7.1384596824646, "kl": 0.038847163936588915, "learning_rate": 4.9165273628640865e-06, "loss": 0.0024, "num_tokens": 18137008.0, "reward": 0.3875, "reward_std": 0.49628907442092896, "rewards/reward_fn/mean": 0.3875, "rewards/reward_fn/std": 0.4962890863418579, "sampling/importance_sampling_ratio/max": 1.2498883724212646, "sampling/importance_sampling_ratio/mean": 0.9994864761829376, "sampling/importance_sampling_ratio/min": 0.7944707155227662, "sampling/sampling_logp_difference/max": 0.25554182529449465, "sampling/sampling_logp_difference/mean": 0.020502215065062047, "step": 4190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.6, "completions/max_terminated_length": 275.6, "completions/mean_length": 185.6, "completions/mean_terminated_length": 185.6, "completions/min_length": 114.3, "completions/min_terminated_length": 114.3, "entropy": 1.0363887690007687, "epoch": 0.3861358830559897, "frac_reward_zero_std": 0.0, "grad_norm": 7.004512786865234, "kl": 0.048218690371140835, "learning_rate": 4.91587688465711e-06, "loss": 0.0022, "num_tokens": 18180288.0, "reward": -0.025, "reward_std": 0.48346473276615143, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.48346475064754485, "sampling/importance_sampling_ratio/max": 1.270919132232666, "sampling/importance_sampling_ratio/mean": 0.9998312711715698, "sampling/importance_sampling_ratio/min": 0.7961426794528961, "sampling/sampling_logp_difference/max": 0.26231467723846436, "sampling/sampling_logp_difference/mean": 0.02152935937047005, "step": 4200 }, { "epoch": 0.3861358830559897, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0004363001745200698, "eval_completions/max_length": 273.195462478185, "eval_completions/max_terminated_length": 272.6561954624782, "eval_completions/mean_length": 194.6014397905759, "eval_completions/mean_terminated_length": 194.48382579011235, "eval_completions/min_length": 125.17102966841186, "eval_completions/min_terminated_length": 125.17102966841186, "eval_entropy": 1.0418714970922387, "eval_frac_reward_zero_std": 0.2844677137870855, "eval_kl": 0.042036805220637435, "eval_loss": 0.002092598704621196, "eval_num_tokens": 18180288.0, "eval_reward": 0.05606457242582897, "eval_reward_std": 0.368489447584535, "eval_rewards/reward_fn/mean": 0.05606457242582897, "eval_rewards/reward_fn/std": 0.3684894585068521, "eval_runtime": 1611.9064, "eval_samples_per_second": 0.355, "eval_sampling/importance_sampling_ratio/max": 1.2763111385375416, "eval_sampling/importance_sampling_ratio/mean": 0.9999771187858848, "eval_sampling/importance_sampling_ratio/min": 0.77395340431007, "eval_sampling/sampling_logp_difference/max": 0.2815808745891012, "eval_sampling/sampling_logp_difference/mean": 0.021152757574685477, "eval_steps_per_second": 0.045, "step": 4200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.5, "completions/max_terminated_length": 260.5, "completions/mean_length": 197.9875, "completions/mean_terminated_length": 197.9875, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.003240905329585, "epoch": 0.387055254206123, "frac_reward_zero_std": 0.4, "grad_norm": 8.37614917755127, "kl": 0.14689252984244378, "learning_rate": 4.915223925150916e-06, "loss": 0.0078, "num_tokens": 18221527.0, "reward": 0.2125, "reward_std": 0.29291952252388, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.2929195284843445, "sampling/importance_sampling_ratio/max": 1.259115171432495, "sampling/importance_sampling_ratio/mean": 0.9999430298805236, "sampling/importance_sampling_ratio/min": 0.7786946892738342, "sampling/sampling_logp_difference/max": 0.26787192821502687, "sampling/sampling_logp_difference/mean": 0.020318554993718863, "step": 4210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.9, "completions/max_terminated_length": 263.9, "completions/mean_length": 186.3625, "completions/mean_terminated_length": 186.3625, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 1.0737589441239834, "epoch": 0.3879746253562563, "frac_reward_zero_std": 0.2, "grad_norm": 7.920482158660889, "kl": 0.036211347358766945, "learning_rate": 4.914568485016148e-06, "loss": 0.0019, "num_tokens": 18261156.0, "reward": -0.05625, "reward_std": 0.40856347382068636, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.4085634797811508, "sampling/importance_sampling_ratio/max": 1.2665090441703797, "sampling/importance_sampling_ratio/mean": 1.000233346223831, "sampling/importance_sampling_ratio/min": 0.7647046029567719, "sampling/sampling_logp_difference/max": 0.2854976117610931, "sampling/sampling_logp_difference/mean": 0.021909939125180244, "step": 4220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 215.5125, "completions/mean_terminated_length": 215.5125, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.0910020902752877, "epoch": 0.3888939965063896, "frac_reward_zero_std": 0.1, "grad_norm": 5.885628700256348, "kl": 0.06614169023232534, "learning_rate": 4.9139105649259945e-06, "loss": 0.0031, "num_tokens": 18308045.0, "reward": -0.3375, "reward_std": 0.4683582723140717, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.46835830211639407, "sampling/importance_sampling_ratio/max": 1.285937774181366, "sampling/importance_sampling_ratio/mean": 1.0004385709762573, "sampling/importance_sampling_ratio/min": 0.8008832097053528, "sampling/sampling_logp_difference/max": 0.26284568309783934, "sampling/sampling_logp_difference/mean": 0.021850558184087276, "step": 4230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.1, "completions/max_terminated_length": 246.1, "completions/mean_length": 180.125, "completions/mean_terminated_length": 180.125, "completions/min_length": 115.1, "completions/min_terminated_length": 115.1, "entropy": 1.0903586514294148, "epoch": 0.3898133676565229, "frac_reward_zero_std": 0.3, "grad_norm": 0.1424350142478943, "kl": 0.054549929895438255, "learning_rate": 4.913250165556192e-06, "loss": 0.0021, "num_tokens": 18347559.0, "reward": -0.1875, "reward_std": 0.3483661413192749, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.3483661592006683, "sampling/importance_sampling_ratio/max": 1.242164146900177, "sampling/importance_sampling_ratio/mean": 0.9999526917934418, "sampling/importance_sampling_ratio/min": 0.7780108630657196, "sampling/sampling_logp_difference/max": 0.2678714394569397, "sampling/sampling_logp_difference/mean": 0.022136073932051657, "step": 4240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.9, "completions/max_terminated_length": 294.9, "completions/mean_length": 200.1, "completions/mean_terminated_length": 200.1, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0689876221120358, "epoch": 0.39073273880665627, "frac_reward_zero_std": 0.3, "grad_norm": 6.241642475128174, "kl": 0.05478483536862768, "learning_rate": 4.912587287585023e-06, "loss": 0.0028, "num_tokens": 18389223.0, "reward": 0.1125, "reward_std": 0.3257186770439148, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.32571869492530825, "sampling/importance_sampling_ratio/max": 1.30518639087677, "sampling/importance_sampling_ratio/mean": 1.0006941497325896, "sampling/importance_sampling_ratio/min": 0.7768790602684021, "sampling/sampling_logp_difference/max": 0.2857847332954407, "sampling/sampling_logp_difference/mean": 0.021296378038823605, "step": 4250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.0, "completions/max_terminated_length": 319.0, "completions/mean_length": 226.6625, "completions/mean_terminated_length": 226.6625, "completions/min_length": 150.2, "completions/min_terminated_length": 150.2, "entropy": 1.116308617591858, "epoch": 0.39165210995678956, "frac_reward_zero_std": 0.1, "grad_norm": 6.940467357635498, "kl": 0.052260009263409304, "learning_rate": 4.911921931693317e-06, "loss": 0.0026, "num_tokens": 18436252.0, "reward": 0.1125, "reward_std": 0.40190111100673676, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.4019011229276657, "sampling/importance_sampling_ratio/max": 1.2438296675682068, "sampling/importance_sampling_ratio/mean": 1.0005820214748382, "sampling/importance_sampling_ratio/min": 0.7571478366851807, "sampling/sampling_logp_difference/max": 0.2950997710227966, "sampling/sampling_logp_difference/mean": 0.02172163166105747, "step": 4260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 341.2, "completions/max_terminated_length": 323.4, "completions/mean_length": 220.85, "completions/mean_terminated_length": 217.21964416503906, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.1871419869363309, "epoch": 0.39257148110692286, "frac_reward_zero_std": 0.5, "grad_norm": 5.601526737213135, "kl": 0.05504003544338047, "learning_rate": 4.91125409856445e-06, "loss": 0.0026, "num_tokens": 18480464.0, "reward": 0.175, "reward_std": 0.24116460382938384, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.24116460382938384, "sampling/importance_sampling_ratio/max": 1.2678253769874572, "sampling/importance_sampling_ratio/mean": 0.9998971939086914, "sampling/importance_sampling_ratio/min": 0.7594729900360108, "sampling/sampling_logp_difference/max": 0.2952740788459778, "sampling/sampling_logp_difference/mean": 0.022710669226944445, "step": 4270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.3, "completions/max_terminated_length": 320.3, "completions/mean_length": 216.125, "completions/mean_terminated_length": 216.125, "completions/min_length": 140.2, "completions/min_terminated_length": 140.2, "entropy": 1.1805534809827805, "epoch": 0.39349085225705616, "frac_reward_zero_std": 0.1, "grad_norm": 0.15249940752983093, "kl": 0.06016960216220468, "learning_rate": 4.910583788884336e-06, "loss": 0.0028, "num_tokens": 18523242.0, "reward": 0.075, "reward_std": 0.40020377933979034, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.40020379722118377, "sampling/importance_sampling_ratio/max": 1.2664490342140198, "sampling/importance_sampling_ratio/mean": 0.999782133102417, "sampling/importance_sampling_ratio/min": 0.7839413523674011, "sampling/sampling_logp_difference/max": 0.2572047233581543, "sampling/sampling_logp_difference/mean": 0.0226281451061368, "step": 4280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.9, "completions/max_terminated_length": 290.8, "completions/mean_length": 217.9625, "completions/mean_terminated_length": 214.2500015258789, "completions/min_length": 137.1, "completions/min_terminated_length": 137.1, "entropy": 1.0520618453621864, "epoch": 0.3944102234071895, "frac_reward_zero_std": 0.2, "grad_norm": 7.128655910491943, "kl": 0.05966647712048143, "learning_rate": 4.90991100334144e-06, "loss": 0.0029, "num_tokens": 18567279.0, "reward": -0.06875, "reward_std": 0.392415389418602, "rewards/reward_fn/mean": -0.06875, "rewards/reward_fn/std": 0.3924154072999954, "sampling/importance_sampling_ratio/max": 1.2285534381866454, "sampling/importance_sampling_ratio/mean": 0.9997376441955567, "sampling/importance_sampling_ratio/min": 0.7765109121799469, "sampling/sampling_logp_difference/max": 0.2538025975227356, "sampling/sampling_logp_difference/mean": 0.02082740664482117, "step": 4290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.9, "completions/max_terminated_length": 305.9, "completions/mean_length": 211.0, "completions/mean_terminated_length": 211.0, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.1813451118767262, "epoch": 0.3953295945573228, "frac_reward_zero_std": 0.7, "grad_norm": 0.12464196979999542, "kl": 0.049402165017090736, "learning_rate": 4.909235742626765e-06, "loss": 0.0026, "num_tokens": 18613535.0, "reward": 0.15, "reward_std": 0.13340125977993011, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.1334012657403946, "sampling/importance_sampling_ratio/max": 1.313716423511505, "sampling/importance_sampling_ratio/mean": 1.0005593836307525, "sampling/importance_sampling_ratio/min": 0.7994739353656769, "sampling/sampling_logp_difference/max": 0.28026113510131834, "sampling/sampling_logp_difference/mean": 0.021760441176593304, "step": 4300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.7, "completions/max_terminated_length": 255.7, "completions/mean_length": 195.325, "completions/mean_terminated_length": 195.325, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.1389208287000656, "epoch": 0.3962489657074561, "frac_reward_zero_std": 0.4, "grad_norm": 5.981790065765381, "kl": 0.05124405643437058, "learning_rate": 4.908558007433859e-06, "loss": 0.0025, "num_tokens": 18656009.0, "reward": -0.15, "reward_std": 0.22853160798549652, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.228531613945961, "sampling/importance_sampling_ratio/max": 1.2597496151924132, "sampling/importance_sampling_ratio/mean": 0.9997198283672333, "sampling/importance_sampling_ratio/min": 0.760997986793518, "sampling/sampling_logp_difference/max": 0.2984266757965088, "sampling/sampling_logp_difference/mean": 0.02232794966548681, "step": 4310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.4, "completions/max_terminated_length": 294.4, "completions/mean_length": 217.125, "completions/mean_terminated_length": 217.125, "completions/min_length": 148.1, "completions/min_terminated_length": 148.1, "entropy": 1.133064690977335, "epoch": 0.3971683368575894, "frac_reward_zero_std": 0.4, "grad_norm": 0.1506035327911377, "kl": 0.0594520561164245, "learning_rate": 4.907877798458809e-06, "loss": 0.0028, "num_tokens": 18701403.0, "reward": 0.1, "reward_std": 0.267578649520874, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.267578661441803, "sampling/importance_sampling_ratio/max": 1.2770514369010926, "sampling/importance_sampling_ratio/mean": 1.0003732800483705, "sampling/importance_sampling_ratio/min": 0.7623413741588593, "sampling/sampling_logp_difference/max": 0.2804722785949707, "sampling/sampling_logp_difference/mean": 0.022318172641098498, "step": 4320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 279.5, "completions/max_terminated_length": 277.5, "completions/mean_length": 210.6125, "completions/mean_terminated_length": 209.22678833007814, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0104252938181162, "epoch": 0.3980877080077227, "frac_reward_zero_std": 0.3, "grad_norm": 0.14306038618087769, "kl": 0.03761104336590506, "learning_rate": 4.907195116400245e-06, "loss": 0.0021, "num_tokens": 18743748.0, "reward": 0.26875, "reward_std": 0.3223089873790741, "rewards/reward_fn/mean": 0.26875, "rewards/reward_fn/std": 0.32230899333953855, "sampling/importance_sampling_ratio/max": 1.2912437438964843, "sampling/importance_sampling_ratio/mean": 0.9997095704078675, "sampling/importance_sampling_ratio/min": 0.7927197873592376, "sampling/sampling_logp_difference/max": 0.2589628338813782, "sampling/sampling_logp_difference/mean": 0.02098599597811699, "step": 4330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 216.85, "completions/mean_terminated_length": 216.85, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 1.0362945929169656, "epoch": 0.39900707915785605, "frac_reward_zero_std": 0.4, "grad_norm": 8.333477020263672, "kl": 0.04038164290832356, "learning_rate": 4.906509961959337e-06, "loss": 0.0023, "num_tokens": 18788640.0, "reward": 0.175, "reward_std": 0.3558045417070389, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.35580454766750336, "sampling/importance_sampling_ratio/max": 1.2765664935112, "sampling/importance_sampling_ratio/mean": 0.9996415257453919, "sampling/importance_sampling_ratio/min": 0.7695736050605774, "sampling/sampling_logp_difference/max": 0.2857661724090576, "sampling/sampling_logp_difference/mean": 0.02036453168839216, "step": 4340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.6, "completions/max_terminated_length": 275.6, "completions/mean_length": 195.4375, "completions/mean_terminated_length": 195.4375, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.1462503150105476, "epoch": 0.39992645030798935, "frac_reward_zero_std": 0.4, "grad_norm": 6.944637298583984, "kl": 0.05211303613614291, "learning_rate": 4.9058223358397924e-06, "loss": 0.0026, "num_tokens": 18831995.0, "reward": -0.275, "reward_std": 0.30157616436481477, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.3015761703252792, "sampling/importance_sampling_ratio/max": 1.2654507160186768, "sampling/importance_sampling_ratio/mean": 0.9993409037590026, "sampling/importance_sampling_ratio/min": 0.786357718706131, "sampling/sampling_logp_difference/max": 0.2626077175140381, "sampling/sampling_logp_difference/mean": 0.022217776626348495, "step": 4350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.3, "completions/max_terminated_length": 281.3, "completions/mean_length": 209.225, "completions/mean_terminated_length": 209.225, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.1078680358827113, "epoch": 0.40084582145812264, "frac_reward_zero_std": 0.0, "grad_norm": 5.837707996368408, "kl": 0.04209608676610514, "learning_rate": 4.905132238747858e-06, "loss": 0.0021, "num_tokens": 18874613.0, "reward": -0.05, "reward_std": 0.4246234506368637, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.42462346851825716, "sampling/importance_sampling_ratio/max": 1.2636314630508423, "sampling/importance_sampling_ratio/mean": 1.000173419713974, "sampling/importance_sampling_ratio/min": 0.8058146178722382, "sampling/sampling_logp_difference/max": 0.24692773818969727, "sampling/sampling_logp_difference/mean": 0.021321115642786027, "step": 4360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.8, "completions/max_terminated_length": 254.8, "completions/mean_length": 181.525, "completions/mean_terminated_length": 181.525, "completions/min_length": 126.4, "completions/min_terminated_length": 126.4, "entropy": 1.0825784616172314, "epoch": 0.40176519260825594, "frac_reward_zero_std": 0.2, "grad_norm": 6.313657283782959, "kl": 0.040845442190766336, "learning_rate": 4.904439671392321e-06, "loss": 0.0022, "num_tokens": 18915055.0, "reward": -0.05625, "reward_std": 0.40697675943374634, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.40697677731513976, "sampling/importance_sampling_ratio/max": 1.2808258652687072, "sampling/importance_sampling_ratio/mean": 0.999857610464096, "sampling/importance_sampling_ratio/min": 0.756984430551529, "sampling/sampling_logp_difference/max": 0.3004852533340454, "sampling/sampling_logp_difference/mean": 0.021898503974080087, "step": 4370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.0, "completions/max_terminated_length": 279.0, "completions/mean_length": 193.8625, "completions/mean_terminated_length": 193.8625, "completions/min_length": 111.6, "completions/min_terminated_length": 111.6, "entropy": 1.1068538449704648, "epoch": 0.40268456375838924, "frac_reward_zero_std": 0.3, "grad_norm": 0.112775519490242, "kl": 0.05176783774513751, "learning_rate": 4.9037446344845e-06, "loss": 0.0026, "num_tokens": 18958972.0, "reward": 0.00625, "reward_std": 0.35268451273441315, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.35268453061580657, "sampling/importance_sampling_ratio/max": 1.2986793398857117, "sampling/importance_sampling_ratio/mean": 0.9998490869998932, "sampling/importance_sampling_ratio/min": 0.7972592234611511, "sampling/sampling_logp_difference/max": 0.2788501262664795, "sampling/sampling_logp_difference/mean": 0.021204747818410398, "step": 4380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.5, "completions/max_terminated_length": 322.5, "completions/mean_length": 232.2625, "completions/mean_terminated_length": 232.2625, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.1205988615751266, "epoch": 0.4036039349085226, "frac_reward_zero_std": 0.0, "grad_norm": 5.673587322235107, "kl": 0.053909664938692, "learning_rate": 4.903047128738257e-06, "loss": 0.0029, "num_tokens": 19006401.0, "reward": 0.03125, "reward_std": 0.5421775996685028, "rewards/reward_fn/mean": 0.03125, "rewards/reward_fn/std": 0.5421776235103607, "sampling/importance_sampling_ratio/max": 1.2923802137374878, "sampling/importance_sampling_ratio/mean": 1.0001834630966187, "sampling/importance_sampling_ratio/min": 0.7839033603668213, "sampling/sampling_logp_difference/max": 0.2786647915840149, "sampling/sampling_logp_difference/mean": 0.021829541958868503, "step": 4390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.5, "completions/max_terminated_length": 319.5, "completions/mean_length": 228.5125, "completions/mean_terminated_length": 228.5125, "completions/min_length": 156.6, "completions/min_terminated_length": 156.6, "entropy": 1.1149950563907622, "epoch": 0.4045233060586559, "frac_reward_zero_std": 0.2, "grad_norm": 5.860590934753418, "kl": 0.044792854588013145, "learning_rate": 4.902347154869984e-06, "loss": 0.0022, "num_tokens": 19051914.0, "reward": -0.125, "reward_std": 0.4301289528608322, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.43012896478176116, "sampling/importance_sampling_ratio/max": 1.2460772275924683, "sampling/importance_sampling_ratio/mean": 0.9999591708183289, "sampling/importance_sampling_ratio/min": 0.7567124605178833, "sampling/sampling_logp_difference/max": 0.2889558672904968, "sampling/sampling_logp_difference/mean": 0.022890967689454554, "step": 4400 }, { "epoch": 0.4045233060586559, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 300.35602094240835, "eval_completions/max_terminated_length": 298.92844677137873, "eval_completions/mean_length": 210.85296684118674, "eval_completions/mean_terminated_length": 210.42673288714823, "eval_completions/min_length": 135.087260034904, "eval_completions/min_terminated_length": 135.087260034904, "eval_entropy": 1.0885618127765455, "eval_frac_reward_zero_std": 0.25654450261780104, "eval_kl": 0.043799373920987414, "eval_loss": 0.002299663843587041, "eval_num_tokens": 19051914.0, "eval_reward": 0.05399214659685864, "eval_reward_std": 0.3752189955786261, "eval_rewards/reward_fn/mean": 0.05399214659685864, "eval_rewards/reward_fn/std": 0.3752190072290976, "eval_runtime": 1751.794, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2712574837303494, "eval_sampling/importance_sampling_ratio/mean": 1.0000437070561954, "eval_sampling/importance_sampling_ratio/min": 0.7767157262324454, "eval_sampling/sampling_logp_difference/max": 0.2802350786343919, "eval_sampling/sampling_logp_difference/mean": 0.02125193019369926, "eval_steps_per_second": 0.041, "step": 4400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 190.9375, "completions/mean_terminated_length": 190.9375, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0574560962617396, "epoch": 0.4054426772087892, "frac_reward_zero_std": 0.3, "grad_norm": 6.079658508300781, "kl": 0.05369033552706241, "learning_rate": 4.901644713598612e-06, "loss": 0.0028, "num_tokens": 19090869.0, "reward": -0.0875, "reward_std": 0.29291952252388, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.2929195284843445, "sampling/importance_sampling_ratio/max": 1.2977915167808534, "sampling/importance_sampling_ratio/mean": 0.9996715247631073, "sampling/importance_sampling_ratio/min": 0.770642626285553, "sampling/sampling_logp_difference/max": 0.2975643157958984, "sampling/sampling_logp_difference/mean": 0.022062713094055653, "step": 4410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 207.025, "completions/mean_terminated_length": 207.025, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 0.9994460038840771, "epoch": 0.4063620483589225, "frac_reward_zero_std": 0.1, "grad_norm": 6.009789943695068, "kl": 0.03408454702002928, "learning_rate": 4.9009398056456045e-06, "loss": 0.0021, "num_tokens": 19132407.0, "reward": -0.2, "reward_std": 0.6041975826025009, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.6041975945234299, "sampling/importance_sampling_ratio/max": 1.331460964679718, "sampling/importance_sampling_ratio/mean": 0.9997465968132019, "sampling/importance_sampling_ratio/min": 0.7817839324474335, "sampling/sampling_logp_difference/max": 0.3036469578742981, "sampling/sampling_logp_difference/mean": 0.022120250202715398, "step": 4420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 331.0, "completions/max_terminated_length": 323.2, "completions/mean_length": 209.5875, "completions/mean_terminated_length": 206.29107208251952, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.1555320933461188, "epoch": 0.40728141950905583, "frac_reward_zero_std": 0.1, "grad_norm": 5.998329162597656, "kl": 0.04895743831875734, "learning_rate": 4.900232431734957e-06, "loss": 0.0021, "num_tokens": 19176246.0, "reward": -0.00625, "reward_std": 0.5835410565137863, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.5835410743951798, "sampling/importance_sampling_ratio/max": 1.24947350025177, "sampling/importance_sampling_ratio/mean": 1.0002531945705413, "sampling/importance_sampling_ratio/min": 0.7625175654888153, "sampling/sampling_logp_difference/max": 0.2922655284404755, "sampling/sampling_logp_difference/mean": 0.02250595297664404, "step": 4430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 224.5625, "completions/mean_terminated_length": 224.5625, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0103722549974918, "epoch": 0.4082007906591891, "frac_reward_zero_std": 0.0, "grad_norm": 6.733419895172119, "kl": 0.040536357468226925, "learning_rate": 4.899522592593199e-06, "loss": 0.0025, "num_tokens": 19218419.0, "reward": -0.1, "reward_std": 0.4372564494609833, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4372564613819122, "sampling/importance_sampling_ratio/max": 1.283979845046997, "sampling/importance_sampling_ratio/mean": 1.0002211093902589, "sampling/importance_sampling_ratio/min": 0.7649264752864837, "sampling/sampling_logp_difference/max": 0.30294662714004517, "sampling/sampling_logp_difference/mean": 0.020980802923440935, "step": 4440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.4, "completions/max_terminated_length": 329.4, "completions/mean_length": 224.725, "completions/mean_terminated_length": 224.725, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.1161303229629993, "epoch": 0.4091201618093224, "frac_reward_zero_std": 0.5, "grad_norm": 6.516714096069336, "kl": 0.060422635986469686, "learning_rate": 4.898810288949394e-06, "loss": 0.0032, "num_tokens": 19263629.0, "reward": -0.1125, "reward_std": 0.2635735034942627, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.2635735034942627, "sampling/importance_sampling_ratio/max": 1.2781274795532227, "sampling/importance_sampling_ratio/mean": 0.9999723732471466, "sampling/importance_sampling_ratio/min": 0.7407774329185486, "sampling/sampling_logp_difference/max": 0.33312050104141233, "sampling/sampling_logp_difference/mean": 0.022814949974417686, "step": 4450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.7, "completions/max_terminated_length": 317.7, "completions/mean_length": 214.5375, "completions/mean_terminated_length": 214.5375, "completions/min_length": 142.9, "completions/min_terminated_length": 142.9, "entropy": 1.046873278170824, "epoch": 0.4100395329594557, "frac_reward_zero_std": 0.2, "grad_norm": 6.215446472167969, "kl": 0.047878540342207995, "learning_rate": 4.898095521535134e-06, "loss": 0.0023, "num_tokens": 19309192.0, "reward": -0.0375, "reward_std": 0.4269921988248825, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.42699221670627596, "sampling/importance_sampling_ratio/max": 1.2364967584609985, "sampling/importance_sampling_ratio/mean": 0.9997314095497132, "sampling/importance_sampling_ratio/min": 0.783577561378479, "sampling/sampling_logp_difference/max": 0.24711763858795166, "sampling/sampling_logp_difference/mean": 0.020391737669706346, "step": 4460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.7, "completions/max_terminated_length": 311.7, "completions/mean_length": 244.8875, "completions/mean_terminated_length": 244.8875, "completions/min_length": 177.6, "completions/min_terminated_length": 177.6, "entropy": 1.1675960309803486, "epoch": 0.410958904109589, "frac_reward_zero_std": 0.4, "grad_norm": 4.997447490692139, "kl": 0.042925568885402755, "learning_rate": 4.897378291084543e-06, "loss": 0.002, "num_tokens": 19359831.0, "reward": 0.35, "reward_std": 0.26472542583942416, "rewards/reward_fn/mean": 0.35, "rewards/reward_fn/std": 0.2647254317998886, "sampling/importance_sampling_ratio/max": 1.2740905284881592, "sampling/importance_sampling_ratio/mean": 1.0000465214252472, "sampling/importance_sampling_ratio/min": 0.7818454623222351, "sampling/sampling_logp_difference/max": 0.2711315453052521, "sampling/sampling_logp_difference/mean": 0.022124696336686612, "step": 4470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.1, "completions/max_terminated_length": 329.1, "completions/mean_length": 240.7125, "completions/mean_terminated_length": 240.7125, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.146951925754547, "epoch": 0.41187827525972237, "frac_reward_zero_std": 0.2, "grad_norm": 4.70084285736084, "kl": 0.04147570205386728, "learning_rate": 4.896658598334273e-06, "loss": 0.0021, "num_tokens": 19408376.0, "reward": 0.2875, "reward_std": 0.45269069373607634, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.45269071161746977, "sampling/importance_sampling_ratio/max": 1.3358041524887085, "sampling/importance_sampling_ratio/mean": 1.00005641579628, "sampling/importance_sampling_ratio/min": 0.7707864463329315, "sampling/sampling_logp_difference/max": 0.31489416360855105, "sampling/sampling_logp_difference/mean": 0.022473588399589062, "step": 4480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.6, "completions/max_terminated_length": 291.6, "completions/mean_length": 217.875, "completions/mean_terminated_length": 217.875, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.053855038434267, "epoch": 0.41279764640985567, "frac_reward_zero_std": 0.5, "grad_norm": 7.070992946624756, "kl": 0.03861591417808086, "learning_rate": 4.8959364440235065e-06, "loss": 0.0018, "num_tokens": 19451510.0, "reward": 0.45, "reward_std": 0.22853160798549652, "rewards/reward_fn/mean": 0.45, "rewards/reward_fn/std": 0.228531613945961, "sampling/importance_sampling_ratio/max": 1.2772159934043885, "sampling/importance_sampling_ratio/mean": 1.0004786193370818, "sampling/importance_sampling_ratio/min": 0.7876056969165802, "sampling/sampling_logp_difference/max": 0.2735481858253479, "sampling/sampling_logp_difference/mean": 0.021447963453829287, "step": 4490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.9, "completions/max_terminated_length": 275.9, "completions/mean_length": 186.65, "completions/mean_terminated_length": 186.65, "completions/min_length": 118.3, "completions/min_terminated_length": 118.3, "entropy": 1.1102378405630589, "epoch": 0.41371701755998896, "frac_reward_zero_std": 0.2, "grad_norm": 0.12575484812259674, "kl": 0.0463074947358109, "learning_rate": 4.8952118288939555e-06, "loss": 0.0022, "num_tokens": 19494154.0, "reward": -0.1875, "reward_std": 0.3998240172863007, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.39982402324676514, "sampling/importance_sampling_ratio/max": 1.259126365184784, "sampling/importance_sampling_ratio/mean": 0.9999365329742431, "sampling/importance_sampling_ratio/min": 0.7282272160053254, "sampling/sampling_logp_difference/max": 0.32773895263671876, "sampling/sampling_logp_difference/mean": 0.022174909338355066, "step": 4500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.5, "completions/max_terminated_length": 307.5, "completions/mean_length": 211.2875, "completions/mean_terminated_length": 211.2875, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.043286493420601, "epoch": 0.41463638871012226, "frac_reward_zero_std": 0.3, "grad_norm": 5.417031764984131, "kl": 0.030020576249808074, "learning_rate": 4.894484753689855e-06, "loss": 0.0012, "num_tokens": 19535025.0, "reward": 0.1, "reward_std": 0.4276267558336258, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4276267737150192, "sampling/importance_sampling_ratio/max": 1.2816107273101807, "sampling/importance_sampling_ratio/mean": 0.9998485207557678, "sampling/importance_sampling_ratio/min": 0.726537162065506, "sampling/sampling_logp_difference/max": 0.34288318157196046, "sampling/sampling_logp_difference/mean": 0.021475453302264212, "step": 4510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 206.7, "completions/mean_terminated_length": 206.7, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.091755896061659, "epoch": 0.41555575986025556, "frac_reward_zero_std": 0.2, "grad_norm": 5.505802154541016, "kl": 0.0328896620485466, "learning_rate": 4.8937552191579716e-06, "loss": 0.001, "num_tokens": 19578841.0, "reward": -0.0375, "reward_std": 0.47562678158283234, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.4756267935037613, "sampling/importance_sampling_ratio/max": 1.2823106527328492, "sampling/importance_sampling_ratio/mean": 1.0006014227867126, "sampling/importance_sampling_ratio/min": 0.7985706090927124, "sampling/sampling_logp_difference/max": 0.2697652578353882, "sampling/sampling_logp_difference/mean": 0.021643362194299697, "step": 4520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.6, "completions/max_terminated_length": 326.6, "completions/mean_length": 213.3125, "completions/mean_terminated_length": 213.3125, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0473481640219688, "epoch": 0.4164751310103889, "frac_reward_zero_std": 0.1, "grad_norm": 6.6811933517456055, "kl": 0.05285923631163314, "learning_rate": 4.893023226047595e-06, "loss": 0.0028, "num_tokens": 19624122.0, "reward": 0.125, "reward_std": 0.4038677543401718, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4038677603006363, "sampling/importance_sampling_ratio/max": 1.2622378826141358, "sampling/importance_sampling_ratio/mean": 0.9996782839298248, "sampling/importance_sampling_ratio/min": 0.7822768211364746, "sampling/sampling_logp_difference/max": 0.25872786045074464, "sampling/sampling_logp_difference/mean": 0.02213543336838484, "step": 4530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.4, "completions/max_terminated_length": 263.4, "completions/mean_length": 189.4875, "completions/mean_terminated_length": 189.4875, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 0.9849121943116188, "epoch": 0.4173945021605222, "frac_reward_zero_std": 0.3, "grad_norm": 5.543390274047852, "kl": 0.036978136870311575, "learning_rate": 4.892288775110542e-06, "loss": 0.0017, "num_tokens": 19663361.0, "reward": -0.05, "reward_std": 0.4437026858329773, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.4437026917934418, "sampling/importance_sampling_ratio/max": 1.261613941192627, "sampling/importance_sampling_ratio/mean": 1.0005469858646392, "sampling/importance_sampling_ratio/min": 0.7773848116397858, "sampling/sampling_logp_difference/max": 0.2585180938243866, "sampling/sampling_logp_difference/mean": 0.021519438363611698, "step": 4540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.9, "completions/max_terminated_length": 299.9, "completions/mean_length": 208.225, "completions/mean_terminated_length": 208.225, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.0951564125716686, "epoch": 0.4183138733106555, "frac_reward_zero_std": 0.3, "grad_norm": 0.1212257593870163, "kl": 0.03991349912248552, "learning_rate": 4.891551867101151e-06, "loss": 0.0019, "num_tokens": 19707235.0, "reward": 0.1, "reward_std": 0.3504155069589615, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.35041551291942596, "sampling/importance_sampling_ratio/max": 1.2868492960929871, "sampling/importance_sampling_ratio/mean": 1.0000524044036865, "sampling/importance_sampling_ratio/min": 0.7804197371006012, "sampling/sampling_logp_difference/max": 0.2710308790206909, "sampling/sampling_logp_difference/mean": 0.023127567395567893, "step": 4550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.3, "completions/max_terminated_length": 307.3, "completions/mean_length": 203.1125, "completions/mean_terminated_length": 203.1125, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.132548500597477, "epoch": 0.4192332444607888, "frac_reward_zero_std": 0.2, "grad_norm": 1.4325469732284546, "kl": 0.04296427817316726, "learning_rate": 4.890812502776285e-06, "loss": 0.0023, "num_tokens": 19749236.0, "reward": -0.175, "reward_std": 0.32111361920833587, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3211136251688004, "sampling/importance_sampling_ratio/max": 1.2543392539024354, "sampling/importance_sampling_ratio/mean": 0.9999577879905701, "sampling/importance_sampling_ratio/min": 0.7559376358985901, "sampling/sampling_logp_difference/max": 0.3011161208152771, "sampling/sampling_logp_difference/mean": 0.022630528546869755, "step": 4560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 362.5, "completions/max_terminated_length": 362.5, "completions/mean_length": 242.5375, "completions/mean_terminated_length": 242.5375, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.0516948409378528, "epoch": 0.42015261561092215, "frac_reward_zero_std": 0.4, "grad_norm": 7.431897163391113, "kl": 0.044447302498156205, "learning_rate": 4.8900706828953335e-06, "loss": 0.0023, "num_tokens": 19799527.0, "reward": -0.0375, "reward_std": 0.3011962354183197, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.3011962354183197, "sampling/importance_sampling_ratio/max": 1.2429888486862182, "sampling/importance_sampling_ratio/mean": 0.9991742849349976, "sampling/importance_sampling_ratio/min": 0.7532529413700104, "sampling/sampling_logp_difference/max": 0.2950943112373352, "sampling/sampling_logp_difference/mean": 0.02096956018358469, "step": 4570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.8, "completions/max_terminated_length": 300.8, "completions/mean_length": 214.2375, "completions/mean_terminated_length": 214.2375, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.1176028296351432, "epoch": 0.42107198676105545, "frac_reward_zero_std": 0.2, "grad_norm": 5.951211452484131, "kl": 0.044005175190977755, "learning_rate": 4.8893264082202015e-06, "loss": 0.0023, "num_tokens": 19843434.0, "reward": -0.125, "reward_std": 0.3501461952924728, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.35014620125293733, "sampling/importance_sampling_ratio/max": 1.2603368520736695, "sampling/importance_sampling_ratio/mean": 1.000041949748993, "sampling/importance_sampling_ratio/min": 0.7734138786792755, "sampling/sampling_logp_difference/max": 0.2829380989074707, "sampling/sampling_logp_difference/mean": 0.02187867760658264, "step": 4580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 199.1, "completions/mean_terminated_length": 199.1, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 1.0559039577841758, "epoch": 0.42199135791118875, "frac_reward_zero_std": 0.2, "grad_norm": 8.331087112426758, "kl": 0.05335252875229344, "learning_rate": 4.888579679515321e-06, "loss": 0.0025, "num_tokens": 19886554.0, "reward": 0.1125, "reward_std": 0.4403586119413376, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.4403586357831955, "sampling/importance_sampling_ratio/max": 1.3077358245849608, "sampling/importance_sampling_ratio/mean": 0.9999307096004486, "sampling/importance_sampling_ratio/min": 0.7760605692863465, "sampling/sampling_logp_difference/max": 0.2890470206737518, "sampling/sampling_logp_difference/mean": 0.022006846405565737, "step": 4590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.7, "completions/max_terminated_length": 338.7, "completions/mean_length": 218.2, "completions/mean_terminated_length": 218.2, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.1244510293006897, "epoch": 0.42291072906132204, "frac_reward_zero_std": 0.2, "grad_norm": 5.269723415374756, "kl": 0.05518236457137391, "learning_rate": 4.887830497547641e-06, "loss": 0.0024, "num_tokens": 19932002.0, "reward": -0.0125, "reward_std": 0.45299161076545713, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4529916286468506, "sampling/importance_sampling_ratio/max": 1.3063086986541748, "sampling/importance_sampling_ratio/mean": 0.9997603058815002, "sampling/importance_sampling_ratio/min": 0.717989158630371, "sampling/sampling_logp_difference/max": 0.364738917350769, "sampling/sampling_logp_difference/mean": 0.02262552436441183, "step": 4600 }, { "epoch": 0.42291072906132204, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 311.68411867364745, "eval_completions/max_terminated_length": 310.0, "eval_completions/mean_length": 220.85732984293193, "eval_completions/mean_terminated_length": 220.26623053159597, "eval_completions/min_length": 142.27399650959862, "eval_completions/min_terminated_length": 142.27399650959862, "eval_entropy": 1.1087293722138563, "eval_frac_reward_zero_std": 0.27399650959860383, "eval_kl": 0.044565417282088265, "eval_loss": 0.002313245553523302, "eval_num_tokens": 19932002.0, "eval_reward": 0.05355584642233857, "eval_reward_std": 0.35679650665577795, "eval_rewards/reward_fn/mean": 0.05355584642233857, "eval_rewards/reward_fn/std": 0.35679651518558747, "eval_runtime": 1807.2424, "eval_samples_per_second": 0.317, "eval_sampling/importance_sampling_ratio/max": 1.2738187360929984, "eval_sampling/importance_sampling_ratio/mean": 1.0000183957825572, "eval_sampling/importance_sampling_ratio/min": 0.7690312488436907, "eval_sampling/sampling_logp_difference/max": 0.28902619817078007, "eval_sampling/sampling_logp_difference/mean": 0.02185382135374956, "eval_steps_per_second": 0.04, "step": 4600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 212.475, "completions/mean_terminated_length": 212.475, "completions/min_length": 153.4, "completions/min_terminated_length": 153.4, "entropy": 1.109876587241888, "epoch": 0.42383010021145534, "frac_reward_zero_std": 0.2, "grad_norm": 0.14331884682178497, "kl": 0.04352916833013296, "learning_rate": 4.887078863086633e-06, "loss": 0.0025, "num_tokens": 19974944.0, "reward": 0.2, "reward_std": 0.43530933260917665, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.43530936241149903, "sampling/importance_sampling_ratio/max": 1.276979398727417, "sampling/importance_sampling_ratio/mean": 0.999796575307846, "sampling/importance_sampling_ratio/min": 0.7827986299991607, "sampling/sampling_logp_difference/max": 0.27153608202934265, "sampling/sampling_logp_difference/mean": 0.022779017314314844, "step": 4610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 347.5, "completions/max_terminated_length": 347.5, "completions/mean_length": 239.2375, "completions/mean_terminated_length": 239.2375, "completions/min_length": 166.6, "completions/min_terminated_length": 166.6, "entropy": 1.1225914478302002, "epoch": 0.4247494713615887, "frac_reward_zero_std": 0.5, "grad_norm": 5.802147388458252, "kl": 0.0531507924082689, "learning_rate": 4.886324776904284e-06, "loss": 0.0029, "num_tokens": 20021827.0, "reward": -0.1375, "reward_std": 0.19317626953125, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.19317627549171448, "sampling/importance_sampling_ratio/max": 1.3378105282783508, "sampling/importance_sampling_ratio/mean": 0.9999402284622192, "sampling/importance_sampling_ratio/min": 0.7696754157543182, "sampling/sampling_logp_difference/max": 0.33963828086853026, "sampling/sampling_logp_difference/mean": 0.022954441793262958, "step": 4620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.1, "completions/max_terminated_length": 335.1, "completions/mean_length": 228.2125, "completions/mean_terminated_length": 228.2125, "completions/min_length": 139.9, "completions/min_terminated_length": 139.9, "entropy": 1.069108320772648, "epoch": 0.425668842511722, "frac_reward_zero_std": 0.1, "grad_norm": 5.100437164306641, "kl": 0.05007398951565847, "learning_rate": 4.885568239775101e-06, "loss": 0.0027, "num_tokens": 20066420.0, "reward": -0.05, "reward_std": 0.45408297181129453, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.4540829837322235, "sampling/importance_sampling_ratio/max": 1.24729722738266, "sampling/importance_sampling_ratio/mean": 0.999903815984726, "sampling/importance_sampling_ratio/min": 0.7635926723480224, "sampling/sampling_logp_difference/max": 0.2841993570327759, "sampling/sampling_logp_difference/mean": 0.021710199117660523, "step": 4630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.1, "completions/max_terminated_length": 315.1, "completions/mean_length": 225.4, "completions/mean_terminated_length": 225.4, "completions/min_length": 133.0, "completions/min_terminated_length": 133.0, "entropy": 1.1081184215843678, "epoch": 0.4265882136618553, "frac_reward_zero_std": 0.3, "grad_norm": 0.13854734599590302, "kl": 0.04665623733308166, "learning_rate": 4.884809252476111e-06, "loss": 0.0021, "num_tokens": 20111724.0, "reward": -0.1625, "reward_std": 0.43781724870204924, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.4378172606229782, "sampling/importance_sampling_ratio/max": 1.274485993385315, "sampling/importance_sampling_ratio/mean": 1.0005553781986236, "sampling/importance_sampling_ratio/min": 0.7626090228557587, "sampling/sampling_logp_difference/max": 0.29525926113128664, "sampling/sampling_logp_difference/mean": 0.022124290838837622, "step": 4640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.6, "completions/max_terminated_length": 306.6, "completions/mean_length": 217.4125, "completions/mean_terminated_length": 217.4125, "completions/min_length": 122.3, "completions/min_terminated_length": 122.3, "entropy": 1.1606886588037013, "epoch": 0.4275075848119886, "frac_reward_zero_std": 0.4, "grad_norm": 6.082587718963623, "kl": 0.05521926496876404, "learning_rate": 4.884047815786852e-06, "loss": 0.0024, "num_tokens": 20159229.0, "reward": 0.0375, "reward_std": 0.31461594700813295, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.3146159708499908, "sampling/importance_sampling_ratio/max": 1.325911819934845, "sampling/importance_sampling_ratio/mean": 1.0003619074821473, "sampling/importance_sampling_ratio/min": 0.7949834585189819, "sampling/sampling_logp_difference/max": 0.28434593677520753, "sampling/sampling_logp_difference/mean": 0.02268270291388035, "step": 4650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.5, "completions/max_terminated_length": 315.5, "completions/mean_length": 222.6375, "completions/mean_terminated_length": 222.6375, "completions/min_length": 153.4, "completions/min_terminated_length": 153.4, "entropy": 1.1878398574888707, "epoch": 0.42842695596212194, "frac_reward_zero_std": 0.4, "grad_norm": 5.096914768218994, "kl": 0.05556288667721674, "learning_rate": 4.883283930489381e-06, "loss": 0.0028, "num_tokens": 20204544.0, "reward": -0.175, "reward_std": 0.3554430425167084, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3554430603981018, "sampling/importance_sampling_ratio/max": 1.229722809791565, "sampling/importance_sampling_ratio/mean": 1.0001407861709595, "sampling/importance_sampling_ratio/min": 0.7895393967628479, "sampling/sampling_logp_difference/max": 0.2393401861190796, "sampling/sampling_logp_difference/mean": 0.022674289904534815, "step": 4660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 310.5, "completions/max_terminated_length": 303.2, "completions/mean_length": 216.725, "completions/mean_terminated_length": 213.97321472167968, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.0784543931484223, "epoch": 0.42934632711225523, "frac_reward_zero_std": 0.4, "grad_norm": 0.13704660534858704, "kl": 0.06340418923064135, "learning_rate": 4.88251759736827e-06, "loss": 0.003, "num_tokens": 20250114.0, "reward": 0.11875, "reward_std": 0.27900843024253846, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.2790084481239319, "sampling/importance_sampling_ratio/max": 1.3357831954956054, "sampling/importance_sampling_ratio/mean": 1.0006938338279725, "sampling/importance_sampling_ratio/min": 0.7587795197963715, "sampling/sampling_logp_difference/max": 0.3546781659126282, "sampling/sampling_logp_difference/mean": 0.02190141975879669, "step": 4670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.3, "completions/max_terminated_length": 318.3, "completions/mean_length": 220.8875, "completions/mean_terminated_length": 220.8875, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0997953221201897, "epoch": 0.43026569826238853, "frac_reward_zero_std": 0.4, "grad_norm": 6.3704142570495605, "kl": 0.06075691705918871, "learning_rate": 4.881748817210604e-06, "loss": 0.0033, "num_tokens": 20296049.0, "reward": 0.1625, "reward_std": 0.26849985122680664, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.2684998571872711, "sampling/importance_sampling_ratio/max": 1.2410308361053466, "sampling/importance_sampling_ratio/mean": 1.0001990437507629, "sampling/importance_sampling_ratio/min": 0.7861220121383667, "sampling/sampling_logp_difference/max": 0.24477518796920777, "sampling/sampling_logp_difference/mean": 0.02236610483378172, "step": 4680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.6, "completions/max_terminated_length": 305.6, "completions/mean_length": 215.2375, "completions/mean_terminated_length": 215.2375, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.0569058526307344, "epoch": 0.4311850694125218, "frac_reward_zero_std": 0.6, "grad_norm": 0.1456536501646042, "kl": 0.04264894136576913, "learning_rate": 4.880977590805983e-06, "loss": 0.0019, "num_tokens": 20340132.0, "reward": 0.325, "reward_std": 0.20411193668842315, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.20411194264888763, "sampling/importance_sampling_ratio/max": 1.3091317653656005, "sampling/importance_sampling_ratio/mean": 0.9999290585517884, "sampling/importance_sampling_ratio/min": 0.7606586873531341, "sampling/sampling_logp_difference/max": 0.32407853603363035, "sampling/sampling_logp_difference/mean": 0.021431529615074395, "step": 4690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 205.575, "completions/mean_terminated_length": 205.575, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.061689742654562, "epoch": 0.4321044405626551, "frac_reward_zero_std": 0.3, "grad_norm": 8.998540878295898, "kl": 0.04379493425949477, "learning_rate": 4.8802039189465164e-06, "loss": 0.0022, "num_tokens": 20383642.0, "reward": 0.05, "reward_std": 0.32402134537696836, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3240213692188263, "sampling/importance_sampling_ratio/max": 1.2807414531707764, "sampling/importance_sampling_ratio/mean": 1.0003158509731294, "sampling/importance_sampling_ratio/min": 0.7573738515377044, "sampling/sampling_logp_difference/max": 0.29787570238113403, "sampling/sampling_logp_difference/mean": 0.021973891928792, "step": 4700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.8, "completions/max_terminated_length": 294.8, "completions/mean_length": 197.5625, "completions/mean_terminated_length": 197.5625, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.1184421628713608, "epoch": 0.4330238117127885, "frac_reward_zero_std": 0.3, "grad_norm": 0.11805090308189392, "kl": 0.03993665155721828, "learning_rate": 4.879427802426829e-06, "loss": 0.0021, "num_tokens": 20423959.0, "reward": 0.325, "reward_std": 0.3601123929023743, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.36011239886283875, "sampling/importance_sampling_ratio/max": 1.2567843079566956, "sampling/importance_sampling_ratio/mean": 0.9999920904636384, "sampling/importance_sampling_ratio/min": 0.7700425803661346, "sampling/sampling_logp_difference/max": 0.2839848279953003, "sampling/sampling_logp_difference/mean": 0.022143154032528402, "step": 4710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.7, "completions/max_terminated_length": 296.7, "completions/mean_length": 207.0375, "completions/mean_terminated_length": 207.0375, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.0847640939056873, "epoch": 0.4339431828629218, "frac_reward_zero_std": 0.1, "grad_norm": 7.291732311248779, "kl": 0.04596530375420116, "learning_rate": 4.878649242044055e-06, "loss": 0.0027, "num_tokens": 20465938.0, "reward": 0.325, "reward_std": 0.3745658665895462, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.37456587255001067, "sampling/importance_sampling_ratio/max": 1.274799907207489, "sampling/importance_sampling_ratio/mean": 1.0002844333648682, "sampling/importance_sampling_ratio/min": 0.77854705452919, "sampling/sampling_logp_difference/max": 0.27930507659912107, "sampling/sampling_logp_difference/mean": 0.021757892705500124, "step": 4720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 199.0875, "completions/mean_terminated_length": 199.0875, "completions/min_length": 140.1, "completions/min_terminated_length": 140.1, "entropy": 1.0461763590574265, "epoch": 0.43486255401305507, "frac_reward_zero_std": 0.3, "grad_norm": 6.0319013595581055, "kl": 0.0356659253127873, "learning_rate": 4.877868238597836e-06, "loss": 0.0016, "num_tokens": 20508065.0, "reward": 0.25, "reward_std": 0.3090260297060013, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3090260297060013, "sampling/importance_sampling_ratio/max": 1.2430699944496155, "sampling/importance_sampling_ratio/mean": 1.0004068553447723, "sampling/importance_sampling_ratio/min": 0.7657820165157319, "sampling/sampling_logp_difference/max": 0.2779232501983643, "sampling/sampling_logp_difference/mean": 0.021514142118394374, "step": 4730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.5, "completions/max_terminated_length": 316.5, "completions/mean_length": 227.4375, "completions/mean_terminated_length": 227.4375, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 1.1788320749998094, "epoch": 0.43578192516318837, "frac_reward_zero_std": 0.3, "grad_norm": 0.1087602823972702, "kl": 0.04807308245217427, "learning_rate": 4.877084792890327e-06, "loss": 0.0019, "num_tokens": 20558324.0, "reward": -0.34375, "reward_std": 0.32785561978816985, "rewards/reward_fn/mean": -0.34375, "rewards/reward_fn/std": 0.32785562574863436, "sampling/importance_sampling_ratio/max": 1.2695066571235656, "sampling/importance_sampling_ratio/mean": 1.000443547964096, "sampling/importance_sampling_ratio/min": 0.7891834795475006, "sampling/sampling_logp_difference/max": 0.27142399549484253, "sampling/sampling_logp_difference/mean": 0.0224968783557415, "step": 4740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 205.475, "completions/mean_terminated_length": 205.475, "completions/min_length": 123.7, "completions/min_terminated_length": 123.7, "entropy": 1.1261117294430734, "epoch": 0.43670129631332166, "frac_reward_zero_std": 0.3, "grad_norm": 7.507055282592773, "kl": 0.050419921055436134, "learning_rate": 4.876298905726189e-06, "loss": 0.0026, "num_tokens": 20601690.0, "reward": 0.15, "reward_std": 0.3446744382381439, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3446744501590729, "sampling/importance_sampling_ratio/max": 1.251662278175354, "sampling/importance_sampling_ratio/mean": 0.9999021172523499, "sampling/importance_sampling_ratio/min": 0.7818563878536224, "sampling/sampling_logp_difference/max": 0.25606557726860046, "sampling/sampling_logp_difference/mean": 0.022135718166828154, "step": 4750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.3, "completions/max_terminated_length": 325.3, "completions/mean_length": 228.475, "completions/mean_terminated_length": 228.475, "completions/min_length": 149.5, "completions/min_terminated_length": 149.5, "entropy": 1.1918197348713875, "epoch": 0.437620667463455, "frac_reward_zero_std": 0.1, "grad_norm": 6.183346271514893, "kl": 0.04097719464916736, "learning_rate": 4.8755105779125915e-06, "loss": 0.0017, "num_tokens": 20647776.0, "reward": -0.2625, "reward_std": 0.422067266702652, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.4220672965049744, "sampling/importance_sampling_ratio/max": 1.3097922205924988, "sampling/importance_sampling_ratio/mean": 0.9999755859375, "sampling/importance_sampling_ratio/min": 0.7744962632656097, "sampling/sampling_logp_difference/max": 0.27636181712150576, "sampling/sampling_logp_difference/mean": 0.022780642844736576, "step": 4760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.8, "completions/max_terminated_length": 315.8, "completions/mean_length": 229.075, "completions/mean_terminated_length": 229.075, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.158103872090578, "epoch": 0.4385400386135883, "frac_reward_zero_std": 0.3, "grad_norm": 4.777389049530029, "kl": 0.04606992704793811, "learning_rate": 4.874719810259209e-06, "loss": 0.0024, "num_tokens": 20692478.0, "reward": 0.3, "reward_std": 0.3722868412733078, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.37228684723377226, "sampling/importance_sampling_ratio/max": 1.2318024516105652, "sampling/importance_sampling_ratio/mean": 1.0001806914806366, "sampling/importance_sampling_ratio/min": 0.7900952875614167, "sampling/sampling_logp_difference/max": 0.2426864504814148, "sampling/sampling_logp_difference/mean": 0.021408633701503275, "step": 4770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.5, "completions/max_terminated_length": 314.5, "completions/mean_length": 203.825, "completions/mean_terminated_length": 203.825, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 1.1198556058108806, "epoch": 0.4394594097637216, "frac_reward_zero_std": 0.1, "grad_norm": 0.2967608571052551, "kl": 0.05348022039979696, "learning_rate": 4.873926603578225e-06, "loss": 0.0025, "num_tokens": 20736312.0, "reward": -0.25, "reward_std": 0.490106862783432, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.490106874704361, "sampling/importance_sampling_ratio/max": 1.2574563026428223, "sampling/importance_sampling_ratio/mean": 0.999538105726242, "sampling/importance_sampling_ratio/min": 0.7676050066947937, "sampling/sampling_logp_difference/max": 0.2833777368068695, "sampling/sampling_logp_difference/mean": 0.02239927966147661, "step": 4780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.4, "completions/max_terminated_length": 314.4, "completions/mean_length": 220.4875, "completions/mean_terminated_length": 220.4875, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.0699482671916485, "epoch": 0.4403787809138549, "frac_reward_zero_std": 0.1, "grad_norm": 6.024379253387451, "kl": 0.034763780643697825, "learning_rate": 4.873130958684325e-06, "loss": 0.0015, "num_tokens": 20776679.0, "reward": -0.01875, "reward_std": 0.4183605372905731, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.4183605372905731, "sampling/importance_sampling_ratio/max": 1.3236806988716125, "sampling/importance_sampling_ratio/mean": 1.0003363728523254, "sampling/importance_sampling_ratio/min": 0.7648263692855835, "sampling/sampling_logp_difference/max": 0.2967477381229401, "sampling/sampling_logp_difference/mean": 0.021180013380944728, "step": 4790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.2, "completions/max_terminated_length": 289.2, "completions/mean_length": 216.15, "completions/mean_terminated_length": 216.15, "completions/min_length": 143.4, "completions/min_terminated_length": 143.4, "entropy": 1.0816289708018303, "epoch": 0.44129815206398826, "frac_reward_zero_std": 0.2, "grad_norm": 7.013596534729004, "kl": 0.0373613010160625, "learning_rate": 4.872332876394701e-06, "loss": 0.0015, "num_tokens": 20819579.0, "reward": -0.375, "reward_std": 0.38671192824840545, "rewards/reward_fn/mean": -0.375, "rewards/reward_fn/std": 0.38671195805072783, "sampling/importance_sampling_ratio/max": 1.2334882140159606, "sampling/importance_sampling_ratio/mean": 0.9999730110168457, "sampling/importance_sampling_ratio/min": 0.8005930781364441, "sampling/sampling_logp_difference/max": 0.23208605647087097, "sampling/sampling_logp_difference/mean": 0.02125703003257513, "step": 4800 }, { "epoch": 0.44129815206398826, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 296.08202443280976, "eval_completions/max_terminated_length": 295.1413612565445, "eval_completions/mean_length": 209.43041012216406, "eval_completions/mean_terminated_length": 209.15108462029102, "eval_completions/min_length": 135.54101221640488, "eval_completions/min_terminated_length": 135.54101221640488, "eval_entropy": 1.1251758550786224, "eval_frac_reward_zero_std": 0.2530541012216405, "eval_kl": 0.036473345590095456, "eval_loss": 0.0018871371867135167, "eval_num_tokens": 20819579.0, "eval_reward": 0.025632635253054103, "eval_reward_std": 0.40977246084138363, "eval_rewards/reward_fn/mean": 0.025632635253054103, "eval_rewards/reward_fn/std": 0.4097724723878331, "eval_runtime": 1730.3128, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2654379200560884, "eval_sampling/importance_sampling_ratio/mean": 0.9999881724828616, "eval_sampling/importance_sampling_ratio/min": 0.7834413237804726, "eval_sampling/sampling_logp_difference/max": 0.2702022853322054, "eval_sampling/sampling_logp_difference/mean": 0.021100410039213213, "eval_steps_per_second": 0.042, "step": 4800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 191.3875, "completions/mean_terminated_length": 191.3875, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.1443199023604393, "epoch": 0.44221752321412156, "frac_reward_zero_std": 0.1, "grad_norm": 5.949553966522217, "kl": 0.03462678255746141, "learning_rate": 4.871532357529048e-06, "loss": 0.0016, "num_tokens": 20860546.0, "reward": 0.375, "reward_std": 0.5418508559465408, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.5418508738279343, "sampling/importance_sampling_ratio/max": 1.298871600627899, "sampling/importance_sampling_ratio/mean": 1.0003972351551056, "sampling/importance_sampling_ratio/min": 0.7718465030193329, "sampling/sampling_logp_difference/max": 0.3070435106754303, "sampling/sampling_logp_difference/mean": 0.022321824356913565, "step": 4810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.7, "completions/max_terminated_length": 316.1, "completions/mean_length": 224.275, "completions/mean_terminated_length": 221.37142944335938, "completions/min_length": 152.9, "completions/min_terminated_length": 152.9, "entropy": 1.115078853815794, "epoch": 0.44313689436425485, "frac_reward_zero_std": 0.1, "grad_norm": 5.023745536804199, "kl": 0.03538187982048839, "learning_rate": 4.870729402909561e-06, "loss": 0.0014, "num_tokens": 20904856.0, "reward": -0.15625, "reward_std": 0.454889839887619, "rewards/reward_fn/mean": -0.15625, "rewards/reward_fn/std": 0.45488986372947693, "sampling/importance_sampling_ratio/max": 1.2468860387802123, "sampling/importance_sampling_ratio/mean": 1.0005113661289216, "sampling/importance_sampling_ratio/min": 0.7849171757698059, "sampling/sampling_logp_difference/max": 0.2617574095726013, "sampling/sampling_logp_difference/mean": 0.021430672705173494, "step": 4820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 353.8, "completions/max_terminated_length": 339.5, "completions/mean_length": 227.6875, "completions/mean_terminated_length": 224.91607360839845, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.1646183796226979, "epoch": 0.44405626551438815, "frac_reward_zero_std": 0.1, "grad_norm": 5.444431304931641, "kl": 0.04444430838339031, "learning_rate": 4.869924013360941e-06, "loss": 0.0026, "num_tokens": 20949615.0, "reward": -0.10625, "reward_std": 0.47085602283477784, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.4708560347557068, "sampling/importance_sampling_ratio/max": 1.236397647857666, "sampling/importance_sampling_ratio/mean": 0.9997000753879547, "sampling/importance_sampling_ratio/min": 0.7919482290744781, "sampling/sampling_logp_difference/max": 0.2479753702878952, "sampling/sampling_logp_difference/mean": 0.021419833600521087, "step": 4830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.2, "completions/max_terminated_length": 261.2, "completions/mean_length": 188.3375, "completions/mean_terminated_length": 188.3375, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.1247911766171454, "epoch": 0.44497563666452145, "frac_reward_zero_std": 0.4, "grad_norm": 6.723733901977539, "kl": 0.03544203532510437, "learning_rate": 4.869116189710386e-06, "loss": 0.0018, "num_tokens": 20995274.0, "reward": -0.125, "reward_std": 0.3544388711452484, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3544388830661774, "sampling/importance_sampling_ratio/max": 1.2171321392059327, "sampling/importance_sampling_ratio/mean": 0.9997370958328247, "sampling/importance_sampling_ratio/min": 0.7929406702518463, "sampling/sampling_logp_difference/max": 0.23929535150527953, "sampling/sampling_logp_difference/mean": 0.021614062413573266, "step": 4840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.8, "completions/max_terminated_length": 304.8, "completions/mean_length": 202.425, "completions/mean_terminated_length": 202.425, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0920110821723938, "epoch": 0.4458950078146548, "frac_reward_zero_std": 0.2, "grad_norm": 7.844337463378906, "kl": 0.04696904121665284, "learning_rate": 4.868305932787596e-06, "loss": 0.0026, "num_tokens": 21039276.0, "reward": 0.0375, "reward_std": 0.3739841729402542, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.3739841729402542, "sampling/importance_sampling_ratio/max": 1.3499962449073792, "sampling/importance_sampling_ratio/mean": 0.9996867001056671, "sampling/importance_sampling_ratio/min": 0.7630034983158112, "sampling/sampling_logp_difference/max": 0.34782400727272034, "sampling/sampling_logp_difference/mean": 0.02176199834793806, "step": 4850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 230.9625, "completions/mean_terminated_length": 230.9625, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.1516052424907683, "epoch": 0.4468143789647881, "frac_reward_zero_std": 0.2, "grad_norm": 4.6868438720703125, "kl": 0.050775047903880474, "learning_rate": 4.8674932434247715e-06, "loss": 0.0023, "num_tokens": 21088361.0, "reward": -0.23125, "reward_std": 0.43632136285305023, "rewards/reward_fn/mean": -0.23125, "rewards/reward_fn/std": 0.4363213688135147, "sampling/importance_sampling_ratio/max": 1.3010708332061767, "sampling/importance_sampling_ratio/mean": 0.9998128712177277, "sampling/importance_sampling_ratio/min": 0.7771371245384217, "sampling/sampling_logp_difference/max": 0.2982410728931427, "sampling/sampling_logp_difference/mean": 0.02164751309901476, "step": 4860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.5, "completions/max_terminated_length": 256.5, "completions/mean_length": 188.1125, "completions/mean_terminated_length": 188.1125, "completions/min_length": 118.1, "completions/min_terminated_length": 118.1, "entropy": 1.1325742185115815, "epoch": 0.4477337501149214, "frac_reward_zero_std": 0.2, "grad_norm": 5.810248374938965, "kl": 0.04936469867825508, "learning_rate": 4.866678122456607e-06, "loss": 0.0028, "num_tokens": 21126426.0, "reward": 0.025, "reward_std": 0.39796746969223024, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3979674756526947, "sampling/importance_sampling_ratio/max": 1.302277398109436, "sampling/importance_sampling_ratio/mean": 1.0005680859088897, "sampling/importance_sampling_ratio/min": 0.7990141689777375, "sampling/sampling_logp_difference/max": 0.26777092218399046, "sampling/sampling_logp_difference/mean": 0.02273658588528633, "step": 4870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.4, "completions/max_terminated_length": 294.4, "completions/mean_length": 203.8625, "completions/mean_terminated_length": 203.8625, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.1043013699352742, "epoch": 0.4486531212650547, "frac_reward_zero_std": 0.2, "grad_norm": 6.195316314697266, "kl": 0.054411167721264066, "learning_rate": 4.865860570720298e-06, "loss": 0.0026, "num_tokens": 21169183.0, "reward": 0.15625, "reward_std": 0.4739816963672638, "rewards/reward_fn/mean": 0.15625, "rewards/reward_fn/std": 0.4739817142486572, "sampling/importance_sampling_ratio/max": 1.3177698731422425, "sampling/importance_sampling_ratio/mean": 1.0009174585342406, "sampling/importance_sampling_ratio/min": 0.7824495315551758, "sampling/sampling_logp_difference/max": 0.3104393482208252, "sampling/sampling_logp_difference/mean": 0.021653312258422375, "step": 4880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.5, "completions/max_terminated_length": 279.5, "completions/mean_length": 197.5125, "completions/mean_terminated_length": 197.5125, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.1416750863194465, "epoch": 0.449572492415188, "frac_reward_zero_std": 0.3, "grad_norm": 6.275676250457764, "kl": 0.06414212052477523, "learning_rate": 4.865040589055537e-06, "loss": 0.0035, "num_tokens": 21210336.0, "reward": -0.0375, "reward_std": 0.3291133403778076, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.3291133463382721, "sampling/importance_sampling_ratio/max": 1.251700758934021, "sampling/importance_sampling_ratio/mean": 1.0004221558570863, "sampling/importance_sampling_ratio/min": 0.7801296293735505, "sampling/sampling_logp_difference/max": 0.2666925609111786, "sampling/sampling_logp_difference/mean": 0.02234300263226032, "step": 4890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.6, "completions/max_terminated_length": 282.6, "completions/mean_length": 206.125, "completions/mean_terminated_length": 206.125, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 0.9738481394946575, "epoch": 0.45049186356532134, "frac_reward_zero_std": 0.3, "grad_norm": 4.883378505706787, "kl": 0.06306481685605832, "learning_rate": 4.86421817830451e-06, "loss": 0.0032, "num_tokens": 21251994.0, "reward": 0.125, "reward_std": 0.37551296651363375, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.37551297843456266, "sampling/importance_sampling_ratio/max": 1.2437256813049316, "sampling/importance_sampling_ratio/mean": 1.0002299964427948, "sampling/importance_sampling_ratio/min": 0.7948164463043212, "sampling/sampling_logp_difference/max": 0.24511125087738037, "sampling/sampling_logp_difference/mean": 0.019641385413706303, "step": 4900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.8, "completions/max_terminated_length": 249.8, "completions/mean_length": 185.4875, "completions/mean_terminated_length": 185.4875, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.0819780327379704, "epoch": 0.45141123471545463, "frac_reward_zero_std": 0.0, "grad_norm": 7.20682430267334, "kl": 0.050309620436746624, "learning_rate": 4.863393339311897e-06, "loss": 0.0024, "num_tokens": 21292905.0, "reward": 0.1375, "reward_std": 0.5635587096214294, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.5635587275028229, "sampling/importance_sampling_ratio/max": 1.276726531982422, "sampling/importance_sampling_ratio/mean": 0.9999016582965851, "sampling/importance_sampling_ratio/min": 0.7862579941749572, "sampling/sampling_logp_difference/max": 0.27232807874679565, "sampling/sampling_logp_difference/mean": 0.020635491982102395, "step": 4910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.0, "completions/max_terminated_length": 313.0, "completions/mean_length": 226.3375, "completions/mean_terminated_length": 226.3375, "completions/min_length": 156.6, "completions/min_terminated_length": 156.6, "entropy": 1.1471611127257346, "epoch": 0.45233060586558793, "frac_reward_zero_std": 0.3, "grad_norm": 5.578601837158203, "kl": 0.041793910751584916, "learning_rate": 4.8625660729248765e-06, "loss": 0.0029, "num_tokens": 21338524.0, "reward": 0.025, "reward_std": 0.33258684873580935, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3325868546962738, "sampling/importance_sampling_ratio/max": 1.2573898077011108, "sampling/importance_sampling_ratio/mean": 0.9996989667415619, "sampling/importance_sampling_ratio/min": 0.7774382889270782, "sampling/sampling_logp_difference/max": 0.28128833770751954, "sampling/sampling_logp_difference/mean": 0.021383467689156534, "step": 4920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.3, "completions/max_terminated_length": 337.3, "completions/mean_length": 225.2625, "completions/mean_terminated_length": 225.2625, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0893308449536563, "epoch": 0.45324997701572123, "frac_reward_zero_std": 0.3, "grad_norm": 4.952234745025635, "kl": 0.0390682342054788, "learning_rate": 4.861736379993116e-06, "loss": 0.0015, "num_tokens": 21386161.0, "reward": 0.05, "reward_std": 0.3898056745529175, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3898056924343109, "sampling/importance_sampling_ratio/max": 1.303667378425598, "sampling/importance_sampling_ratio/mean": 0.9997180998325348, "sampling/importance_sampling_ratio/min": 0.7679412007331848, "sampling/sampling_logp_difference/max": 0.29677664041519164, "sampling/sampling_logp_difference/mean": 0.02083776155486703, "step": 4930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.3, "completions/max_terminated_length": 302.3, "completions/mean_length": 203.2375, "completions/mean_terminated_length": 203.2375, "completions/min_length": 104.6, "completions/min_terminated_length": 104.6, "entropy": 1.118368787318468, "epoch": 0.4541693481658546, "frac_reward_zero_std": 0.3, "grad_norm": 7.013526439666748, "kl": 0.03668456432060339, "learning_rate": 4.8609042613687766e-06, "loss": 0.0019, "num_tokens": 21428508.0, "reward": -0.125, "reward_std": 0.3771726757287979, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3771726995706558, "sampling/importance_sampling_ratio/max": 1.3006525754928588, "sampling/importance_sampling_ratio/mean": 0.9996533751487732, "sampling/importance_sampling_ratio/min": 0.8079426527023316, "sampling/sampling_logp_difference/max": 0.26807049512863157, "sampling/sampling_logp_difference/mean": 0.021350833773612975, "step": 4940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.3, "completions/max_terminated_length": 298.3, "completions/mean_length": 200.075, "completions/mean_terminated_length": 200.075, "completions/min_length": 127.7, "completions/min_terminated_length": 127.7, "entropy": 1.011567797511816, "epoch": 0.4550887193159879, "frac_reward_zero_std": 0.3, "grad_norm": 6.646734237670898, "kl": 0.033018376323161645, "learning_rate": 4.860069717906511e-06, "loss": 0.0016, "num_tokens": 21465562.0, "reward": 0.1625, "reward_std": 0.2966861009597778, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.29668611884117124, "sampling/importance_sampling_ratio/max": 1.2530601263046264, "sampling/importance_sampling_ratio/mean": 0.9997290253639222, "sampling/importance_sampling_ratio/min": 0.7835807919502258, "sampling/sampling_logp_difference/max": 0.2518495798110962, "sampling/sampling_logp_difference/mean": 0.02033227076753974, "step": 4950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.9, "completions/max_terminated_length": 288.9, "completions/mean_length": 193.325, "completions/mean_terminated_length": 193.325, "completions/min_length": 110.4, "completions/min_terminated_length": 110.4, "entropy": 1.1031865365803242, "epoch": 0.4560080904661212, "frac_reward_zero_std": 0.4, "grad_norm": 0.2061709761619568, "kl": 0.03829234721488319, "learning_rate": 4.859232750463463e-06, "loss": 0.0022, "num_tokens": 21505996.0, "reward": -0.4125, "reward_std": 0.30762961506843567, "rewards/reward_fn/mean": -0.4125, "rewards/reward_fn/std": 0.3076296210289001, "sampling/importance_sampling_ratio/max": 1.2420978546142578, "sampling/importance_sampling_ratio/mean": 0.9997165083885193, "sampling/importance_sampling_ratio/min": 0.771349960565567, "sampling/sampling_logp_difference/max": 0.26861617565155027, "sampling/sampling_logp_difference/mean": 0.021324186585843563, "step": 4960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 200.5625, "completions/mean_terminated_length": 200.5625, "completions/min_length": 106.5, "completions/min_terminated_length": 106.5, "entropy": 1.1076211370527744, "epoch": 0.45692746161625447, "frac_reward_zero_std": 0.0, "grad_norm": 6.930084705352783, "kl": 0.051061311236117034, "learning_rate": 4.858393359899265e-06, "loss": 0.0024, "num_tokens": 21548369.0, "reward": -0.1375, "reward_std": 0.5144637852907181, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.5144638031721115, "sampling/importance_sampling_ratio/max": 1.2712450623512268, "sampling/importance_sampling_ratio/mean": 0.9993958890438079, "sampling/importance_sampling_ratio/min": 0.7928924083709716, "sampling/sampling_logp_difference/max": 0.2608424425125122, "sampling/sampling_logp_difference/mean": 0.021413145028054713, "step": 4970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.1, "completions/max_terminated_length": 306.1, "completions/mean_length": 204.1, "completions/mean_terminated_length": 204.1, "completions/min_length": 117.1, "completions/min_terminated_length": 117.1, "entropy": 1.0390336617827416, "epoch": 0.45784683276638777, "frac_reward_zero_std": 0.4, "grad_norm": 0.12613804638385773, "kl": 0.04744800207554363, "learning_rate": 4.857551547076037e-06, "loss": 0.0023, "num_tokens": 21590177.0, "reward": -0.1, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.2713823318481445, "sampling/importance_sampling_ratio/mean": 1.0003105998039246, "sampling/importance_sampling_ratio/min": 0.785928875207901, "sampling/sampling_logp_difference/max": 0.2674295246601105, "sampling/sampling_logp_difference/mean": 0.020596507750451565, "step": 4980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.5, "completions/max_terminated_length": 313.5, "completions/mean_length": 223.2125, "completions/mean_terminated_length": 223.2125, "completions/min_length": 118.8, "completions/min_terminated_length": 118.8, "entropy": 1.081412636488676, "epoch": 0.4587662039165211, "frac_reward_zero_std": 0.4, "grad_norm": 5.8120036125183105, "kl": 0.044370586139848456, "learning_rate": 4.85670731285839e-06, "loss": 0.0017, "num_tokens": 21636170.0, "reward": -0.075, "reward_std": 0.3574239045381546, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3574239045381546, "sampling/importance_sampling_ratio/max": 1.2946985483169555, "sampling/importance_sampling_ratio/mean": 1.0002689242362977, "sampling/importance_sampling_ratio/min": 0.7857405126094819, "sampling/sampling_logp_difference/max": 0.29253565073013305, "sampling/sampling_logp_difference/mean": 0.020496607571840287, "step": 4990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.3, "completions/max_terminated_length": 294.3, "completions/mean_length": 208.95, "completions/mean_terminated_length": 208.95, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1115488819777966, "epoch": 0.4596855750666544, "frac_reward_zero_std": 0.1, "grad_norm": 0.16221123933792114, "kl": 0.05902964059496298, "learning_rate": 4.855860658113419e-06, "loss": 0.0034, "num_tokens": 21678390.0, "reward": -0.025, "reward_std": 0.4237645983695984, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4237646222114563, "sampling/importance_sampling_ratio/max": 1.3044022798538208, "sampling/importance_sampling_ratio/mean": 0.9994701564311981, "sampling/importance_sampling_ratio/min": 0.7895667552947998, "sampling/sampling_logp_difference/max": 0.28730525970458987, "sampling/sampling_logp_difference/mean": 0.02126925028860569, "step": 5000 }, { "epoch": 0.4596855750666544, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 296.6020942408377, "eval_completions/max_terminated_length": 295.61605584642234, "eval_completions/mean_length": 204.22687609075044, "eval_completions/mean_terminated_length": 203.81525819255836, "eval_completions/min_length": 123.58638743455498, "eval_completions/min_terminated_length": 123.58638743455498, "eval_entropy": 1.0980207964283841, "eval_frac_reward_zero_std": 0.31239092495637, "eval_kl": 0.05347816798556433, "eval_loss": 0.0028688579332083464, "eval_num_tokens": 21678390.0, "eval_reward": 0.05606457242582897, "eval_reward_std": 0.33468144091844143, "eval_rewards/reward_fn/mean": 0.05606457242582897, "eval_rewards/reward_fn/std": 0.33468145017640544, "eval_runtime": 1729.3611, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2829591175232467, "eval_sampling/importance_sampling_ratio/mean": 1.0000462963735037, "eval_sampling/importance_sampling_ratio/min": 0.7834602964174061, "eval_sampling/sampling_logp_difference/max": 0.2806670146150739, "eval_sampling/sampling_logp_difference/mean": 0.02100302376028599, "eval_steps_per_second": 0.042, "step": 5000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 215.05, "completions/mean_terminated_length": 215.05, "completions/min_length": 153.1, "completions/min_terminated_length": 153.1, "entropy": 1.0642635196447372, "epoch": 0.4606049462167877, "frac_reward_zero_std": 0.5, "grad_norm": 0.21094852685928345, "kl": 0.054026177397463473, "learning_rate": 4.855011583710707e-06, "loss": 0.0029, "num_tokens": 21721586.0, "reward": -0.00625, "reward_std": 0.25884227454662323, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.25884227454662323, "sampling/importance_sampling_ratio/max": 1.2514773607254028, "sampling/importance_sampling_ratio/mean": 1.0000546872615814, "sampling/importance_sampling_ratio/min": 0.8022564351558685, "sampling/sampling_logp_difference/max": 0.2431874692440033, "sampling/sampling_logp_difference/mean": 0.021205791831016542, "step": 5010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.1, "completions/max_terminated_length": 291.1, "completions/mean_length": 209.7625, "completions/mean_terminated_length": 209.7625, "completions/min_length": 135.4, "completions/min_terminated_length": 135.4, "entropy": 1.1110947892069816, "epoch": 0.461524317366921, "frac_reward_zero_std": 0.2, "grad_norm": 5.1219000816345215, "kl": 0.055259722052142024, "learning_rate": 4.854160090522321e-06, "loss": 0.0026, "num_tokens": 21762631.0, "reward": 0.05, "reward_std": 0.3429849535226822, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.34298495948314667, "sampling/importance_sampling_ratio/max": 1.2838879108428956, "sampling/importance_sampling_ratio/mean": 0.9997478723526001, "sampling/importance_sampling_ratio/min": 0.7638672709465026, "sampling/sampling_logp_difference/max": 0.30507512092590333, "sampling/sampling_logp_difference/mean": 0.02249348033219576, "step": 5020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.9, "completions/max_terminated_length": 263.9, "completions/mean_length": 186.8375, "completions/mean_terminated_length": 186.8375, "completions/min_length": 130.3, "completions/min_terminated_length": 130.3, "entropy": 1.1042222172021865, "epoch": 0.4624436885170543, "frac_reward_zero_std": 0.3, "grad_norm": 8.015363693237305, "kl": 0.051652857108274476, "learning_rate": 4.853306179422814e-06, "loss": 0.0029, "num_tokens": 21804634.0, "reward": -0.1125, "reward_std": 0.3538378983736038, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.35383791029453276, "sampling/importance_sampling_ratio/max": 1.2638363718986512, "sampling/importance_sampling_ratio/mean": 0.9997810602188111, "sampling/importance_sampling_ratio/min": 0.791809868812561, "sampling/sampling_logp_difference/max": 0.2542734980583191, "sampling/sampling_logp_difference/mean": 0.02173410803079605, "step": 5030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.2, "completions/max_terminated_length": 273.2, "completions/mean_length": 180.925, "completions/mean_terminated_length": 180.925, "completions/min_length": 109.1, "completions/min_terminated_length": 109.1, "entropy": 1.0299954634159803, "epoch": 0.46336305966718766, "frac_reward_zero_std": 0.2, "grad_norm": 6.978346347808838, "kl": 0.06670718252426014, "learning_rate": 4.852449851289219e-06, "loss": 0.0034, "num_tokens": 21847108.0, "reward": 0.1375, "reward_std": 0.3946571469306946, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.394657164812088, "sampling/importance_sampling_ratio/max": 1.288669967651367, "sampling/importance_sampling_ratio/mean": 0.99981968998909, "sampling/importance_sampling_ratio/min": 0.7698420405387878, "sampling/sampling_logp_difference/max": 0.3021678805351257, "sampling/sampling_logp_difference/mean": 0.02039196193218231, "step": 5040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.0, "completions/max_terminated_length": 278.0, "completions/mean_length": 202.65, "completions/mean_terminated_length": 202.65, "completions/min_length": 115.6, "completions/min_terminated_length": 115.6, "entropy": 1.121798100322485, "epoch": 0.46428243081732096, "frac_reward_zero_std": 0.2, "grad_norm": 5.911308765411377, "kl": 0.05855126996757463, "learning_rate": 4.851591107001055e-06, "loss": 0.0027, "num_tokens": 21889848.0, "reward": -0.05, "reward_std": 0.40020377933979034, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.40020379722118377, "sampling/importance_sampling_ratio/max": 1.2433643817901612, "sampling/importance_sampling_ratio/mean": 0.9993038415908814, "sampling/importance_sampling_ratio/min": 0.7627947509288788, "sampling/sampling_logp_difference/max": 0.2757129192352295, "sampling/sampling_logp_difference/mean": 0.022194026969373226, "step": 5050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.8, "completions/max_terminated_length": 264.8, "completions/mean_length": 190.275, "completions/mean_terminated_length": 190.275, "completions/min_length": 94.1, "completions/min_terminated_length": 94.1, "entropy": 0.9969781950116158, "epoch": 0.46520180196745425, "frac_reward_zero_std": 0.3, "grad_norm": 7.016033172607422, "kl": 0.04298663109075278, "learning_rate": 4.850729947440321e-06, "loss": 0.0021, "num_tokens": 21930542.0, "reward": 0.25, "reward_std": 0.3147830098867416, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.31478302776813505, "sampling/importance_sampling_ratio/max": 1.313794457912445, "sampling/importance_sampling_ratio/mean": 0.9997335433959961, "sampling/importance_sampling_ratio/min": 0.7648922443389893, "sampling/sampling_logp_difference/max": 0.31616237163543703, "sampling/sampling_logp_difference/mean": 0.02098549772053957, "step": 5060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 332.4, "completions/max_terminated_length": 310.8, "completions/mean_length": 235.4625, "completions/mean_terminated_length": 229.71607360839843, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 1.0497488506138324, "epoch": 0.46612117311758755, "frac_reward_zero_std": 0.2, "grad_norm": 0.1476282775402069, "kl": 0.04426348961424083, "learning_rate": 4.8498663734914975e-06, "loss": 0.0022, "num_tokens": 21974763.0, "reward": 0.0, "reward_std": 0.41783192455768586, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.41783193647861483, "sampling/importance_sampling_ratio/max": 1.2631876111030578, "sampling/importance_sampling_ratio/mean": 0.9993622124195098, "sampling/importance_sampling_ratio/min": 0.783275592327118, "sampling/sampling_logp_difference/max": 0.2601447343826294, "sampling/sampling_logp_difference/mean": 0.021352509409189223, "step": 5070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 198.7, "completions/mean_terminated_length": 198.7, "completions/min_length": 119.7, "completions/min_terminated_length": 119.7, "entropy": 1.1181830674409867, "epoch": 0.4670405442677209, "frac_reward_zero_std": 0.3, "grad_norm": 0.18036657571792603, "kl": 0.054512911546044054, "learning_rate": 4.8490003860415436e-06, "loss": 0.0026, "num_tokens": 22015267.0, "reward": 0.0125, "reward_std": 0.3348544090986252, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.3348544090986252, "sampling/importance_sampling_ratio/max": 1.2961976051330566, "sampling/importance_sampling_ratio/mean": 1.000363713502884, "sampling/importance_sampling_ratio/min": 0.7825570344924927, "sampling/sampling_logp_difference/max": 0.2936065196990967, "sampling/sampling_logp_difference/mean": 0.022312752902507782, "step": 5080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.3, "completions/max_terminated_length": 282.3, "completions/mean_length": 200.85, "completions/mean_terminated_length": 200.85, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.059295652806759, "epoch": 0.4679599154178542, "frac_reward_zero_std": 0.3, "grad_norm": 6.634043216705322, "kl": 0.04717031229520217, "learning_rate": 4.848131985979897e-06, "loss": 0.0023, "num_tokens": 22059839.0, "reward": 0.2, "reward_std": 0.3302692323923111, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3302692502737045, "sampling/importance_sampling_ratio/max": 1.3089182257652283, "sampling/importance_sampling_ratio/mean": 1.0001337468624114, "sampling/importance_sampling_ratio/min": 0.7484601140022278, "sampling/sampling_logp_difference/max": 0.3212741196155548, "sampling/sampling_logp_difference/mean": 0.02130128536373377, "step": 5090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.0, "completions/max_terminated_length": 266.0, "completions/mean_length": 182.15, "completions/mean_terminated_length": 182.15, "completions/min_length": 112.6, "completions/min_terminated_length": 112.6, "entropy": 1.0159906573593616, "epoch": 0.4688792865679875, "frac_reward_zero_std": 0.4, "grad_norm": 6.790402889251709, "kl": 0.03802815774688497, "learning_rate": 4.847261174198475e-06, "loss": 0.0021, "num_tokens": 22104979.0, "reward": 0.2375, "reward_std": 0.26302809417247774, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.26302810609340666, "sampling/importance_sampling_ratio/max": 1.2474716305732727, "sampling/importance_sampling_ratio/mean": 0.9999119460582733, "sampling/importance_sampling_ratio/min": 0.8018891036510467, "sampling/sampling_logp_difference/max": 0.2434978485107422, "sampling/sampling_logp_difference/mean": 0.020131850615143776, "step": 5100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.4, "completions/max_terminated_length": 263.4, "completions/mean_length": 185.8375, "completions/mean_terminated_length": 185.8375, "completions/min_length": 129.7, "completions/min_terminated_length": 129.7, "entropy": 1.0033727928996086, "epoch": 0.4697986577181208, "frac_reward_zero_std": 0.1, "grad_norm": 5.781394958496094, "kl": 0.04538027112139389, "learning_rate": 4.846387951591672e-06, "loss": 0.0016, "num_tokens": 22146982.0, "reward": 0.0, "reward_std": 0.42685834467411043, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.42685835659503935, "sampling/importance_sampling_ratio/max": 1.2272173047065735, "sampling/importance_sampling_ratio/mean": 1.0000843107700348, "sampling/importance_sampling_ratio/min": 0.8051585793495178, "sampling/sampling_logp_difference/max": 0.2425686836242676, "sampling/sampling_logp_difference/mean": 0.020243636891245843, "step": 5110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.0, "completions/max_terminated_length": 316.0, "completions/mean_length": 208.275, "completions/mean_terminated_length": 208.275, "completions/min_length": 118.0, "completions/min_terminated_length": 118.0, "entropy": 1.0271643541753293, "epoch": 0.4707180288682541, "frac_reward_zero_std": 0.2, "grad_norm": 6.1978535652160645, "kl": 0.04922398622147739, "learning_rate": 4.8455123190563556e-06, "loss": 0.0028, "num_tokens": 22188748.0, "reward": 0.225, "reward_std": 0.37944808304309846, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3794480890035629, "sampling/importance_sampling_ratio/max": 1.2611426711082458, "sampling/importance_sampling_ratio/mean": 1.0002716302871704, "sampling/importance_sampling_ratio/min": 0.732643648982048, "sampling/sampling_logp_difference/max": 0.3451880097389221, "sampling/sampling_logp_difference/mean": 0.02061178386211395, "step": 5120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.3, "completions/max_terminated_length": 283.3, "completions/mean_length": 197.725, "completions/mean_terminated_length": 197.725, "completions/min_length": 122.4, "completions/min_terminated_length": 122.4, "entropy": 1.0985075183212758, "epoch": 0.47163740001838744, "frac_reward_zero_std": 0.1, "grad_norm": 5.584001064300537, "kl": 0.0392339939950034, "learning_rate": 4.844634277491873e-06, "loss": 0.0025, "num_tokens": 22231718.0, "reward": -0.1, "reward_std": 0.3909654438495636, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3909654557704926, "sampling/importance_sampling_ratio/max": 1.3278156399726868, "sampling/importance_sampling_ratio/mean": 1.0000971794128417, "sampling/importance_sampling_ratio/min": 0.7678834438323975, "sampling/sampling_logp_difference/max": 0.3281996488571167, "sampling/sampling_logp_difference/mean": 0.02178528197109699, "step": 5130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.2, "completions/max_terminated_length": 268.2, "completions/mean_length": 181.925, "completions/mean_terminated_length": 181.925, "completions/min_length": 116.9, "completions/min_terminated_length": 116.9, "entropy": 1.101681013405323, "epoch": 0.47255677116852074, "frac_reward_zero_std": 0.3, "grad_norm": 0.21135449409484863, "kl": 0.046971906477119775, "learning_rate": 4.843753827800041e-06, "loss": 0.0023, "num_tokens": 22273152.0, "reward": 0.225, "reward_std": 0.28029437065124513, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.28029437065124513, "sampling/importance_sampling_ratio/max": 1.273150885105133, "sampling/importance_sampling_ratio/mean": 1.000169610977173, "sampling/importance_sampling_ratio/min": 0.78610959649086, "sampling/sampling_logp_difference/max": 0.26658374071121216, "sampling/sampling_logp_difference/mean": 0.021707371808588505, "step": 5140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.9, "completions/max_terminated_length": 289.9, "completions/mean_length": 201.0625, "completions/mean_terminated_length": 201.0625, "completions/min_length": 124.3, "completions/min_terminated_length": 124.3, "entropy": 1.081572512537241, "epoch": 0.47347614231865404, "frac_reward_zero_std": 0.1, "grad_norm": 5.036871433258057, "kl": 0.05442186284344643, "learning_rate": 4.842870970885153e-06, "loss": 0.0025, "num_tokens": 22317069.0, "reward": -0.175, "reward_std": 0.37833244502544405, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3783324629068375, "sampling/importance_sampling_ratio/max": 1.242033851146698, "sampling/importance_sampling_ratio/mean": 0.9998442113399506, "sampling/importance_sampling_ratio/min": 0.7473058223724365, "sampling/sampling_logp_difference/max": 0.2957075357437134, "sampling/sampling_logp_difference/mean": 0.021647975780069828, "step": 5150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.1, "completions/max_terminated_length": 243.1, "completions/mean_length": 175.5125, "completions/mean_terminated_length": 175.5125, "completions/min_length": 116.7, "completions/min_terminated_length": 116.7, "entropy": 1.0082119680941104, "epoch": 0.47439551346878733, "frac_reward_zero_std": 0.2, "grad_norm": 6.256570816040039, "kl": 0.05352669929852709, "learning_rate": 4.841985707653974e-06, "loss": 0.003, "num_tokens": 22355974.0, "reward": 0.25, "reward_std": 0.3863084077835083, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3863084137439728, "sampling/importance_sampling_ratio/max": 1.2449208736419677, "sampling/importance_sampling_ratio/mean": 1.0003450989723206, "sampling/importance_sampling_ratio/min": 0.7393296301364899, "sampling/sampling_logp_difference/max": 0.3140349149703979, "sampling/sampling_logp_difference/mean": 0.02000609301030636, "step": 5160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.8, "completions/max_terminated_length": 308.8, "completions/mean_length": 208.75, "completions/mean_terminated_length": 208.75, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.1051113750785588, "epoch": 0.4753148846189207, "frac_reward_zero_std": 0.3, "grad_norm": 6.527437686920166, "kl": 0.03379192901775241, "learning_rate": 4.84109803901574e-06, "loss": 0.0016, "num_tokens": 22399602.0, "reward": -0.0625, "reward_std": 0.390112441778183, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.39011244773864745, "sampling/importance_sampling_ratio/max": 1.2987023949623109, "sampling/importance_sampling_ratio/mean": 0.9995889306068421, "sampling/importance_sampling_ratio/min": 0.7780885815620422, "sampling/sampling_logp_difference/max": 0.2858575820922852, "sampling/sampling_logp_difference/mean": 0.021906653419137, "step": 5170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 191.5875, "completions/mean_terminated_length": 191.5875, "completions/min_length": 110.5, "completions/min_terminated_length": 110.5, "entropy": 1.1301247969269752, "epoch": 0.476234255769054, "frac_reward_zero_std": 0.2, "grad_norm": 0.14744135737419128, "kl": 0.04660897683352232, "learning_rate": 4.840207965882158e-06, "loss": 0.0028, "num_tokens": 22440129.0, "reward": 0.0, "reward_std": 0.4498067259788513, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.44980673789978026, "sampling/importance_sampling_ratio/max": 1.2783320546150208, "sampling/importance_sampling_ratio/mean": 0.9997207224369049, "sampling/importance_sampling_ratio/min": 0.7876246988773346, "sampling/sampling_logp_difference/max": 0.27246793508529665, "sampling/sampling_logp_difference/mean": 0.022337763011455535, "step": 5180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 217.7, "completions/mean_terminated_length": 217.7, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0966159895062446, "epoch": 0.4771536269191873, "frac_reward_zero_std": 0.2, "grad_norm": 6.527858734130859, "kl": 0.048502184508834034, "learning_rate": 4.839315489167405e-06, "loss": 0.0024, "num_tokens": 22482289.0, "reward": -0.225, "reward_std": 0.4447881907224655, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4447882086038589, "sampling/importance_sampling_ratio/max": 1.2794241547584533, "sampling/importance_sampling_ratio/mean": 0.9999913692474365, "sampling/importance_sampling_ratio/min": 0.7823644757270813, "sampling/sampling_logp_difference/max": 0.2830728769302368, "sampling/sampling_logp_difference/mean": 0.02094872798770666, "step": 5190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 206.6625, "completions/mean_terminated_length": 206.6625, "completions/min_length": 138.8, "completions/min_terminated_length": 138.8, "entropy": 1.0814496617764235, "epoch": 0.4780729980693206, "frac_reward_zero_std": 0.2, "grad_norm": 6.223296165466309, "kl": 0.038792222726624456, "learning_rate": 4.838420609788125e-06, "loss": 0.0019, "num_tokens": 22526494.0, "reward": -0.1, "reward_std": 0.45408297181129453, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4540829837322235, "sampling/importance_sampling_ratio/max": 1.3048575401306153, "sampling/importance_sampling_ratio/mean": 1.0005558133125305, "sampling/importance_sampling_ratio/min": 0.7743797719478607, "sampling/sampling_logp_difference/max": 0.29603824615478513, "sampling/sampling_logp_difference/mean": 0.021037834510207177, "step": 5200 }, { "epoch": 0.4780729980693206, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 306.6020942408377, "eval_completions/max_terminated_length": 304.95462478184993, "eval_completions/mean_length": 215.4962914485166, "eval_completions/mean_terminated_length": 215.05709314263095, "eval_completions/min_length": 140.19197207678883, "eval_completions/min_terminated_length": 140.19197207678883, "eval_entropy": 1.136109718841616, "eval_frac_reward_zero_std": 0.23036649214659685, "eval_kl": 0.03790728024078653, "eval_loss": 0.0019444205099716783, "eval_num_tokens": 22526494.0, "eval_reward": 0.035231239092495634, "eval_reward_std": 0.41636209993462286, "eval_rewards/reward_fn/mean": 0.035231239092495634, "eval_rewards/reward_fn/std": 0.4163621130414034, "eval_runtime": 1782.7889, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.2755815710280787, "eval_sampling/importance_sampling_ratio/mean": 1.0000133960658968, "eval_sampling/importance_sampling_ratio/min": 0.7840934941645901, "eval_sampling/sampling_logp_difference/max": 0.27744009159830435, "eval_sampling/sampling_logp_difference/mean": 0.02109494226713247, "eval_steps_per_second": 0.04, "step": 5200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 200.8375, "completions/mean_terminated_length": 200.8375, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.1420616939663888, "epoch": 0.4789923692194539, "frac_reward_zero_std": 0.2, "grad_norm": 5.956714630126953, "kl": 0.049311512301210315, "learning_rate": 4.8375233286634315e-06, "loss": 0.0027, "num_tokens": 22569065.0, "reward": 0.025, "reward_std": 0.4520098477602005, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.45200985968112944, "sampling/importance_sampling_ratio/max": 1.2439144253730774, "sampling/importance_sampling_ratio/mean": 1.0002358198165893, "sampling/importance_sampling_ratio/min": 0.7658241093158722, "sampling/sampling_logp_difference/max": 0.27454921007156374, "sampling/sampling_logp_difference/mean": 0.021910637430846692, "step": 5210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 237.0, "completions/mean_terminated_length": 237.0, "completions/min_length": 146.3, "completions/min_terminated_length": 146.3, "entropy": 1.2142364338040352, "epoch": 0.4799117403695872, "frac_reward_zero_std": 0.3, "grad_norm": 5.120321273803711, "kl": 0.039680343423970046, "learning_rate": 4.836623646714904e-06, "loss": 0.002, "num_tokens": 22615033.0, "reward": -0.225, "reward_std": 0.3110164314508438, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.3110164374113083, "sampling/importance_sampling_ratio/max": 1.2647359848022461, "sampling/importance_sampling_ratio/mean": 0.9999931931495667, "sampling/importance_sampling_ratio/min": 0.7919558465480805, "sampling/sampling_logp_difference/max": 0.25625980496406553, "sampling/sampling_logp_difference/mean": 0.0225582379847765, "step": 5220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.3, "completions/max_terminated_length": 320.9, "completions/mean_length": 224.5375, "completions/mean_terminated_length": 220.97857208251952, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.1301677040755749, "epoch": 0.4808311115197205, "frac_reward_zero_std": 0.2, "grad_norm": 6.9213457107543945, "kl": 0.0445600493112579, "learning_rate": 4.8357215648665876e-06, "loss": 0.0023, "num_tokens": 22660356.0, "reward": 0.175, "reward_std": 0.5354843109846115, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.5354843288660049, "sampling/importance_sampling_ratio/max": 1.2766311645507813, "sampling/importance_sampling_ratio/mean": 0.9998451352119446, "sampling/importance_sampling_ratio/min": 0.7980450689792633, "sampling/sampling_logp_difference/max": 0.25270800590515136, "sampling/sampling_logp_difference/mean": 0.02100846953690052, "step": 5230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.0, "completions/max_terminated_length": 338.0, "completions/mean_length": 235.3125, "completions/mean_terminated_length": 235.3125, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0732504196465016, "epoch": 0.4817504826698538, "frac_reward_zero_std": 0.1, "grad_norm": 4.815831661224365, "kl": 0.038973855052608995, "learning_rate": 4.834817084044994e-06, "loss": 0.002, "num_tokens": 22703733.0, "reward": 0.225, "reward_std": 0.4262101650238037, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.42621017098426817, "sampling/importance_sampling_ratio/max": 1.3021227717399597, "sampling/importance_sampling_ratio/mean": 1.0006640374660491, "sampling/importance_sampling_ratio/min": 0.7851735472679138, "sampling/sampling_logp_difference/max": 0.2739617943763733, "sampling/sampling_logp_difference/mean": 0.02052662502974272, "step": 5240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.0, "completions/max_terminated_length": 314.0, "completions/mean_length": 204.6, "completions/mean_terminated_length": 204.6, "completions/min_length": 128.3, "completions/min_terminated_length": 128.3, "entropy": 1.0754360709339381, "epoch": 0.4826698538199871, "frac_reward_zero_std": 0.4, "grad_norm": 1.600217580795288, "kl": 0.030810918210772798, "learning_rate": 4.833910205179096e-06, "loss": 0.0015, "num_tokens": 22745581.0, "reward": 0.1, "reward_std": 0.40830574929714203, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.40830577313899996, "sampling/importance_sampling_ratio/max": 1.280623996257782, "sampling/importance_sampling_ratio/mean": 1.0000560104846954, "sampling/importance_sampling_ratio/min": 0.7976553618907929, "sampling/sampling_logp_difference/max": 0.26327252984046934, "sampling/sampling_logp_difference/mean": 0.02123839780688286, "step": 5250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.2, "completions/max_terminated_length": 314.2, "completions/mean_length": 215.4875, "completions/mean_terminated_length": 215.4875, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.126778892427683, "epoch": 0.4835892249701204, "frac_reward_zero_std": 0.0, "grad_norm": 5.7373857498168945, "kl": 0.04488398185931146, "learning_rate": 4.833000929200331e-06, "loss": 0.002, "num_tokens": 22787508.0, "reward": 0.1125, "reward_std": 0.518820071220398, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.5188200891017913, "sampling/importance_sampling_ratio/max": 1.2564231872558593, "sampling/importance_sampling_ratio/mean": 0.9996687948703766, "sampling/importance_sampling_ratio/min": 0.7922789454460144, "sampling/sampling_logp_difference/max": 0.25233088731765746, "sampling/sampling_logp_difference/mean": 0.021650151722133158, "step": 5260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 232.7, "completions/mean_terminated_length": 232.7, "completions/min_length": 163.0, "completions/min_terminated_length": 163.0, "entropy": 1.0743748657405376, "epoch": 0.48450859612025376, "frac_reward_zero_std": 0.1, "grad_norm": 5.6547369956970215, "kl": 0.045266679662745446, "learning_rate": 4.8320892570426e-06, "loss": 0.0019, "num_tokens": 22832148.0, "reward": -0.125, "reward_std": 0.5091026455163956, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.509102675318718, "sampling/importance_sampling_ratio/max": 1.3167165398597718, "sampling/importance_sampling_ratio/mean": 1.0006289899349212, "sampling/importance_sampling_ratio/min": 0.7421549737453461, "sampling/sampling_logp_difference/max": 0.3439781904220581, "sampling/sampling_logp_difference/mean": 0.02123083993792534, "step": 5270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.7, "completions/max_terminated_length": 271.7, "completions/mean_length": 183.125, "completions/mean_terminated_length": 183.125, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 1.0387254439294338, "epoch": 0.48542796727038706, "frac_reward_zero_std": 0.2, "grad_norm": 9.258883476257324, "kl": 0.04214065321721137, "learning_rate": 4.831175189642261e-06, "loss": 0.0021, "num_tokens": 22869646.0, "reward": -0.075, "reward_std": 0.3799549013376236, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.37995490729808806, "sampling/importance_sampling_ratio/max": 1.3570650696754456, "sampling/importance_sampling_ratio/mean": 1.000565952062607, "sampling/importance_sampling_ratio/min": 0.7757083117961884, "sampling/sampling_logp_difference/max": 0.3312030553817749, "sampling/sampling_logp_difference/mean": 0.021324425749480725, "step": 5280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 195.0, "completions/mean_terminated_length": 195.0, "completions/min_length": 126.3, "completions/min_terminated_length": 126.3, "entropy": 1.1406998798251151, "epoch": 0.48634733842052036, "frac_reward_zero_std": 0.3, "grad_norm": 0.14811979234218597, "kl": 0.045686901442240926, "learning_rate": 4.8302587279381365e-06, "loss": 0.0028, "num_tokens": 22908750.0, "reward": -0.0875, "reward_std": 0.4017985433340073, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.40179855525493624, "sampling/importance_sampling_ratio/max": 1.266722023487091, "sampling/importance_sampling_ratio/mean": 1.0002274930477142, "sampling/importance_sampling_ratio/min": 0.7796203076839447, "sampling/sampling_logp_difference/max": 0.2722718119621277, "sampling/sampling_logp_difference/mean": 0.022155637852847576, "step": 5290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 209.7875, "completions/mean_terminated_length": 209.7875, "completions/min_length": 135.1, "completions/min_terminated_length": 135.1, "entropy": 1.0300621554255485, "epoch": 0.48726670957065366, "frac_reward_zero_std": 0.4, "grad_norm": 0.1667054146528244, "kl": 0.04274160105269402, "learning_rate": 4.829339872871505e-06, "loss": 0.0018, "num_tokens": 22951013.0, "reward": 0.0875, "reward_std": 0.29780173897743223, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.29780174493789674, "sampling/importance_sampling_ratio/max": 1.250805640220642, "sampling/importance_sampling_ratio/mean": 0.9995734453201294, "sampling/importance_sampling_ratio/min": 0.7733888864517212, "sampling/sampling_logp_difference/max": 0.27021238803863523, "sampling/sampling_logp_difference/mean": 0.020219090208411217, "step": 5300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.9, "completions/max_terminated_length": 314.2, "completions/mean_length": 217.8375, "completions/mean_terminated_length": 215.01785888671876, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.0944420523941516, "epoch": 0.488186080720787, "frac_reward_zero_std": 0.2, "grad_norm": 5.961233615875244, "kl": 0.05282329951878637, "learning_rate": 4.828418625386105e-06, "loss": 0.0024, "num_tokens": 22993328.0, "reward": -0.06875, "reward_std": 0.49629092812538145, "rewards/reward_fn/mean": -0.06875, "rewards/reward_fn/std": 0.4962909460067749, "sampling/importance_sampling_ratio/max": 1.2744558334350586, "sampling/importance_sampling_ratio/mean": 0.9995882093906403, "sampling/importance_sampling_ratio/min": 0.7371620237827301, "sampling/sampling_logp_difference/max": 0.3139346122741699, "sampling/sampling_logp_difference/mean": 0.02146372217684984, "step": 5310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.0, "completions/max_terminated_length": 303.0, "completions/mean_length": 195.6, "completions/mean_terminated_length": 195.6, "completions/min_length": 118.2, "completions/min_terminated_length": 118.2, "entropy": 1.0200982823967935, "epoch": 0.4891054518709203, "frac_reward_zero_std": 0.2, "grad_norm": 4.7303314208984375, "kl": 0.03553521225694567, "learning_rate": 4.8274949864281304e-06, "loss": 0.0015, "num_tokens": 23034368.0, "reward": 0.175, "reward_std": 0.3961169719696045, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.39611698389053346, "sampling/importance_sampling_ratio/max": 1.269530713558197, "sampling/importance_sampling_ratio/mean": 1.0001214146614075, "sampling/importance_sampling_ratio/min": 0.7562114894390106, "sampling/sampling_logp_difference/max": 0.3183754563331604, "sampling/sampling_logp_difference/mean": 0.0205043263733387, "step": 5320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.3, "completions/max_terminated_length": 295.3, "completions/mean_length": 208.6125, "completions/mean_terminated_length": 208.6125, "completions/min_length": 144.2, "completions/min_terminated_length": 144.2, "entropy": 1.0772612024098636, "epoch": 0.4900248230210536, "frac_reward_zero_std": 0.3, "grad_norm": 6.967926979064941, "kl": 0.04195050619309768, "learning_rate": 4.826568956946233e-06, "loss": 0.0021, "num_tokens": 23077857.0, "reward": -0.1, "reward_std": 0.3446744382381439, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3446744501590729, "sampling/importance_sampling_ratio/max": 1.2724981427192688, "sampling/importance_sampling_ratio/mean": 1.0001750588417053, "sampling/importance_sampling_ratio/min": 0.794041758775711, "sampling/sampling_logp_difference/max": 0.2588728666305542, "sampling/sampling_logp_difference/mean": 0.021231592260301115, "step": 5330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.7, "completions/max_terminated_length": 284.7, "completions/mean_length": 209.325, "completions/mean_terminated_length": 209.325, "completions/min_length": 142.4, "completions/min_terminated_length": 142.4, "entropy": 1.0945632435381412, "epoch": 0.4909441941711869, "frac_reward_zero_std": 0.2, "grad_norm": 9.656747817993164, "kl": 0.04670930468128063, "learning_rate": 4.825640537891521e-06, "loss": 0.0025, "num_tokens": 23121091.0, "reward": 0.1375, "reward_std": 0.44562808573246004, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.4456281214952469, "sampling/importance_sampling_ratio/max": 1.348485779762268, "sampling/importance_sampling_ratio/mean": 1.0002808809280395, "sampling/importance_sampling_ratio/min": 0.7791234493255615, "sampling/sampling_logp_difference/max": 0.3120794057846069, "sampling/sampling_logp_difference/mean": 0.022376133129000664, "step": 5340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.5, "completions/max_terminated_length": 284.5, "completions/mean_length": 192.9625, "completions/mean_terminated_length": 192.9625, "completions/min_length": 128.1, "completions/min_terminated_length": 128.1, "entropy": 1.1277014277875423, "epoch": 0.4918635653213202, "frac_reward_zero_std": 0.5, "grad_norm": 5.8403520584106445, "kl": 0.030757042882032692, "learning_rate": 4.824709730217553e-06, "loss": 0.0013, "num_tokens": 23161264.0, "reward": 0.3875, "reward_std": 0.2833965331315994, "rewards/reward_fn/mean": 0.3875, "rewards/reward_fn/std": 0.28339654505252837, "sampling/importance_sampling_ratio/max": 1.319850754737854, "sampling/importance_sampling_ratio/mean": 1.0002212703227997, "sampling/importance_sampling_ratio/min": 0.7982283234596252, "sampling/sampling_logp_difference/max": 0.28737926483154297, "sampling/sampling_logp_difference/mean": 0.022109304368495942, "step": 5350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.7, "completions/max_terminated_length": 316.7, "completions/mean_length": 212.0625, "completions/mean_terminated_length": 212.0625, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.0923405088484288, "epoch": 0.49278293647145355, "frac_reward_zero_std": 0.2, "grad_norm": 7.041152477264404, "kl": 0.054811205936130133, "learning_rate": 4.8237765348803435e-06, "loss": 0.0031, "num_tokens": 23208213.0, "reward": -0.275, "reward_std": 0.3915108531713486, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.3915108531713486, "sampling/importance_sampling_ratio/max": 1.2654842853546142, "sampling/importance_sampling_ratio/mean": 0.999446302652359, "sampling/importance_sampling_ratio/min": 0.7852972567081451, "sampling/sampling_logp_difference/max": 0.26122102737426756, "sampling/sampling_logp_difference/mean": 0.02175639197230339, "step": 5360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.5, "completions/max_terminated_length": 301.5, "completions/mean_length": 202.9875, "completions/mean_terminated_length": 202.9875, "completions/min_length": 115.8, "completions/min_terminated_length": 115.8, "entropy": 1.031253406777978, "epoch": 0.49370230762158684, "frac_reward_zero_std": 0.5, "grad_norm": 6.265824317932129, "kl": 0.03812641363474541, "learning_rate": 4.8228409528383595e-06, "loss": 0.0015, "num_tokens": 23250668.0, "reward": -0.1125, "reward_std": 0.2920606702566147, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.29206068217754366, "sampling/importance_sampling_ratio/max": 1.2718768835067749, "sampling/importance_sampling_ratio/mean": 0.999932837486267, "sampling/importance_sampling_ratio/min": 0.8028461933135986, "sampling/sampling_logp_difference/max": 0.25324827432632446, "sampling/sampling_logp_difference/mean": 0.02090995218604803, "step": 5370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 207.475, "completions/mean_terminated_length": 207.475, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 1.0277273558080195, "epoch": 0.49462167877172014, "frac_reward_zero_std": 0.4, "grad_norm": 7.609491348266602, "kl": 0.05440606109332293, "learning_rate": 4.821902985052518e-06, "loss": 0.0027, "num_tokens": 23292042.0, "reward": 0.24375, "reward_std": 0.2875739336013794, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.2875739336013794, "sampling/importance_sampling_ratio/max": 1.2699689745903016, "sampling/importance_sampling_ratio/mean": 0.9999425411224365, "sampling/importance_sampling_ratio/min": 0.791676652431488, "sampling/sampling_logp_difference/max": 0.25165589451789855, "sampling/sampling_logp_difference/mean": 0.021200604364275934, "step": 5380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.6, "completions/max_terminated_length": 325.6, "completions/mean_length": 231.3125, "completions/mean_terminated_length": 231.3125, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0847119815647601, "epoch": 0.49554104992185344, "frac_reward_zero_std": 0.3, "grad_norm": 4.297759532928467, "kl": 0.034514278906863184, "learning_rate": 4.820962632486187e-06, "loss": 0.0013, "num_tokens": 23337795.0, "reward": 0.1125, "reward_std": 0.40591218173503874, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.4059121936559677, "sampling/importance_sampling_ratio/max": 1.3074236512184143, "sampling/importance_sampling_ratio/mean": 1.0004106163978577, "sampling/importance_sampling_ratio/min": 0.724758043885231, "sampling/sampling_logp_difference/max": 0.354433536529541, "sampling/sampling_logp_difference/mean": 0.021418475918471812, "step": 5390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 216.05, "completions/mean_terminated_length": 216.05, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.0616689071059227, "epoch": 0.49646042107198673, "frac_reward_zero_std": 0.0, "grad_norm": 5.7572455406188965, "kl": 0.036152307875454426, "learning_rate": 4.820019896105186e-06, "loss": 0.0017, "num_tokens": 23381615.0, "reward": 0.2375, "reward_std": 0.43865286409854887, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.4386528700590134, "sampling/importance_sampling_ratio/max": 1.2574617743492127, "sampling/importance_sampling_ratio/mean": 1.0003314852714538, "sampling/importance_sampling_ratio/min": 0.7687016904354096, "sampling/sampling_logp_difference/max": 0.28773764371871946, "sampling/sampling_logp_difference/mean": 0.021228195913136006, "step": 5400 }, { "epoch": 0.49646042107198673, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0028359511343804536, "eval_completions/max_length": 307.5602094240838, "eval_completions/max_terminated_length": 305.1378708551483, "eval_completions/mean_length": 214.98887434554973, "eval_completions/mean_terminated_length": 214.2825357593583, "eval_completions/min_length": 138.60383944153577, "eval_completions/min_terminated_length": 138.60383944153577, "eval_entropy": 1.097362355516011, "eval_frac_reward_zero_std": 0.2705061082024433, "eval_kl": 0.041150790996797526, "eval_loss": 0.002088259207084775, "eval_num_tokens": 23381615.0, "eval_reward": 0.04602966841186736, "eval_reward_std": 0.38701856157126435, "eval_rewards/reward_fn/mean": 0.04602966841186736, "eval_rewards/reward_fn/std": 0.38701857280564766, "eval_runtime": 1785.6175, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.2632229927322627, "eval_sampling/importance_sampling_ratio/mean": 0.9999714185846206, "eval_sampling/importance_sampling_ratio/min": 0.7768413987251268, "eval_sampling/sampling_logp_difference/max": 0.27680796405615815, "eval_sampling/sampling_logp_difference/mean": 0.020709345676329954, "eval_steps_per_second": 0.04, "step": 5400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 208.7125, "completions/mean_terminated_length": 208.7125, "completions/min_length": 157.2, "completions/min_terminated_length": 157.2, "entropy": 1.0855384014546872, "epoch": 0.4973797922221201, "frac_reward_zero_std": 0.3, "grad_norm": 0.11374184489250183, "kl": 0.04197303925757297, "learning_rate": 4.819074776877778e-06, "loss": 0.002, "num_tokens": 23430008.0, "reward": 0.1625, "reward_std": 0.345512917637825, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.34551292955875396, "sampling/importance_sampling_ratio/max": 1.295212173461914, "sampling/importance_sampling_ratio/mean": 1.0001162230968474, "sampling/importance_sampling_ratio/min": 0.7823019981384277, "sampling/sampling_logp_difference/max": 0.29045695066452026, "sampling/sampling_logp_difference/mean": 0.021257317438721657, "step": 5410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 331.3, "completions/max_terminated_length": 326.7, "completions/mean_length": 225.75, "completions/mean_terminated_length": 222.89642944335938, "completions/min_length": 149.7, "completions/min_terminated_length": 149.7, "entropy": 1.088605421036482, "epoch": 0.4982991633722534, "frac_reward_zero_std": 0.2, "grad_norm": 5.4198384284973145, "kl": 0.05059795609558933, "learning_rate": 4.818127275774678e-06, "loss": 0.0027, "num_tokens": 23475036.0, "reward": -0.025, "reward_std": 0.5018741875886917, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.5018741935491562, "sampling/importance_sampling_ratio/max": 1.2814471125602722, "sampling/importance_sampling_ratio/mean": 1.0002063870429994, "sampling/importance_sampling_ratio/min": 0.771407401561737, "sampling/sampling_logp_difference/max": 0.27978352904319764, "sampling/sampling_logp_difference/mean": 0.02201297674328089, "step": 5420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.7, "completions/max_terminated_length": 309.7, "completions/mean_length": 220.2875, "completions/mean_terminated_length": 220.2875, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.1716218397021294, "epoch": 0.4992185345223867, "frac_reward_zero_std": 0.3, "grad_norm": 0.18368206918239594, "kl": 0.046855731157120314, "learning_rate": 4.817177393769046e-06, "loss": 0.0024, "num_tokens": 23522891.0, "reward": 0.0, "reward_std": 0.4315427541732788, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.43154277205467223, "sampling/importance_sampling_ratio/max": 1.2900241613388062, "sampling/importance_sampling_ratio/mean": 1.0004517495632173, "sampling/importance_sampling_ratio/min": 0.7981406569480896, "sampling/sampling_logp_difference/max": 0.26332831382751465, "sampling/sampling_logp_difference/mean": 0.022118272632360457, "step": 5430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.1, "completions/max_terminated_length": 313.1, "completions/mean_length": 208.7125, "completions/mean_terminated_length": 208.7125, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0546998620033263, "epoch": 0.50013790567252, "frac_reward_zero_std": 0.6, "grad_norm": 4.902407646179199, "kl": 0.04262739559053443, "learning_rate": 4.816225131836485e-06, "loss": 0.0019, "num_tokens": 23566548.0, "reward": 0.1875, "reward_std": 0.20353024303913117, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.20353024303913117, "sampling/importance_sampling_ratio/max": 1.2638328313827514, "sampling/importance_sampling_ratio/mean": 0.9998726129531861, "sampling/importance_sampling_ratio/min": 0.8027492523193359, "sampling/sampling_logp_difference/max": 0.2428574800491333, "sampling/sampling_logp_difference/mean": 0.021070289239287377, "step": 5440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.7, "completions/max_terminated_length": 275.7, "completions/mean_length": 197.15, "completions/mean_terminated_length": 197.15, "completions/min_length": 133.4, "completions/min_terminated_length": 133.4, "entropy": 1.0130454123020172, "epoch": 0.5010572768226533, "frac_reward_zero_std": 0.4, "grad_norm": 4.8741374015808105, "kl": 0.03473435846390203, "learning_rate": 4.8152704909550455e-06, "loss": 0.0016, "num_tokens": 23609680.0, "reward": -0.125, "reward_std": 0.2958350956439972, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.29583510756492615, "sampling/importance_sampling_ratio/max": 1.2904534578323363, "sampling/importance_sampling_ratio/mean": 0.9997602343559265, "sampling/importance_sampling_ratio/min": 0.7649597525596619, "sampling/sampling_logp_difference/max": 0.31208584308624265, "sampling/sampling_logp_difference/mean": 0.020262810960412027, "step": 5450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.7, "completions/max_terminated_length": 325.7, "completions/mean_length": 213.925, "completions/mean_terminated_length": 213.925, "completions/min_length": 146.2, "completions/min_terminated_length": 146.2, "entropy": 1.1191648043692113, "epoch": 0.5019766479727866, "frac_reward_zero_std": 0.3, "grad_norm": 8.293253898620605, "kl": 0.053668214788194744, "learning_rate": 4.8143134721052205e-06, "loss": 0.003, "num_tokens": 23655090.0, "reward": 0.1375, "reward_std": 0.3743988037109375, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.37439881563186644, "sampling/importance_sampling_ratio/max": 1.277535653114319, "sampling/importance_sampling_ratio/mean": 0.9996655941009521, "sampling/importance_sampling_ratio/min": 0.7746275663375854, "sampling/sampling_logp_difference/max": 0.2920875698328018, "sampling/sampling_logp_difference/mean": 0.021859637834131716, "step": 5460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.8, "completions/max_terminated_length": 282.8, "completions/mean_length": 211.0625, "completions/mean_terminated_length": 211.0625, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0439486525952817, "epoch": 0.5028960191229199, "frac_reward_zero_std": 0.4, "grad_norm": 4.73133659362793, "kl": 0.04664284294121899, "learning_rate": 4.8133540762699445e-06, "loss": 0.002, "num_tokens": 23699231.0, "reward": 0.2, "reward_std": 0.279134601354599, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.27913460731506345, "sampling/importance_sampling_ratio/max": 1.2586267232894897, "sampling/importance_sampling_ratio/mean": 1.0003837823867798, "sampling/importance_sampling_ratio/min": 0.7755716264247894, "sampling/sampling_logp_difference/max": 0.26464965343475344, "sampling/sampling_logp_difference/mean": 0.021272764541208744, "step": 5470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 214.0625, "completions/mean_terminated_length": 214.0625, "completions/min_length": 126.8, "completions/min_terminated_length": 126.8, "entropy": 1.0752910524606705, "epoch": 0.5038153902730532, "frac_reward_zero_std": 0.4, "grad_norm": 5.440293788909912, "kl": 0.04500622688210569, "learning_rate": 4.812392304434595e-06, "loss": 0.0021, "num_tokens": 23741532.0, "reward": 0.15, "reward_std": 0.30942772030830384, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.30942773818969727, "sampling/importance_sampling_ratio/max": 1.320425820350647, "sampling/importance_sampling_ratio/mean": 0.9999340176582336, "sampling/importance_sampling_ratio/min": 0.7474205732345581, "sampling/sampling_logp_difference/max": 0.3206120550632477, "sampling/sampling_logp_difference/mean": 0.022241763584315778, "step": 5480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.7, "completions/max_terminated_length": 280.7, "completions/mean_length": 185.2875, "completions/mean_terminated_length": 185.2875, "completions/min_length": 116.8, "completions/min_terminated_length": 116.8, "entropy": 1.1046483531594276, "epoch": 0.5047347614231865, "frac_reward_zero_std": 0.5, "grad_norm": 6.75298547744751, "kl": 0.0329540359089151, "learning_rate": 4.811428157586987e-06, "loss": 0.0011, "num_tokens": 23781219.0, "reward": 0.4375, "reward_std": 0.25470708310604095, "rewards/reward_fn/mean": 0.4375, "rewards/reward_fn/std": 0.2547071009874344, "sampling/importance_sampling_ratio/max": 1.2729754805564881, "sampling/importance_sampling_ratio/mean": 1.0001352250576019, "sampling/importance_sampling_ratio/min": 0.7876677334308624, "sampling/sampling_logp_difference/max": 0.2548651576042175, "sampling/sampling_logp_difference/mean": 0.022425617650151254, "step": 5490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.8, "completions/max_terminated_length": 305.8, "completions/mean_length": 219.45, "completions/mean_terminated_length": 219.45, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.0868753015995025, "epoch": 0.5056541325733198, "frac_reward_zero_std": 0.6, "grad_norm": 5.9453043937683105, "kl": 0.035435753501951696, "learning_rate": 4.810461636717378e-06, "loss": 0.0015, "num_tokens": 23828383.0, "reward": 0.3625, "reward_std": 0.1974882572889328, "rewards/reward_fn/mean": 0.3625, "rewards/reward_fn/std": 0.19748826324939728, "sampling/importance_sampling_ratio/max": 1.2394713640213013, "sampling/importance_sampling_ratio/mean": 0.9994851529598237, "sampling/importance_sampling_ratio/min": 0.7812922775745392, "sampling/sampling_logp_difference/max": 0.27424551248550416, "sampling/sampling_logp_difference/mean": 0.02108355090022087, "step": 5500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.6, "completions/max_terminated_length": 340.6, "completions/mean_length": 240.4, "completions/mean_terminated_length": 240.4, "completions/min_length": 161.1, "completions/min_terminated_length": 161.1, "entropy": 1.080581258237362, "epoch": 0.5065735037234531, "frac_reward_zero_std": 0.3, "grad_norm": 0.3365761637687683, "kl": 0.04969004578888416, "learning_rate": 4.809492742818461e-06, "loss": 0.0026, "num_tokens": 23873775.0, "reward": -0.025, "reward_std": 0.3202547669410706, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.3202547788619995, "sampling/importance_sampling_ratio/max": 1.3583197593688965, "sampling/importance_sampling_ratio/mean": 1.0000873923301696, "sampling/importance_sampling_ratio/min": 0.7611360907554626, "sampling/sampling_logp_difference/max": 0.36482442617416383, "sampling/sampling_logp_difference/mean": 0.02113449703902006, "step": 5510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.7, "completions/max_terminated_length": 312.7, "completions/mean_length": 235.7375, "completions/mean_terminated_length": 235.7375, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.1689024038612843, "epoch": 0.5074928748735865, "frac_reward_zero_std": 0.2, "grad_norm": 0.155808225274086, "kl": 0.0653985949466005, "learning_rate": 4.808521476885369e-06, "loss": 0.0035, "num_tokens": 23920354.0, "reward": -0.1125, "reward_std": 0.33921052813529967, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.3392105340957642, "sampling/importance_sampling_ratio/max": 1.2750370264053346, "sampling/importance_sampling_ratio/mean": 1.0001432359218598, "sampling/importance_sampling_ratio/min": 0.7733922660350799, "sampling/sampling_logp_difference/max": 0.2868382573127747, "sampling/sampling_logp_difference/mean": 0.022046180628240108, "step": 5520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 345.0, "completions/max_terminated_length": 345.0, "completions/mean_length": 244.65, "completions/mean_terminated_length": 244.65, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.118749164789915, "epoch": 0.5084122460237198, "frac_reward_zero_std": 0.3, "grad_norm": 4.13883113861084, "kl": 0.04233728646649979, "learning_rate": 4.80754783991567e-06, "loss": 0.0023, "num_tokens": 23968942.0, "reward": 0.225, "reward_std": 0.3486855089664459, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.34868552088737487, "sampling/importance_sampling_ratio/max": 1.2819775581359862, "sampling/importance_sampling_ratio/mean": 0.9992317497730255, "sampling/importance_sampling_ratio/min": 0.7228488445281982, "sampling/sampling_logp_difference/max": 0.35451061725616456, "sampling/sampling_logp_difference/mean": 0.020751421339809895, "step": 5530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 203.075, "completions/mean_terminated_length": 203.075, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.0664518512785435, "epoch": 0.5093316171738531, "frac_reward_zero_std": 0.4, "grad_norm": 6.4434919357299805, "kl": 0.03897197861224413, "learning_rate": 4.806571832909367e-06, "loss": 0.0023, "num_tokens": 24013916.0, "reward": -0.025, "reward_std": 0.2411646068096161, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2411646068096161, "sampling/importance_sampling_ratio/max": 1.2860379338264465, "sampling/importance_sampling_ratio/mean": 0.9992936015129089, "sampling/importance_sampling_ratio/min": 0.7554073631763458, "sampling/sampling_logp_difference/max": 0.31667001247406007, "sampling/sampling_logp_difference/mean": 0.021709810197353362, "step": 5540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 331.6, "completions/max_terminated_length": 331.6, "completions/mean_length": 210.875, "completions/mean_terminated_length": 210.875, "completions/min_length": 120.8, "completions/min_terminated_length": 120.8, "entropy": 1.0497647486627102, "epoch": 0.5102509883239864, "frac_reward_zero_std": 0.5, "grad_norm": 7.073035717010498, "kl": 0.05006906929193065, "learning_rate": 4.805593456868897e-06, "loss": 0.0025, "num_tokens": 24056306.0, "reward": 0.225, "reward_std": 0.23831535279750823, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.23831535279750823, "sampling/importance_sampling_ratio/max": 1.2860930919647218, "sampling/importance_sampling_ratio/mean": 1.0004346251487732, "sampling/importance_sampling_ratio/min": 0.7597083330154419, "sampling/sampling_logp_difference/max": 0.3008319616317749, "sampling/sampling_logp_difference/mean": 0.021445170044898987, "step": 5550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.7, "completions/max_terminated_length": 299.7, "completions/mean_length": 201.975, "completions/mean_terminated_length": 201.975, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.026543417572975, "epoch": 0.5111703594741197, "frac_reward_zero_std": 0.1, "grad_norm": 6.548365116119385, "kl": 0.03880849097040482, "learning_rate": 4.804612712799131e-06, "loss": 0.002, "num_tokens": 24096016.0, "reward": -0.0125, "reward_std": 0.45168755352497103, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.45168757140636445, "sampling/importance_sampling_ratio/max": 1.2572989583015441, "sampling/importance_sampling_ratio/mean": 1.0003212630748748, "sampling/importance_sampling_ratio/min": 0.7860771536827087, "sampling/sampling_logp_difference/max": 0.25114492177963255, "sampling/sampling_logp_difference/mean": 0.020918383449316024, "step": 5560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 200.575, "completions/mean_terminated_length": 200.575, "completions/min_length": 115.9, "completions/min_terminated_length": 115.9, "entropy": 1.017763302475214, "epoch": 0.512089730624253, "frac_reward_zero_std": 0.3, "grad_norm": 5.291685581207275, "kl": 0.039697503374191, "learning_rate": 4.803629601707371e-06, "loss": 0.002, "num_tokens": 24137646.0, "reward": -0.1125, "reward_std": 0.36288144290447233, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.36288144886493684, "sampling/importance_sampling_ratio/max": 1.258324921131134, "sampling/importance_sampling_ratio/mean": 1.0002608120441436, "sampling/importance_sampling_ratio/min": 0.7740411341190339, "sampling/sampling_logp_difference/max": 0.26839998066425325, "sampling/sampling_logp_difference/mean": 0.020473778247833252, "step": 5570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.5, "completions/max_terminated_length": 263.5, "completions/mean_length": 194.0125, "completions/mean_terminated_length": 194.0125, "completions/min_length": 115.3, "completions/min_terminated_length": 115.3, "entropy": 1.1239424429833889, "epoch": 0.5130091017743863, "frac_reward_zero_std": 0.5, "grad_norm": 4.650895595550537, "kl": 0.04226044338429347, "learning_rate": 4.802644124603352e-06, "loss": 0.0022, "num_tokens": 24179487.0, "reward": 0.075, "reward_std": 0.2597990781068802, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.2597990781068802, "sampling/importance_sampling_ratio/max": 1.2658164501190186, "sampling/importance_sampling_ratio/mean": 0.9998365223407746, "sampling/importance_sampling_ratio/min": 0.743103152513504, "sampling/sampling_logp_difference/max": 0.32296501398086547, "sampling/sampling_logp_difference/mean": 0.022668720223009586, "step": 5580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 327.4, "completions/max_terminated_length": 321.8, "completions/mean_length": 226.8875, "completions/mean_terminated_length": 223.66071472167968, "completions/min_length": 146.3, "completions/min_terminated_length": 146.3, "entropy": 1.0531333975493908, "epoch": 0.5139284729245196, "frac_reward_zero_std": 0.4, "grad_norm": 6.46544885635376, "kl": 0.06050649849930778, "learning_rate": 4.8016562824992375e-06, "loss": 0.0027, "num_tokens": 24222814.0, "reward": -0.24375, "reward_std": 0.32643375992774964, "rewards/reward_fn/mean": -0.24375, "rewards/reward_fn/std": 0.32643375992774964, "sampling/importance_sampling_ratio/max": 1.4233837723731995, "sampling/importance_sampling_ratio/mean": 1.0004831790924071, "sampling/importance_sampling_ratio/min": 0.7718040168285369, "sampling/sampling_logp_difference/max": 0.37423638105392454, "sampling/sampling_logp_difference/mean": 0.020690672285854816, "step": 5590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.9, "completions/max_terminated_length": 324.9, "completions/mean_length": 241.8, "completions/mean_terminated_length": 241.8, "completions/min_length": 167.0, "completions/min_terminated_length": 167.0, "entropy": 1.044118883088231, "epoch": 0.5148478440746529, "frac_reward_zero_std": 0.4, "grad_norm": 5.0964155197143555, "kl": 0.04273007106385194, "learning_rate": 4.80066607640962e-06, "loss": 0.0017, "num_tokens": 24269926.0, "reward": 0.2, "reward_std": 0.28198385536670684, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.28198386132717135, "sampling/importance_sampling_ratio/max": 1.2610069870948792, "sampling/importance_sampling_ratio/mean": 0.9994114875793457, "sampling/importance_sampling_ratio/min": 0.7865147888660431, "sampling/sampling_logp_difference/max": 0.2615241646766663, "sampling/sampling_logp_difference/mean": 0.020354469120502473, "step": 5600 }, { "epoch": 0.5148478440746529, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.005671902268760907, "eval_completions/max_length": 315.58987783595114, "eval_completions/max_terminated_length": 309.56893542757416, "eval_completions/mean_length": 219.44328097731238, "eval_completions/mean_terminated_length": 217.974965067226, "eval_completions/min_length": 140.5130890052356, "eval_completions/min_terminated_length": 140.5130890052356, "eval_entropy": 1.1105714183931368, "eval_frac_reward_zero_std": 0.30715532286212915, "eval_kl": 0.04778910652908251, "eval_loss": 0.0024661123752593994, "eval_num_tokens": 24269926.0, "eval_reward": 0.0549738219895288, "eval_reward_std": 0.34697055358953294, "eval_rewards/reward_fn/mean": 0.0549738219895288, "eval_rewards/reward_fn/std": 0.34697056378369556, "eval_runtime": 1827.6072, "eval_samples_per_second": 0.314, "eval_sampling/importance_sampling_ratio/max": 1.2673593607664524, "eval_sampling/importance_sampling_ratio/mean": 1.0000326239209734, "eval_sampling/importance_sampling_ratio/min": 0.7793972575955782, "eval_sampling/sampling_logp_difference/max": 0.2750438252044598, "eval_sampling/sampling_logp_difference/mean": 0.021003692288985427, "eval_steps_per_second": 0.039, "step": 5600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 337.7, "completions/max_terminated_length": 337.2, "completions/mean_length": 233.2125, "completions/mean_terminated_length": 227.69166870117186, "completions/min_length": 123.1, "completions/min_terminated_length": 123.1, "entropy": 1.0501929476857186, "epoch": 0.5157672152247862, "frac_reward_zero_std": 0.1, "grad_norm": 5.607937335968018, "kl": 0.06056828326545656, "learning_rate": 4.799673507351522e-06, "loss": 0.003, "num_tokens": 24312359.0, "reward": -0.2125, "reward_std": 0.46867029964923856, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.46867031753063204, "sampling/importance_sampling_ratio/max": 1.2412830114364624, "sampling/importance_sampling_ratio/mean": 1.0000840306282044, "sampling/importance_sampling_ratio/min": 0.770388787984848, "sampling/sampling_logp_difference/max": 0.2749655842781067, "sampling/sampling_logp_difference/mean": 0.020925150718539953, "step": 5610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.2, "completions/max_terminated_length": 282.2, "completions/mean_length": 200.6875, "completions/mean_terminated_length": 200.6875, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0710347529500723, "epoch": 0.5166865863749196, "frac_reward_zero_std": 0.2, "grad_norm": 5.6804656982421875, "kl": 0.03572373668430373, "learning_rate": 4.79867857634439e-06, "loss": 0.0017, "num_tokens": 24352510.0, "reward": 0.46875, "reward_std": 0.44541222155094146, "rewards/reward_fn/mean": 0.46875, "rewards/reward_fn/std": 0.4454122453927994, "sampling/importance_sampling_ratio/max": 1.3391463160514832, "sampling/importance_sampling_ratio/mean": 1.0001074016094207, "sampling/importance_sampling_ratio/min": 0.756105387210846, "sampling/sampling_logp_difference/max": 0.3262542188167572, "sampling/sampling_logp_difference/mean": 0.021790395304560662, "step": 5620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.9, "completions/max_terminated_length": 313.9, "completions/mean_length": 220.125, "completions/mean_terminated_length": 220.125, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.098129228502512, "epoch": 0.5176059575250529, "frac_reward_zero_std": 0.3, "grad_norm": 5.304335594177246, "kl": 0.05576671363087371, "learning_rate": 4.797681284410099e-06, "loss": 0.0027, "num_tokens": 24397704.0, "reward": -0.075, "reward_std": 0.34776818454265596, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.34776818454265596, "sampling/importance_sampling_ratio/max": 1.2820573925971985, "sampling/importance_sampling_ratio/mean": 0.9999447584152221, "sampling/importance_sampling_ratio/min": 0.7967436611652374, "sampling/sampling_logp_difference/max": 0.2670341968536377, "sampling/sampling_logp_difference/mean": 0.02156980838626623, "step": 5630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.2, "completions/max_terminated_length": 317.2, "completions/mean_length": 214.6875, "completions/mean_terminated_length": 214.6875, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.016185887902975, "epoch": 0.5185253286751862, "frac_reward_zero_std": 0.6, "grad_norm": 0.14543519914150238, "kl": 0.0483966107130982, "learning_rate": 4.796681632572946e-06, "loss": 0.0022, "num_tokens": 24437607.0, "reward": -0.1625, "reward_std": 0.16875659823417663, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.1687566041946411, "sampling/importance_sampling_ratio/max": 1.2632392406463624, "sampling/importance_sampling_ratio/mean": 0.9994130074977875, "sampling/importance_sampling_ratio/min": 0.7837559282779694, "sampling/sampling_logp_difference/max": 0.2636220932006836, "sampling/sampling_logp_difference/mean": 0.020958153530955315, "step": 5640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 224.1625, "completions/mean_terminated_length": 224.1625, "completions/min_length": 136.8, "completions/min_terminated_length": 136.8, "entropy": 1.0933317504823208, "epoch": 0.5194446998253195, "frac_reward_zero_std": 0.2, "grad_norm": 6.972399711608887, "kl": 0.05069115615915507, "learning_rate": 4.795679621859656e-06, "loss": 0.0021, "num_tokens": 24483484.0, "reward": 0.1, "reward_std": 0.32111361920833587, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.3211136251688004, "sampling/importance_sampling_ratio/max": 1.2839259147644042, "sampling/importance_sampling_ratio/mean": 0.9994974672794342, "sampling/importance_sampling_ratio/min": 0.7433544337749481, "sampling/sampling_logp_difference/max": 0.3108355164527893, "sampling/sampling_logp_difference/mean": 0.02195710614323616, "step": 5650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.2, "completions/max_terminated_length": 298.8, "completions/mean_length": 210.6375, "completions/mean_terminated_length": 207.17857208251954, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.0530016131699085, "epoch": 0.5203640709754528, "frac_reward_zero_std": 0.5, "grad_norm": 0.1349094957113266, "kl": 0.054761074238922444, "learning_rate": 4.794675253299373e-06, "loss": 0.0023, "num_tokens": 24527911.0, "reward": -0.18125, "reward_std": 0.306527179479599, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.30652719736099243, "sampling/importance_sampling_ratio/max": 1.2715108275413514, "sampling/importance_sampling_ratio/mean": 1.0005002439022064, "sampling/importance_sampling_ratio/min": 0.7686617136001587, "sampling/sampling_logp_difference/max": 0.2917294502258301, "sampling/sampling_logp_difference/mean": 0.021296564675867557, "step": 5660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 349.6, "completions/max_terminated_length": 328.8, "completions/mean_length": 239.1, "completions/mean_terminated_length": 235.28214416503906, "completions/min_length": 161.5, "completions/min_terminated_length": 161.5, "entropy": 1.0534119665622712, "epoch": 0.5212834421255861, "frac_reward_zero_std": 0.1, "grad_norm": 5.625903129577637, "kl": 0.05668536044540815, "learning_rate": 4.793668527923664e-06, "loss": 0.0025, "num_tokens": 24576903.0, "reward": 0.06875, "reward_std": 0.3632109582424164, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.36321096420288085, "sampling/importance_sampling_ratio/max": 1.3016605615615844, "sampling/importance_sampling_ratio/mean": 0.9999427795410156, "sampling/importance_sampling_ratio/min": 0.7347596615552903, "sampling/sampling_logp_difference/max": 0.35710935592651366, "sampling/sampling_logp_difference/mean": 0.020988625288009644, "step": 5670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.4, "completions/max_terminated_length": 321.4, "completions/mean_length": 207.8125, "completions/mean_terminated_length": 207.8125, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0410642627626658, "epoch": 0.5222028132757194, "frac_reward_zero_std": 0.5, "grad_norm": 0.201572984457016, "kl": 0.051045022631296885, "learning_rate": 4.792659446766517e-06, "loss": 0.0026, "num_tokens": 24620832.0, "reward": 0.3375, "reward_std": 0.24485630989074708, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.24485631585121154, "sampling/importance_sampling_ratio/max": 1.2378967881202698, "sampling/importance_sampling_ratio/mean": 1.000049066543579, "sampling/importance_sampling_ratio/min": 0.7668177425861359, "sampling/sampling_logp_difference/max": 0.28151760101318357, "sampling/sampling_logp_difference/mean": 0.021273509226739408, "step": 5680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 334.2, "completions/max_terminated_length": 315.8, "completions/mean_length": 218.5875, "completions/mean_terminated_length": 215.53214416503906, "completions/min_length": 145.5, "completions/min_terminated_length": 145.5, "entropy": 1.0421565152704715, "epoch": 0.5231221844258527, "frac_reward_zero_std": 0.5, "grad_norm": 0.11205685138702393, "kl": 0.05851724444655702, "learning_rate": 4.791648010864338e-06, "loss": 0.0032, "num_tokens": 24664423.0, "reward": -0.11875, "reward_std": 0.2398865133523941, "rewards/reward_fn/mean": -0.11875, "rewards/reward_fn/std": 0.2398865193128586, "sampling/importance_sampling_ratio/max": 1.250590181350708, "sampling/importance_sampling_ratio/mean": 1.0005945086479187, "sampling/importance_sampling_ratio/min": 0.7430494248867034, "sampling/sampling_logp_difference/max": 0.3044642448425293, "sampling/sampling_logp_difference/mean": 0.02128320075571537, "step": 5690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 225.8625, "completions/mean_terminated_length": 225.8625, "completions/min_length": 155.4, "completions/min_terminated_length": 155.4, "entropy": 1.0327788598835468, "epoch": 0.524041555575986, "frac_reward_zero_std": 0.3, "grad_norm": 0.10939747095108032, "kl": 0.05683743947884068, "learning_rate": 4.790634221255955e-06, "loss": 0.0026, "num_tokens": 24709876.0, "reward": 0.025, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2294748663902282, "sampling/importance_sampling_ratio/mean": 0.999708741903305, "sampling/importance_sampling_ratio/min": 0.7827854335308075, "sampling/sampling_logp_difference/max": 0.2474528133869171, "sampling/sampling_logp_difference/mean": 0.020984406210482122, "step": 5700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.3, "completions/max_terminated_length": 274.3, "completions/mean_length": 209.15, "completions/mean_terminated_length": 209.15, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.1975786916911602, "epoch": 0.5249609267261194, "frac_reward_zero_std": 0.1, "grad_norm": 0.12670528888702393, "kl": 0.06615693124476821, "learning_rate": 4.789618078982609e-06, "loss": 0.0033, "num_tokens": 24756248.0, "reward": -0.0625, "reward_std": 0.41830068826675415, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.41830070614814757, "sampling/importance_sampling_ratio/max": 1.2364379525184632, "sampling/importance_sampling_ratio/mean": 0.9996397078037262, "sampling/importance_sampling_ratio/min": 0.7818032085895539, "sampling/sampling_logp_difference/max": 0.2549152851104736, "sampling/sampling_logp_difference/mean": 0.02394681181758642, "step": 5710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.1, "completions/max_terminated_length": 309.1, "completions/mean_length": 203.175, "completions/mean_terminated_length": 203.175, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 0.975874250382185, "epoch": 0.5258802978762527, "frac_reward_zero_std": 0.1, "grad_norm": 5.878521919250488, "kl": 0.054947742447257045, "learning_rate": 4.788599585087961e-06, "loss": 0.0024, "num_tokens": 24798830.0, "reward": -0.0875, "reward_std": 0.41799977123737336, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.4179997891187668, "sampling/importance_sampling_ratio/max": 1.2630855560302734, "sampling/importance_sampling_ratio/mean": 1.0002691149711609, "sampling/importance_sampling_ratio/min": 0.768866166472435, "sampling/sampling_logp_difference/max": 0.3106554627418518, "sampling/sampling_logp_difference/mean": 0.0207120256498456, "step": 5720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 200.875, "completions/mean_terminated_length": 200.875, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0568383276462554, "epoch": 0.526799669026386, "frac_reward_zero_std": 0.6, "grad_norm": 5.458049774169922, "kl": 0.059383532774518244, "learning_rate": 4.787578740618085e-06, "loss": 0.0028, "num_tokens": 24844860.0, "reward": 0.1, "reward_std": 0.23594963550567627, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.23594963550567627, "sampling/importance_sampling_ratio/max": 1.2591054797172547, "sampling/importance_sampling_ratio/mean": 0.9997957229614258, "sampling/importance_sampling_ratio/min": 0.7429640352725982, "sampling/sampling_logp_difference/max": 0.31181050539016725, "sampling/sampling_logp_difference/mean": 0.02219392582774162, "step": 5730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 301.9, "completions/max_terminated_length": 280.4, "completions/mean_length": 206.6375, "completions/mean_terminated_length": 202.7875015258789, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.030839866027236, "epoch": 0.5277190401765193, "frac_reward_zero_std": 0.4, "grad_norm": 5.549622058868408, "kl": 0.06440929214004427, "learning_rate": 4.786555546621471e-06, "loss": 0.003, "num_tokens": 24889055.0, "reward": 0.09375, "reward_std": 0.25552249550819395, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.25552249550819395, "sampling/importance_sampling_ratio/max": 1.259114420413971, "sampling/importance_sampling_ratio/mean": 1.0000966727733611, "sampling/importance_sampling_ratio/min": 0.7558643162250519, "sampling/sampling_logp_difference/max": 0.2906364440917969, "sampling/sampling_logp_difference/mean": 0.021959654986858368, "step": 5740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 204.725, "completions/mean_terminated_length": 204.725, "completions/min_length": 126.0, "completions/min_terminated_length": 126.0, "entropy": 1.1053141940385103, "epoch": 0.5286384113266526, "frac_reward_zero_std": 0.6, "grad_norm": 0.14527277648448944, "kl": 0.05932108196429908, "learning_rate": 4.78553000414902e-06, "loss": 0.0028, "num_tokens": 24934417.0, "reward": 0.2625, "reward_std": 0.18165890872478485, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.18165890872478485, "sampling/importance_sampling_ratio/max": 1.3177197337150575, "sampling/importance_sampling_ratio/mean": 0.9995461583137513, "sampling/importance_sampling_ratio/min": 0.7414583563804626, "sampling/sampling_logp_difference/max": 0.3368691593408585, "sampling/sampling_logp_difference/mean": 0.02223340328782797, "step": 5750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.6, "completions/max_terminated_length": 328.6, "completions/mean_length": 211.9625, "completions/mean_terminated_length": 211.9625, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.0027941025793552, "epoch": 0.5295577824767859, "frac_reward_zero_std": 0.3, "grad_norm": 0.11421628296375275, "kl": 0.05854715235182084, "learning_rate": 4.784502114254048e-06, "loss": 0.0032, "num_tokens": 24975278.0, "reward": 0.125, "reward_std": 0.28198385536670684, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.28198386132717135, "sampling/importance_sampling_ratio/max": 1.2803743124008178, "sampling/importance_sampling_ratio/mean": 0.9997711777687073, "sampling/importance_sampling_ratio/min": 0.7688262104988098, "sampling/sampling_logp_difference/max": 0.2703778803348541, "sampling/sampling_logp_difference/mean": 0.020782227255403997, "step": 5760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.2, "completions/max_terminated_length": 296.2, "completions/mean_length": 211.9125, "completions/mean_terminated_length": 211.9125, "completions/min_length": 135.7, "completions/min_terminated_length": 135.7, "entropy": 1.0665974929928779, "epoch": 0.5304771536269192, "frac_reward_zero_std": 0.2, "grad_norm": 0.1431657075881958, "kl": 0.05518575538881123, "learning_rate": 4.783471877992279e-06, "loss": 0.0029, "num_tokens": 25016967.0, "reward": -0.0125, "reward_std": 0.447176194190979, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4471762001514435, "sampling/importance_sampling_ratio/max": 1.230408501625061, "sampling/importance_sampling_ratio/mean": 1.0002275586128235, "sampling/importance_sampling_ratio/min": 0.7900798976421356, "sampling/sampling_logp_difference/max": 0.23983997106552124, "sampling/sampling_logp_difference/mean": 0.0215107848867774, "step": 5770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 320.2, "completions/max_terminated_length": 295.7, "completions/mean_length": 219.3875, "completions/mean_terminated_length": 215.79107208251952, "completions/min_length": 150.1, "completions/min_terminated_length": 150.1, "entropy": 1.0803352303802967, "epoch": 0.5313965247770525, "frac_reward_zero_std": 0.1, "grad_norm": 6.364077091217041, "kl": 0.04843412353657186, "learning_rate": 4.782439296421849e-06, "loss": 0.0024, "num_tokens": 25060574.0, "reward": -0.01875, "reward_std": 0.4253198474645615, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.42531985342502593, "sampling/importance_sampling_ratio/max": 1.2883960366249085, "sampling/importance_sampling_ratio/mean": 1.0003319561481476, "sampling/importance_sampling_ratio/min": 0.7825756907463074, "sampling/sampling_logp_difference/max": 0.2829066514968872, "sampling/sampling_logp_difference/mean": 0.020909501053392888, "step": 5780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.9, "completions/max_terminated_length": 285.9, "completions/mean_length": 205.125, "completions/mean_terminated_length": 205.125, "completions/min_length": 138.2, "completions/min_terminated_length": 138.2, "entropy": 1.0578637324273585, "epoch": 0.5323158959271858, "frac_reward_zero_std": 0.4, "grad_norm": 0.13388663530349731, "kl": 0.03832183027989231, "learning_rate": 4.781404370603301e-06, "loss": 0.0018, "num_tokens": 25103304.0, "reward": 0.25, "reward_std": 0.31589864790439603, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.31589865386486055, "sampling/importance_sampling_ratio/max": 1.2577978253364563, "sampling/importance_sampling_ratio/mean": 0.9995961129665375, "sampling/importance_sampling_ratio/min": 0.747710257768631, "sampling/sampling_logp_difference/max": 0.3071512818336487, "sampling/sampling_logp_difference/mean": 0.021728925220668317, "step": 5790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 355.1, "completions/max_terminated_length": 355.1, "completions/mean_length": 236.575, "completions/mean_terminated_length": 236.575, "completions/min_length": 154.9, "completions/min_terminated_length": 154.9, "entropy": 1.0457797840237617, "epoch": 0.5332352670773192, "frac_reward_zero_std": 0.4, "grad_norm": 0.11066441237926483, "kl": 0.04156140626873821, "learning_rate": 4.7803671015995875e-06, "loss": 0.0023, "num_tokens": 25147110.0, "reward": 0.0, "reward_std": 0.35502841174602506, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3550284177064896, "sampling/importance_sampling_ratio/max": 1.3520105481147766, "sampling/importance_sampling_ratio/mean": 1.0007465183734894, "sampling/importance_sampling_ratio/min": 0.7489111661911011, "sampling/sampling_logp_difference/max": 0.32891182899475097, "sampling/sampling_logp_difference/mean": 0.021299775503575803, "step": 5800 }, { "epoch": 0.5332352670773192, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 311.3176265270506, "eval_completions/max_terminated_length": 308.87085514834206, "eval_completions/mean_length": 214.93106457242584, "eval_completions/mean_terminated_length": 214.31188211224674, "eval_completions/min_length": 138.0349040139616, "eval_completions/min_terminated_length": 138.0349040139616, "eval_entropy": 1.105723311451717, "eval_frac_reward_zero_std": 0.2513089005235602, "eval_kl": 0.0404926140713026, "eval_loss": 0.002022106433287263, "eval_num_tokens": 25147110.0, "eval_reward": 0.0462478184991274, "eval_reward_std": 0.3782694250293219, "eval_rewards/reward_fn/mean": 0.0462478184991274, "eval_rewards/reward_fn/std": 0.37826943709588173, "eval_runtime": 1805.3312, "eval_samples_per_second": 0.317, "eval_sampling/importance_sampling_ratio/max": 1.2700951155567668, "eval_sampling/importance_sampling_ratio/mean": 0.9999895534798321, "eval_sampling/importance_sampling_ratio/min": 0.7736060693209916, "eval_sampling/sampling_logp_difference/max": 0.283167751463712, "eval_sampling/sampling_logp_difference/mean": 0.021514873132499723, "eval_steps_per_second": 0.04, "step": 5800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 311.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 212.525, "completions/mean_terminated_length": 210.09285888671874, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.0477978996932507, "epoch": 0.5341546382274525, "frac_reward_zero_std": 0.1, "grad_norm": 4.900361061096191, "kl": 0.05265176862012595, "learning_rate": 4.779327490476068e-06, "loss": 0.0027, "num_tokens": 25189528.0, "reward": -0.09375, "reward_std": 0.4678551137447357, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.4678551137447357, "sampling/importance_sampling_ratio/max": 1.2737028479576111, "sampling/importance_sampling_ratio/mean": 1.0002426564693452, "sampling/importance_sampling_ratio/min": 0.7759451687335968, "sampling/sampling_logp_difference/max": 0.27017974853515625, "sampling/sampling_logp_difference/mean": 0.021615360863506794, "step": 5810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 219.2125, "completions/mean_terminated_length": 219.2125, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.0800211794674397, "epoch": 0.5350740093775858, "frac_reward_zero_std": 0.2, "grad_norm": 0.12037783861160278, "kl": 0.032971079961862416, "learning_rate": 4.778285538300505e-06, "loss": 0.0015, "num_tokens": 25235785.0, "reward": -0.1375, "reward_std": 0.3899959743022919, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.38999598622322085, "sampling/importance_sampling_ratio/max": 1.2625499725341798, "sampling/importance_sampling_ratio/mean": 0.9999870717525482, "sampling/importance_sampling_ratio/min": 0.782296234369278, "sampling/sampling_logp_difference/max": 0.2733001708984375, "sampling/sampling_logp_difference/mean": 0.021053713746368885, "step": 5820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 345.7, "completions/max_terminated_length": 326.3, "completions/mean_length": 226.575, "completions/mean_terminated_length": 223.30892944335938, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.1620180413126946, "epoch": 0.535993380527719, "frac_reward_zero_std": 0.2, "grad_norm": 0.10908669233322144, "kl": 0.0500280971522443, "learning_rate": 4.777241246143066e-06, "loss": 0.0023, "num_tokens": 25282247.0, "reward": -0.11875, "reward_std": 0.40702066421508787, "rewards/reward_fn/mean": -0.11875, "rewards/reward_fn/std": 0.4070206701755524, "sampling/importance_sampling_ratio/max": 1.3046109199523925, "sampling/importance_sampling_ratio/mean": 1.000963270664215, "sampling/importance_sampling_ratio/min": 0.7893401265144349, "sampling/sampling_logp_difference/max": 0.27059038877487185, "sampling/sampling_logp_difference/mean": 0.022490573488175868, "step": 5830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 193.2875, "completions/mean_terminated_length": 193.2875, "completions/min_length": 124.2, "completions/min_terminated_length": 124.2, "entropy": 1.058194574713707, "epoch": 0.5369127516778524, "frac_reward_zero_std": 0.0, "grad_norm": 8.039623260498047, "kl": 0.040199279098305854, "learning_rate": 4.776194615076325e-06, "loss": 0.002, "num_tokens": 25322894.0, "reward": 0.05, "reward_std": 0.5242056578397751, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5242056638002396, "sampling/importance_sampling_ratio/max": 1.240907084941864, "sampling/importance_sampling_ratio/mean": 0.9999734878540039, "sampling/importance_sampling_ratio/min": 0.7868142783641815, "sampling/sampling_logp_difference/max": 0.2528844833374023, "sampling/sampling_logp_difference/mean": 0.020994765870273113, "step": 5840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.0, "completions/max_terminated_length": 279.0, "completions/mean_length": 196.575, "completions/mean_terminated_length": 196.575, "completions/min_length": 118.5, "completions/min_terminated_length": 118.5, "entropy": 1.0126894980669021, "epoch": 0.5378321228279856, "frac_reward_zero_std": 0.2, "grad_norm": 6.705118656158447, "kl": 0.044608468236401674, "learning_rate": 4.7751456461752545e-06, "loss": 0.0022, "num_tokens": 25364932.0, "reward": 0.4625, "reward_std": 0.3590047687292099, "rewards/reward_fn/mean": 0.4625, "rewards/reward_fn/std": 0.3590047687292099, "sampling/importance_sampling_ratio/max": 1.2606943249702454, "sampling/importance_sampling_ratio/mean": 0.9998015284538269, "sampling/importance_sampling_ratio/min": 0.767030942440033, "sampling/sampling_logp_difference/max": 0.27470220923423766, "sampling/sampling_logp_difference/mean": 0.021057704649865627, "step": 5850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.5, "completions/max_terminated_length": 281.5, "completions/mean_length": 201.9375, "completions/mean_terminated_length": 201.9375, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 0.986550921201706, "epoch": 0.538751493978119, "frac_reward_zero_std": 0.2, "grad_norm": 5.23849630355835, "kl": 0.04298890708014369, "learning_rate": 4.7740943405172305e-06, "loss": 0.0019, "num_tokens": 25409879.0, "reward": 0.24375, "reward_std": 0.4256551146507263, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.42565513253211973, "sampling/importance_sampling_ratio/max": 1.2425489902496338, "sampling/importance_sampling_ratio/mean": 1.0001308917999268, "sampling/importance_sampling_ratio/min": 0.7767520606517792, "sampling/sampling_logp_difference/max": 0.26804039478302, "sampling/sampling_logp_difference/mean": 0.020688995160162447, "step": 5860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 204.3375, "completions/mean_terminated_length": 204.3375, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.040500393509865, "epoch": 0.5396708651282522, "frac_reward_zero_std": 0.1, "grad_norm": 6.9332075119018555, "kl": 0.032301264436682686, "learning_rate": 4.773040699182028e-06, "loss": 0.0013, "num_tokens": 25452218.0, "reward": -0.1875, "reward_std": 0.4052957743406296, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.4052957743406296, "sampling/importance_sampling_ratio/max": 1.2287989854812622, "sampling/importance_sampling_ratio/mean": 0.9999854445457459, "sampling/importance_sampling_ratio/min": 0.7783948719501496, "sampling/sampling_logp_difference/max": 0.258540940284729, "sampling/sampling_logp_difference/mean": 0.02116065640002489, "step": 5870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 193.1875, "completions/mean_terminated_length": 193.1875, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0589107371866704, "epoch": 0.5405902362783855, "frac_reward_zero_std": 0.2, "grad_norm": 0.18212489783763885, "kl": 0.03585461411857978, "learning_rate": 4.77198472325182e-06, "loss": 0.0022, "num_tokens": 25491017.0, "reward": 0.18125, "reward_std": 0.42215809524059295, "rewards/reward_fn/mean": 0.18125, "rewards/reward_fn/std": 0.4221581071615219, "sampling/importance_sampling_ratio/max": 1.2426268339157105, "sampling/importance_sampling_ratio/mean": 1.0000000834465026, "sampling/importance_sampling_ratio/min": 0.776694244146347, "sampling/sampling_logp_difference/max": 0.26067568063735963, "sampling/sampling_logp_difference/mean": 0.02216607313603163, "step": 5880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.6, "completions/max_terminated_length": 256.6, "completions/mean_length": 188.675, "completions/mean_terminated_length": 188.675, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.1061671853065491, "epoch": 0.541509607428519, "frac_reward_zero_std": 0.3, "grad_norm": 4.681638717651367, "kl": 0.034587679780088365, "learning_rate": 4.77092641381118e-06, "loss": 0.0019, "num_tokens": 25531975.0, "reward": 0.325, "reward_std": 0.3519521474838257, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.35195215344429015, "sampling/importance_sampling_ratio/max": 1.2652634382247925, "sampling/importance_sampling_ratio/mean": 0.9998222529888153, "sampling/importance_sampling_ratio/min": 0.7833308219909668, "sampling/sampling_logp_difference/max": 0.29273965358734133, "sampling/sampling_logp_difference/mean": 0.02201755326241255, "step": 5890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 193.3625, "completions/mean_terminated_length": 193.3625, "completions/min_length": 123.4, "completions/min_terminated_length": 123.4, "entropy": 1.048141146451235, "epoch": 0.5424289785786522, "frac_reward_zero_std": 0.0, "grad_norm": 7.654055595397949, "kl": 0.05220045290188864, "learning_rate": 4.769865771947076e-06, "loss": 0.0026, "num_tokens": 25571260.0, "reward": 0.0375, "reward_std": 0.5707158267498016, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.5707158446311951, "sampling/importance_sampling_ratio/max": 1.2761346578598023, "sampling/importance_sampling_ratio/mean": 0.9999378561973572, "sampling/importance_sampling_ratio/min": 0.7819115936756134, "sampling/sampling_logp_difference/max": 0.26040356755256655, "sampling/sampling_logp_difference/mean": 0.021015533432364463, "step": 5900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.8, "completions/max_terminated_length": 299.8, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 125.8, "completions/min_terminated_length": 125.8, "entropy": 1.1307473748922348, "epoch": 0.5433483497287855, "frac_reward_zero_std": 0.2, "grad_norm": 0.13110964000225067, "kl": 0.030620167835149915, "learning_rate": 4.768802798748871e-06, "loss": 0.0011, "num_tokens": 25615010.0, "reward": -0.0125, "reward_std": 0.44991534650325776, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4499153643846512, "sampling/importance_sampling_ratio/max": 1.286659586429596, "sampling/importance_sampling_ratio/mean": 0.9997970759868622, "sampling/importance_sampling_ratio/min": 0.7661869466304779, "sampling/sampling_logp_difference/max": 0.280324125289917, "sampling/sampling_logp_difference/mean": 0.02235572077333927, "step": 5910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 212.6125, "completions/mean_terminated_length": 212.6125, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.119990562647581, "epoch": 0.5442677208789188, "frac_reward_zero_std": 0.4, "grad_norm": 0.11504147201776505, "kl": 0.05544349694391713, "learning_rate": 4.767737495308327e-06, "loss": 0.0025, "num_tokens": 25658131.0, "reward": -0.25, "reward_std": 0.2903633385896683, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.2903633564710617, "sampling/importance_sampling_ratio/max": 1.2705029726028443, "sampling/importance_sampling_ratio/mean": 1.0001543819904328, "sampling/importance_sampling_ratio/min": 0.7840491950511932, "sampling/sampling_logp_difference/max": 0.259272837638855, "sampling/sampling_logp_difference/mean": 0.021829812228679656, "step": 5920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.3, "completions/max_terminated_length": 300.2, "completions/mean_length": 218.7, "completions/mean_terminated_length": 216.0464294433594, "completions/min_length": 148.3, "completions/min_terminated_length": 148.3, "entropy": 1.1391092754900456, "epoch": 0.5451870920290521, "frac_reward_zero_std": 0.2, "grad_norm": 5.687870979309082, "kl": 0.049077473138459025, "learning_rate": 4.766669862719593e-06, "loss": 0.0029, "num_tokens": 25705059.0, "reward": -0.01875, "reward_std": 0.5362760663032532, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.5362760841846466, "sampling/importance_sampling_ratio/max": 1.3044365406036378, "sampling/importance_sampling_ratio/mean": 0.9996832191944123, "sampling/importance_sampling_ratio/min": 0.7870841383934021, "sampling/sampling_logp_difference/max": 0.2838757514953613, "sampling/sampling_logp_difference/mean": 0.021945148706436157, "step": 5930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 364.0, "completions/max_terminated_length": 364.0, "completions/mean_length": 236.55, "completions/mean_terminated_length": 236.55, "completions/min_length": 157.8, "completions/min_terminated_length": 157.8, "entropy": 1.0853421695530414, "epoch": 0.5461064631791854, "frac_reward_zero_std": 0.1, "grad_norm": 5.322802543640137, "kl": 0.043733903742395344, "learning_rate": 4.765599902079214e-06, "loss": 0.0025, "num_tokens": 25752031.0, "reward": 0.15625, "reward_std": 0.5031163722276688, "rewards/reward_fn/mean": 0.15625, "rewards/reward_fn/std": 0.5031163841485977, "sampling/importance_sampling_ratio/max": 1.2658527612686157, "sampling/importance_sampling_ratio/mean": 1.000313937664032, "sampling/importance_sampling_ratio/min": 0.7771980404853821, "sampling/sampling_logp_difference/max": 0.2732072353363037, "sampling/sampling_logp_difference/mean": 0.02070043422281742, "step": 5940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 215.4625, "completions/mean_terminated_length": 215.4625, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.0143737901002168, "epoch": 0.5470258343293187, "frac_reward_zero_std": 0.2, "grad_norm": 0.27232256531715393, "kl": 0.044538831780664624, "learning_rate": 4.764527614486125e-06, "loss": 0.0024, "num_tokens": 25794580.0, "reward": 0.0875, "reward_std": 0.4342138558626175, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.4342138677835464, "sampling/importance_sampling_ratio/max": 1.308932363986969, "sampling/importance_sampling_ratio/mean": 1.0002804160118104, "sampling/importance_sampling_ratio/min": 0.748006272315979, "sampling/sampling_logp_difference/max": 0.33436107635498047, "sampling/sampling_logp_difference/mean": 0.021697475761175155, "step": 5950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.9, "completions/max_terminated_length": 328.9, "completions/mean_length": 236.9125, "completions/mean_terminated_length": 236.9125, "completions/min_length": 176.9, "completions/min_terminated_length": 176.9, "entropy": 1.0702579751610757, "epoch": 0.547945205479452, "frac_reward_zero_std": 0.5, "grad_norm": 4.59385871887207, "kl": 0.04439568083034828, "learning_rate": 4.763453001041653e-06, "loss": 0.002, "num_tokens": 25842317.0, "reward": 0.1625, "reward_std": 0.2692759811878204, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.26927598714828493, "sampling/importance_sampling_ratio/max": 1.2691372513771058, "sampling/importance_sampling_ratio/mean": 1.0001939117908478, "sampling/importance_sampling_ratio/min": 0.7894082248210907, "sampling/sampling_logp_difference/max": 0.26476741433143614, "sampling/sampling_logp_difference/mean": 0.021000272035598753, "step": 5960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.5, "completions/max_terminated_length": 324.5, "completions/mean_length": 221.65, "completions/mean_terminated_length": 221.65, "completions/min_length": 158.4, "completions/min_terminated_length": 158.4, "entropy": 1.1069272071123124, "epoch": 0.5488645766295853, "frac_reward_zero_std": 0.3, "grad_norm": 6.975583076477051, "kl": 0.044007860706187785, "learning_rate": 4.762376062849511e-06, "loss": 0.0023, "num_tokens": 25888185.0, "reward": 0.05, "reward_std": 0.36538600325584414, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.36538602113723756, "sampling/importance_sampling_ratio/max": 1.2860293626785277, "sampling/importance_sampling_ratio/mean": 0.9999909639358521, "sampling/importance_sampling_ratio/min": 0.788635003566742, "sampling/sampling_logp_difference/max": 0.27637232542037965, "sampling/sampling_logp_difference/mean": 0.021719546616077424, "step": 5970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 211.5375, "completions/mean_terminated_length": 211.5375, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.1183372132480145, "epoch": 0.5497839477797186, "frac_reward_zero_std": 0.3, "grad_norm": 4.9230475425720215, "kl": 0.047760227014077825, "learning_rate": 4.7612968010158e-06, "loss": 0.0019, "num_tokens": 25932444.0, "reward": -0.1, "reward_std": 0.3452198475599289, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3452198475599289, "sampling/importance_sampling_ratio/max": 1.334183931350708, "sampling/importance_sampling_ratio/mean": 1.0001567363739015, "sampling/importance_sampling_ratio/min": 0.7768688678741456, "sampling/sampling_logp_difference/max": 0.2979376554489136, "sampling/sampling_logp_difference/mean": 0.02282101195305586, "step": 5980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.3, "completions/max_terminated_length": 284.3, "completions/mean_length": 213.45, "completions/mean_terminated_length": 213.45, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.019435753673315, "epoch": 0.550703318929852, "frac_reward_zero_std": 0.3, "grad_norm": 0.14074622094631195, "kl": 0.04074702991638333, "learning_rate": 4.760215216649009e-06, "loss": 0.002, "num_tokens": 25975888.0, "reward": 0.3125, "reward_std": 0.3818357288837433, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.38183574080467225, "sampling/importance_sampling_ratio/max": 1.224917483329773, "sampling/importance_sampling_ratio/mean": 1.0000465393066407, "sampling/importance_sampling_ratio/min": 0.7752545535564422, "sampling/sampling_logp_difference/max": 0.2612292289733887, "sampling/sampling_logp_difference/mean": 0.02056172639131546, "step": 5990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.1, "completions/max_terminated_length": 252.1, "completions/mean_length": 185.575, "completions/mean_terminated_length": 185.575, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 0.9685670718550682, "epoch": 0.5516226900799853, "frac_reward_zero_std": 0.5, "grad_norm": 0.153370663523674, "kl": 0.03268846122955438, "learning_rate": 4.759131310860012e-06, "loss": 0.0016, "num_tokens": 26018790.0, "reward": 0.275, "reward_std": 0.24315897822380067, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.2431589901447296, "sampling/importance_sampling_ratio/max": 1.2644233584403992, "sampling/importance_sampling_ratio/mean": 1.0000719547271728, "sampling/importance_sampling_ratio/min": 0.7764980137348175, "sampling/sampling_logp_difference/max": 0.27544463872909547, "sampling/sampling_logp_difference/mean": 0.0197725972160697, "step": 6000 }, { "epoch": 0.5516226900799853, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 289.760907504363, "eval_completions/max_terminated_length": 287.95113438045377, "eval_completions/mean_length": 203.84293193717278, "eval_completions/mean_terminated_length": 203.48245460283903, "eval_completions/min_length": 133.26352530541013, "eval_completions/min_terminated_length": 133.26352530541013, "eval_entropy": 1.0827112896384161, "eval_frac_reward_zero_std": 0.28272251308900526, "eval_kl": 0.043951813903609596, "eval_loss": 0.0022731476929038763, "eval_num_tokens": 26018790.0, "eval_reward": 0.04635689354275742, "eval_reward_std": 0.36116907010943894, "eval_rewards/reward_fn/mean": 0.04635689354275742, "eval_rewards/reward_fn/std": 0.3611690798875133, "eval_runtime": 1697.1728, "eval_samples_per_second": 0.338, "eval_sampling/importance_sampling_ratio/max": 1.2731469723982753, "eval_sampling/importance_sampling_ratio/mean": 0.9999923838162713, "eval_sampling/importance_sampling_ratio/min": 0.7796319361132477, "eval_sampling/sampling_logp_difference/max": 0.278133765305108, "eval_sampling/sampling_logp_difference/mean": 0.020720613938325572, "eval_steps_per_second": 0.042, "step": 6000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.3, "completions/max_terminated_length": 307.3, "completions/mean_length": 215.5375, "completions/mean_terminated_length": 215.5375, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.100583217293024, "epoch": 0.5525420612301186, "frac_reward_zero_std": 0.6, "grad_norm": 0.16121993958950043, "kl": 0.04055323851061985, "learning_rate": 4.758045084762068e-06, "loss": 0.0019, "num_tokens": 26065633.0, "reward": -0.175, "reward_std": 0.1940147489309311, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.19401475489139558, "sampling/importance_sampling_ratio/max": 1.2697038888931274, "sampling/importance_sampling_ratio/mean": 0.9998781323432923, "sampling/importance_sampling_ratio/min": 0.7531592965126037, "sampling/sampling_logp_difference/max": 0.32699408531188967, "sampling/sampling_logp_difference/mean": 0.021225535683333872, "step": 6010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 288.8, "completions/max_terminated_length": 267.5, "completions/mean_length": 195.2, "completions/mean_terminated_length": 190.71428680419922, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 0.9775846891105175, "epoch": 0.5534614323802519, "frac_reward_zero_std": 0.4, "grad_norm": 7.146791458129883, "kl": 0.045046556543093176, "learning_rate": 4.756956539470817e-06, "loss": 0.0019, "num_tokens": 26112681.0, "reward": 0.31875, "reward_std": 0.2907280743122101, "rewards/reward_fn/mean": 0.31875, "rewards/reward_fn/std": 0.29072808027267455, "sampling/importance_sampling_ratio/max": 1.2930991411209107, "sampling/importance_sampling_ratio/mean": 1.0000125825405122, "sampling/importance_sampling_ratio/min": 0.7531468898057938, "sampling/sampling_logp_difference/max": 0.34712321758270265, "sampling/sampling_logp_difference/mean": 0.01998790940269828, "step": 6020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.4, "completions/max_terminated_length": 287.7, "completions/mean_length": 215.7125, "completions/mean_terminated_length": 212.2250015258789, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.1139815695583821, "epoch": 0.5543808035303852, "frac_reward_zero_std": 0.3, "grad_norm": 0.11589720845222473, "kl": 0.036113910312997176, "learning_rate": 4.755865676104283e-06, "loss": 0.002, "num_tokens": 26153482.0, "reward": 0.3125, "reward_std": 0.3590047687292099, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.3590047687292099, "sampling/importance_sampling_ratio/max": 1.2823933959007263, "sampling/importance_sampling_ratio/mean": 1.0002814829349518, "sampling/importance_sampling_ratio/min": 0.7840506315231324, "sampling/sampling_logp_difference/max": 0.28293814659118655, "sampling/sampling_logp_difference/mean": 0.02164357490837574, "step": 6030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.4, "completions/max_terminated_length": 326.4, "completions/mean_length": 206.5625, "completions/mean_terminated_length": 206.5625, "completions/min_length": 112.9, "completions/min_terminated_length": 112.9, "entropy": 1.0558997258543967, "epoch": 0.5553001746805185, "frac_reward_zero_std": 0.2, "grad_norm": 7.15067720413208, "kl": 0.045980360300745815, "learning_rate": 4.75477249578287e-06, "loss": 0.0022, "num_tokens": 26197183.0, "reward": -0.0375, "reward_std": 0.32997219264507294, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.32997219264507294, "sampling/importance_sampling_ratio/max": 1.3412793040275575, "sampling/importance_sampling_ratio/mean": 1.0000888228416442, "sampling/importance_sampling_ratio/min": 0.771163946390152, "sampling/sampling_logp_difference/max": 0.305698561668396, "sampling/sampling_logp_difference/mean": 0.020581517927348615, "step": 6040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.2, "completions/max_terminated_length": 316.2, "completions/mean_length": 224.225, "completions/mean_terminated_length": 224.225, "completions/min_length": 141.7, "completions/min_terminated_length": 141.7, "entropy": 1.062789673358202, "epoch": 0.5562195458306518, "frac_reward_zero_std": 0.3, "grad_norm": 6.231579780578613, "kl": 0.040888736740453166, "learning_rate": 4.753676999629362e-06, "loss": 0.0022, "num_tokens": 26239833.0, "reward": 0.0875, "reward_std": 0.36674020886421205, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.3667402148246765, "sampling/importance_sampling_ratio/max": 1.2890511751174927, "sampling/importance_sampling_ratio/mean": 1.0007209539413453, "sampling/importance_sampling_ratio/min": 0.774627935886383, "sampling/sampling_logp_difference/max": 0.27654070854187013, "sampling/sampling_logp_difference/mean": 0.021461549401283263, "step": 6050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 218.2375, "completions/mean_terminated_length": 218.2375, "completions/min_length": 145.4, "completions/min_terminated_length": 145.4, "entropy": 1.1033702749758958, "epoch": 0.5571389169807851, "frac_reward_zero_std": 0.3, "grad_norm": 6.480884075164795, "kl": 0.05576675818301737, "learning_rate": 4.752579188768921e-06, "loss": 0.0033, "num_tokens": 26283268.0, "reward": -0.075, "reward_std": 0.3819254577159882, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.38192546367645264, "sampling/importance_sampling_ratio/max": 1.2925401687622071, "sampling/importance_sampling_ratio/mean": 0.9996954083442688, "sampling/importance_sampling_ratio/min": 0.782156103849411, "sampling/sampling_logp_difference/max": 0.290268075466156, "sampling/sampling_logp_difference/mean": 0.021014150511473418, "step": 6060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.5, "completions/max_terminated_length": 273.5, "completions/mean_length": 199.4375, "completions/mean_terminated_length": 199.4375, "completions/min_length": 140.6, "completions/min_terminated_length": 140.6, "entropy": 1.0867134407162666, "epoch": 0.5580582881309184, "frac_reward_zero_std": 0.3, "grad_norm": 6.266357421875, "kl": 0.05641554001485929, "learning_rate": 4.7514790643290855e-06, "loss": 0.003, "num_tokens": 26325543.0, "reward": -0.1625, "reward_std": 0.31855743527412417, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.3185574531555176, "sampling/importance_sampling_ratio/max": 1.266220796108246, "sampling/importance_sampling_ratio/mean": 1.0006169736385346, "sampling/importance_sampling_ratio/min": 0.7809510588645935, "sampling/sampling_logp_difference/max": 0.2727139115333557, "sampling/sampling_logp_difference/mean": 0.021806595474481584, "step": 6070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.5, "completions/max_terminated_length": 283.5, "completions/mean_length": 207.3875, "completions/mean_terminated_length": 207.3875, "completions/min_length": 134.7, "completions/min_terminated_length": 134.7, "entropy": 1.0787999033927917, "epoch": 0.5589776592810518, "frac_reward_zero_std": 0.4, "grad_norm": 5.612980842590332, "kl": 0.049633268325123936, "learning_rate": 4.750376627439774e-06, "loss": 0.0024, "num_tokens": 26367782.0, "reward": 0.05, "reward_std": 0.2903633385896683, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.2903633564710617, "sampling/importance_sampling_ratio/max": 1.3411411046981812, "sampling/importance_sampling_ratio/mean": 0.9998104274272919, "sampling/importance_sampling_ratio/min": 0.7889857649803161, "sampling/sampling_logp_difference/max": 0.3039317846298218, "sampling/sampling_logp_difference/mean": 0.021997787430882453, "step": 6080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.1, "completions/max_terminated_length": 298.1, "completions/mean_length": 204.325, "completions/mean_terminated_length": 204.325, "completions/min_length": 134.2, "completions/min_terminated_length": 134.2, "entropy": 1.0672012146562337, "epoch": 0.5598970304311851, "frac_reward_zero_std": 0.3, "grad_norm": 7.694453716278076, "kl": 0.0408846702426672, "learning_rate": 4.749271879233276e-06, "loss": 0.0023, "num_tokens": 26413360.0, "reward": 0.0, "reward_std": 0.3038551896810532, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.30385519564151764, "sampling/importance_sampling_ratio/max": 1.2742053270339966, "sampling/importance_sampling_ratio/mean": 0.9997616529464721, "sampling/importance_sampling_ratio/min": 0.7759513616561889, "sampling/sampling_logp_difference/max": 0.2905306339263916, "sampling/sampling_logp_difference/mean": 0.021338155213743448, "step": 6090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.7, "completions/max_terminated_length": 298.7, "completions/mean_length": 215.9625, "completions/mean_terminated_length": 215.9625, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.067601116001606, "epoch": 0.5608164015813184, "frac_reward_zero_std": 0.1, "grad_norm": 5.920677185058594, "kl": 0.04829503845539875, "learning_rate": 4.748164820844257e-06, "loss": 0.0018, "num_tokens": 26458293.0, "reward": -0.15, "reward_std": 0.5433805823326111, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.54338059425354, "sampling/importance_sampling_ratio/max": 1.2721220016479493, "sampling/importance_sampling_ratio/mean": 0.9997223973274231, "sampling/importance_sampling_ratio/min": 0.7419908940792084, "sampling/sampling_logp_difference/max": 0.31930768489837646, "sampling/sampling_logp_difference/mean": 0.021749875135719778, "step": 6100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.2, "completions/max_terminated_length": 310.2, "completions/mean_length": 218.9125, "completions/mean_terminated_length": 218.9125, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1123768582940101, "epoch": 0.5617357727314517, "frac_reward_zero_std": 0.3, "grad_norm": 0.11977187544107437, "kl": 0.047653047821950165, "learning_rate": 4.747055453409755e-06, "loss": 0.0023, "num_tokens": 26506430.0, "reward": -0.0625, "reward_std": 0.3492635846138, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.3492635846138, "sampling/importance_sampling_ratio/max": 1.3172544240951538, "sampling/importance_sampling_ratio/mean": 0.999568372964859, "sampling/importance_sampling_ratio/min": 0.7611725032329559, "sampling/sampling_logp_difference/max": 0.32140268087387086, "sampling/sampling_logp_difference/mean": 0.022083052061498167, "step": 6110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.7, "completions/max_terminated_length": 319.7, "completions/mean_length": 219.975, "completions/mean_terminated_length": 219.975, "completions/min_length": 154.1, "completions/min_terminated_length": 154.1, "entropy": 1.0652639500796794, "epoch": 0.562655143881585, "frac_reward_zero_std": 0.1, "grad_norm": 0.11650632321834564, "kl": 0.04516810227651149, "learning_rate": 4.745943778069177e-06, "loss": 0.0023, "num_tokens": 26554532.0, "reward": 0.0, "reward_std": 0.5017905324697495, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.5017905324697495, "sampling/importance_sampling_ratio/max": 1.2615557789802552, "sampling/importance_sampling_ratio/mean": 1.000525599718094, "sampling/importance_sampling_ratio/min": 0.7610154628753663, "sampling/sampling_logp_difference/max": 0.29589412212371824, "sampling/sampling_logp_difference/mean": 0.02028226088732481, "step": 6120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 351.7, "completions/max_terminated_length": 323.5, "completions/mean_length": 223.1375, "completions/mean_terminated_length": 219.0464294433594, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.1002894632518292, "epoch": 0.5635745150317183, "frac_reward_zero_std": 0.5, "grad_norm": 4.478103160858154, "kl": 0.04988232392934151, "learning_rate": 4.744829795964306e-06, "loss": 0.0024, "num_tokens": 26600975.0, "reward": 0.24375, "reward_std": 0.2298406779766083, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.2298406779766083, "sampling/importance_sampling_ratio/max": 1.245114529132843, "sampling/importance_sampling_ratio/mean": 0.9993971467018128, "sampling/importance_sampling_ratio/min": 0.7815256774425506, "sampling/sampling_logp_difference/max": 0.2746048092842102, "sampling/sampling_logp_difference/mean": 0.021487167477607726, "step": 6130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.5, "completions/max_terminated_length": 274.5, "completions/mean_length": 190.6125, "completions/mean_terminated_length": 190.6125, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.1239974051713943, "epoch": 0.5644938861818516, "frac_reward_zero_std": 0.4, "grad_norm": 0.1537046581506729, "kl": 0.04689802597276867, "learning_rate": 4.74371350823929e-06, "loss": 0.0019, "num_tokens": 26643072.0, "reward": 0.05, "reward_std": 0.3138213872909546, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.31382139325141906, "sampling/importance_sampling_ratio/max": 1.242549991607666, "sampling/importance_sampling_ratio/mean": 0.9996675312519073, "sampling/importance_sampling_ratio/min": 0.7829691529273987, "sampling/sampling_logp_difference/max": 0.25601160526275635, "sampling/sampling_logp_difference/mean": 0.021785255521535873, "step": 6140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 338.5, "completions/max_terminated_length": 325.9, "completions/mean_length": 231.1875, "completions/mean_terminated_length": 227.95892944335938, "completions/min_length": 154.4, "completions/min_terminated_length": 154.4, "entropy": 1.1363309763371945, "epoch": 0.5654132573319849, "frac_reward_zero_std": 0.1, "grad_norm": 7.242461681365967, "kl": 0.04323768197791651, "learning_rate": 4.742594916040644e-06, "loss": 0.0024, "num_tokens": 26688415.0, "reward": 0.24375, "reward_std": 0.4626706510782242, "rewards/reward_fn/mean": 0.24375, "rewards/reward_fn/std": 0.46267067492008207, "sampling/importance_sampling_ratio/max": 1.314042055606842, "sampling/importance_sampling_ratio/mean": 1.0001779854297639, "sampling/importance_sampling_ratio/min": 0.7805741727352142, "sampling/sampling_logp_difference/max": 0.2798584461212158, "sampling/sampling_logp_difference/mean": 0.021532920375466346, "step": 6150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.7, "completions/max_terminated_length": 288.7, "completions/mean_length": 209.875, "completions/mean_terminated_length": 209.875, "completions/min_length": 131.8, "completions/min_terminated_length": 131.8, "entropy": 1.0838081322610378, "epoch": 0.5663326284821182, "frac_reward_zero_std": 0.5, "grad_norm": 0.12752386927604675, "kl": 0.03578949624206871, "learning_rate": 4.741474020517254e-06, "loss": 0.0014, "num_tokens": 26732805.0, "reward": 0.1375, "reward_std": 0.21504760384559632, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.2150476098060608, "sampling/importance_sampling_ratio/max": 1.2556844830513, "sampling/importance_sampling_ratio/mean": 1.0003502786159515, "sampling/importance_sampling_ratio/min": 0.7910115361213684, "sampling/sampling_logp_difference/max": 0.2553782343864441, "sampling/sampling_logp_difference/mean": 0.021159238554537295, "step": 6160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.3, "completions/max_terminated_length": 322.3, "completions/mean_length": 209.4375, "completions/mean_terminated_length": 209.4375, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 1.0653237245976925, "epoch": 0.5672519996322516, "frac_reward_zero_std": 0.3, "grad_norm": 6.03389310836792, "kl": 0.04591074122581631, "learning_rate": 4.7403508228203684e-06, "loss": 0.0022, "num_tokens": 26777632.0, "reward": 0.0375, "reward_std": 0.29291952252388, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.2929195284843445, "sampling/importance_sampling_ratio/max": 1.2589402079582215, "sampling/importance_sampling_ratio/mean": 1.0000591278076172, "sampling/importance_sampling_ratio/min": 0.7985383152961731, "sampling/sampling_logp_difference/max": 0.2497469425201416, "sampling/sampling_logp_difference/mean": 0.021471216157078744, "step": 6170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.0, "completions/max_terminated_length": 281.0, "completions/mean_length": 210.5625, "completions/mean_terminated_length": 210.5625, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.0731035716831685, "epoch": 0.5681713707823849, "frac_reward_zero_std": 0.4, "grad_norm": 5.894872665405273, "kl": 0.05273802954470739, "learning_rate": 4.739225324103601e-06, "loss": 0.0026, "num_tokens": 26822989.0, "reward": -0.15, "reward_std": 0.28029437065124513, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.28029437065124513, "sampling/importance_sampling_ratio/max": 1.2640633821487426, "sampling/importance_sampling_ratio/mean": 1.0004974901676178, "sampling/importance_sampling_ratio/min": 0.7798752546310425, "sampling/sampling_logp_difference/max": 0.2633428066968918, "sampling/sampling_logp_difference/mean": 0.020562943629920484, "step": 6180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.5, "completions/max_terminated_length": 333.5, "completions/mean_length": 240.5125, "completions/mean_terminated_length": 240.5125, "completions/min_length": 169.9, "completions/min_terminated_length": 169.9, "entropy": 1.1044846460223199, "epoch": 0.5690907419325182, "frac_reward_zero_std": 0.2, "grad_norm": 6.202932357788086, "kl": 0.05687112370505929, "learning_rate": 4.738097525522931e-06, "loss": 0.0028, "num_tokens": 26870166.0, "reward": -0.15, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2937981963157654, "sampling/importance_sampling_ratio/mean": 0.9995344758033753, "sampling/importance_sampling_ratio/min": 0.7897027254104614, "sampling/sampling_logp_difference/max": 0.27732162475585936, "sampling/sampling_logp_difference/mean": 0.0214335422962904, "step": 6190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 290.2, "completions/max_terminated_length": 278.9, "completions/mean_length": 216.0, "completions/mean_terminated_length": 213.3589294433594, "completions/min_length": 145.2, "completions/min_terminated_length": 145.2, "entropy": 1.1585246749222278, "epoch": 0.5700101130826515, "frac_reward_zero_std": 0.2, "grad_norm": 5.032841682434082, "kl": 0.05120847553480416, "learning_rate": 4.736967428236697e-06, "loss": 0.0027, "num_tokens": 26914574.0, "reward": -0.09375, "reward_std": 0.4978039592504501, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.4978039711713791, "sampling/importance_sampling_ratio/max": 1.2411821842193604, "sampling/importance_sampling_ratio/mean": 0.9998735904693603, "sampling/importance_sampling_ratio/min": 0.7408751308917999, "sampling/sampling_logp_difference/max": 0.3060741126537323, "sampling/sampling_logp_difference/mean": 0.022881741262972356, "step": 6200 }, { "epoch": 0.5700101130826515, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0004363001745200698, "eval_completions/max_length": 294.5986038394415, "eval_completions/max_terminated_length": 293.81849912739966, "eval_completions/mean_length": 210.05388307155323, "eval_completions/mean_terminated_length": 209.9302231820258, "eval_completions/min_length": 139.92321116928446, "eval_completions/min_terminated_length": 139.92321116928446, "eval_entropy": 1.1115677457101265, "eval_frac_reward_zero_std": 0.3106457242582897, "eval_kl": 0.04608455646041956, "eval_loss": 0.0023816365282982588, "eval_num_tokens": 26914574.0, "eval_reward": 0.0506108202443281, "eval_reward_std": 0.33980271776725274, "eval_rewards/reward_fn/mean": 0.0506108202443281, "eval_rewards/reward_fn/std": 0.33980272723326094, "eval_runtime": 1722.0475, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2645804840440733, "eval_sampling/importance_sampling_ratio/mean": 0.9999712471562531, "eval_sampling/importance_sampling_ratio/min": 0.7805823519383842, "eval_sampling/sampling_logp_difference/max": 0.27004438565871686, "eval_sampling/sampling_logp_difference/mean": 0.021134188845688656, "eval_steps_per_second": 0.042, "step": 6200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.9, "completions/max_terminated_length": 292.7, "completions/mean_length": 208.225, "completions/mean_terminated_length": 204.40178680419922, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 0.9808410197496414, "epoch": 0.5709294842327848, "frac_reward_zero_std": 0.2, "grad_norm": 6.011927604675293, "kl": 0.04370331569225527, "learning_rate": 4.735835033405598e-06, "loss": 0.0019, "num_tokens": 26956656.0, "reward": 0.28125, "reward_std": 0.36870584785938265, "rewards/reward_fn/mean": 0.28125, "rewards/reward_fn/std": 0.36870585978031156, "sampling/importance_sampling_ratio/max": 1.2446932911872863, "sampling/importance_sampling_ratio/mean": 1.0005206286907196, "sampling/importance_sampling_ratio/min": 0.8059887111186981, "sampling/sampling_logp_difference/max": 0.23095146417617798, "sampling/sampling_logp_difference/mean": 0.019828132819384336, "step": 6210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.7, "completions/max_terminated_length": 288.7, "completions/mean_length": 205.725, "completions/mean_terminated_length": 205.725, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.1120463073253632, "epoch": 0.5718488553829181, "frac_reward_zero_std": 0.3, "grad_norm": 8.357625007629395, "kl": 0.05083461130270735, "learning_rate": 4.734700342192697e-06, "loss": 0.0028, "num_tokens": 26999874.0, "reward": -0.075, "reward_std": 0.3110164314508438, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3110164374113083, "sampling/importance_sampling_ratio/max": 1.317823576927185, "sampling/importance_sampling_ratio/mean": 1.0000887751579284, "sampling/importance_sampling_ratio/min": 0.7803912162780762, "sampling/sampling_logp_difference/max": 0.2962776005268097, "sampling/sampling_logp_difference/mean": 0.02169969454407692, "step": 6220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.9, "completions/max_terminated_length": 300.9, "completions/mean_length": 212.7125, "completions/mean_terminated_length": 212.7125, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.1087385043501854, "epoch": 0.5727682265330514, "frac_reward_zero_std": 0.4, "grad_norm": 0.1177276223897934, "kl": 0.036686048936098815, "learning_rate": 4.7335633557634105e-06, "loss": 0.0019, "num_tokens": 27042963.0, "reward": 0.1, "reward_std": 0.4406357705593109, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.4406357824802399, "sampling/importance_sampling_ratio/max": 1.2545434355735778, "sampling/importance_sampling_ratio/mean": 0.9999705791473389, "sampling/importance_sampling_ratio/min": 0.7710280060768128, "sampling/sampling_logp_difference/max": 0.283068859577179, "sampling/sampling_logp_difference/mean": 0.02119218297302723, "step": 6230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 356.4, "completions/max_terminated_length": 343.8, "completions/mean_length": 251.175, "completions/mean_terminated_length": 248.21785888671874, "completions/min_length": 138.1, "completions/min_terminated_length": 138.1, "entropy": 1.1300581611692906, "epoch": 0.5736875976831847, "frac_reward_zero_std": 0.5, "grad_norm": 0.12600478529930115, "kl": 0.053841043740976605, "learning_rate": 4.732424075285517e-06, "loss": 0.0025, "num_tokens": 27092705.0, "reward": 0.06875, "reward_std": 0.27494093477725984, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.2749409407377243, "sampling/importance_sampling_ratio/max": 1.2650659203529357, "sampling/importance_sampling_ratio/mean": 0.9998580515384674, "sampling/importance_sampling_ratio/min": 0.7679423093795776, "sampling/sampling_logp_difference/max": 0.2720667362213135, "sampling/sampling_logp_difference/mean": 0.02121734358370304, "step": 6240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.6, "completions/max_terminated_length": 326.6, "completions/mean_length": 227.65, "completions/mean_terminated_length": 227.65, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.1459137976169587, "epoch": 0.574606968833318, "frac_reward_zero_std": 0.3, "grad_norm": 0.1364678293466568, "kl": 0.0408228462212719, "learning_rate": 4.731282501929145e-06, "loss": 0.0022, "num_tokens": 27138957.0, "reward": 0.09375, "reward_std": 0.34964423179626464, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.3496442437171936, "sampling/importance_sampling_ratio/max": 1.2421517610549926, "sampling/importance_sampling_ratio/mean": 0.9999367952346802, "sampling/importance_sampling_ratio/min": 0.7771855890750885, "sampling/sampling_logp_difference/max": 0.2627410411834717, "sampling/sampling_logp_difference/mean": 0.02135483454912901, "step": 6250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.4, "completions/max_terminated_length": 285.4, "completions/mean_length": 215.1125, "completions/mean_terminated_length": 215.1125, "completions/min_length": 141.2, "completions/min_terminated_length": 141.2, "entropy": 1.0933591924607753, "epoch": 0.5755263399834514, "frac_reward_zero_std": 0.3, "grad_norm": 6.0477614402771, "kl": 0.05091453074710443, "learning_rate": 4.730138636866786e-06, "loss": 0.0026, "num_tokens": 27184582.0, "reward": 0.04375, "reward_std": 0.3333195328712463, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.33331955075263975, "sampling/importance_sampling_ratio/max": 1.2658711433410645, "sampling/importance_sampling_ratio/mean": 1.000095695257187, "sampling/importance_sampling_ratio/min": 0.7742916345596313, "sampling/sampling_logp_difference/max": 0.27751768827438356, "sampling/sampling_logp_difference/mean": 0.02102894466370344, "step": 6260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 192.5875, "completions/mean_terminated_length": 192.5875, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0417233377695083, "epoch": 0.5764457111335847, "frac_reward_zero_std": 0.4, "grad_norm": 6.473278522491455, "kl": 0.04627795366104692, "learning_rate": 4.728992481273279e-06, "loss": 0.0024, "num_tokens": 27225621.0, "reward": 0.3125, "reward_std": 0.33088951706886294, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.33088952898979185, "sampling/importance_sampling_ratio/max": 1.2925965547561646, "sampling/importance_sampling_ratio/mean": 0.9998872399330139, "sampling/importance_sampling_ratio/min": 0.7827262163162232, "sampling/sampling_logp_difference/max": 0.29452760219573976, "sampling/sampling_logp_difference/mean": 0.02098462376743555, "step": 6270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 202.0, "completions/mean_terminated_length": 202.0, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0964640878140925, "epoch": 0.577365082283718, "frac_reward_zero_std": 0.4, "grad_norm": 0.1317955106496811, "kl": 0.042411452380474654, "learning_rate": 4.727844036325819e-06, "loss": 0.0018, "num_tokens": 27269565.0, "reward": 0.0375, "reward_std": 0.2912300378084183, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.2912300378084183, "sampling/importance_sampling_ratio/max": 1.278913974761963, "sampling/importance_sampling_ratio/mean": 0.9997564315795898, "sampling/importance_sampling_ratio/min": 0.7810963153839111, "sampling/sampling_logp_difference/max": 0.282138466835022, "sampling/sampling_logp_difference/mean": 0.021411254070699215, "step": 6280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.8, "completions/max_terminated_length": 268.8, "completions/mean_length": 194.3625, "completions/mean_terminated_length": 194.3625, "completions/min_length": 117.8, "completions/min_terminated_length": 117.8, "entropy": 1.047000838816166, "epoch": 0.5782844534338513, "frac_reward_zero_std": 0.3, "grad_norm": 5.521574020385742, "kl": 0.047210085089318454, "learning_rate": 4.726693303203949e-06, "loss": 0.0023, "num_tokens": 27313458.0, "reward": 0.0625, "reward_std": 0.36765356361866, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.36765357553958894, "sampling/importance_sampling_ratio/max": 1.2391393065452576, "sampling/importance_sampling_ratio/mean": 0.9995747983455658, "sampling/importance_sampling_ratio/min": 0.7767157316207886, "sampling/sampling_logp_difference/max": 0.2723180651664734, "sampling/sampling_logp_difference/mean": 0.020543034188449383, "step": 6290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 263.6, "completions/max_terminated_length": 263.6, "completions/mean_length": 193.4, "completions/mean_terminated_length": 193.4, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 1.1493043258786202, "epoch": 0.5792038245839846, "frac_reward_zero_std": 0.2, "grad_norm": 5.613497257232666, "kl": 0.049664326454512776, "learning_rate": 4.725540283089566e-06, "loss": 0.0022, "num_tokens": 27356082.0, "reward": -0.2125, "reward_std": 0.43406798243522643, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.4340679943561554, "sampling/importance_sampling_ratio/max": 1.2571020603179932, "sampling/importance_sampling_ratio/mean": 1.0005217492580414, "sampling/importance_sampling_ratio/min": 0.8086998701095581, "sampling/sampling_logp_difference/max": 0.2326374053955078, "sampling/sampling_logp_difference/mean": 0.02168071996420622, "step": 6300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.9, "completions/max_terminated_length": 278.9, "completions/mean_length": 191.075, "completions/mean_terminated_length": 191.075, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.1032161392271518, "epoch": 0.5801231957341179, "frac_reward_zero_std": 0.2, "grad_norm": 6.90437126159668, "kl": 0.05803560609929263, "learning_rate": 4.724384977166913e-06, "loss": 0.0028, "num_tokens": 27397072.0, "reward": -0.075, "reward_std": 0.3558872640132904, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3558872640132904, "sampling/importance_sampling_ratio/max": 1.2864754796028137, "sampling/importance_sampling_ratio/mean": 1.0003357708454133, "sampling/importance_sampling_ratio/min": 0.794737946987152, "sampling/sampling_logp_difference/max": 0.25749757289886477, "sampling/sampling_logp_difference/mean": 0.021868198551237582, "step": 6310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 301.3, "completions/max_terminated_length": 290.5, "completions/mean_length": 220.425, "completions/mean_terminated_length": 214.7916687011719, "completions/min_length": 130.4, "completions/min_terminated_length": 130.4, "entropy": 1.0283865824341774, "epoch": 0.5810425668842512, "frac_reward_zero_std": 0.3, "grad_norm": 0.22281497716903687, "kl": 0.053705842152703555, "learning_rate": 4.723227386622583e-06, "loss": 0.0029, "num_tokens": 27439946.0, "reward": 0.0375, "reward_std": 0.3954332768917084, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.3954332947731018, "sampling/importance_sampling_ratio/max": 1.2724909424781798, "sampling/importance_sampling_ratio/mean": 0.9998671650886536, "sampling/importance_sampling_ratio/min": 0.7961435317993164, "sampling/sampling_logp_difference/max": 0.25638039112091066, "sampling/sampling_logp_difference/mean": 0.020488291792571545, "step": 6320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.0, "completions/max_terminated_length": 319.7, "completions/mean_length": 214.7875, "completions/mean_terminated_length": 212.5625, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0846950247883798, "epoch": 0.5819619380343845, "frac_reward_zero_std": 0.4, "grad_norm": 0.1487865298986435, "kl": 0.062440075573977086, "learning_rate": 4.722067512645512e-06, "loss": 0.0035, "num_tokens": 27488017.0, "reward": 0.03125, "reward_std": 0.23616344034671782, "rewards/reward_fn/mean": 0.03125, "rewards/reward_fn/std": 0.23616344034671782, "sampling/importance_sampling_ratio/max": 1.293822467327118, "sampling/importance_sampling_ratio/mean": 0.9998626828193664, "sampling/importance_sampling_ratio/min": 0.7827783286571502, "sampling/sampling_logp_difference/max": 0.29886641502380373, "sampling/sampling_logp_difference/mean": 0.021185122057795526, "step": 6330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 235.4, "completions/max_terminated_length": 235.4, "completions/mean_length": 167.425, "completions/mean_terminated_length": 167.425, "completions/min_length": 115.1, "completions/min_terminated_length": 115.1, "entropy": 0.9618630453944206, "epoch": 0.5828813091845177, "frac_reward_zero_std": 0.3, "grad_norm": 6.749384880065918, "kl": 0.05407567550428212, "learning_rate": 4.7209053564269854e-06, "loss": 0.0026, "num_tokens": 27525035.0, "reward": 0.2875, "reward_std": 0.34459956288337706, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.3445995688438416, "sampling/importance_sampling_ratio/max": 1.2630678057670592, "sampling/importance_sampling_ratio/mean": 0.9997841656208039, "sampling/importance_sampling_ratio/min": 0.7938756883144379, "sampling/sampling_logp_difference/max": 0.24610230922698975, "sampling/sampling_logp_difference/mean": 0.019998835027217866, "step": 6340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.4, "completions/max_terminated_length": 273.4, "completions/mean_length": 203.4875, "completions/mean_terminated_length": 203.4875, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0706591986119747, "epoch": 0.583800680334651, "frac_reward_zero_std": 0.7, "grad_norm": 0.13787001371383667, "kl": 0.060292388102971016, "learning_rate": 4.719740919160631e-06, "loss": 0.003, "num_tokens": 27567858.0, "reward": 0.275, "reward_std": 0.11700168251991272, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.11700168251991272, "sampling/importance_sampling_ratio/max": 1.2388605713844298, "sampling/importance_sampling_ratio/mean": 1.0002129137516023, "sampling/importance_sampling_ratio/min": 0.7904730856418609, "sampling/sampling_logp_difference/max": 0.24790539741516113, "sampling/sampling_logp_difference/mean": 0.020003251917660236, "step": 6350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 208.1, "completions/mean_terminated_length": 208.1, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0725885674357414, "epoch": 0.5847200514847845, "frac_reward_zero_std": 0.2, "grad_norm": 7.065314292907715, "kl": 0.05769384795567021, "learning_rate": 4.718574202042419e-06, "loss": 0.0027, "num_tokens": 27610594.0, "reward": 0.1875, "reward_std": 0.332049286365509, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3320492923259735, "sampling/importance_sampling_ratio/max": 1.2333193302154541, "sampling/importance_sampling_ratio/mean": 0.9994445323944092, "sampling/importance_sampling_ratio/min": 0.7887594103813171, "sampling/sampling_logp_difference/max": 0.2447727680206299, "sampling/sampling_logp_difference/mean": 0.02111423071473837, "step": 6360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.5, "completions/max_terminated_length": 325.5, "completions/mean_length": 217.55, "completions/mean_terminated_length": 217.55, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0757221065461635, "epoch": 0.5856394226349178, "frac_reward_zero_std": 0.4, "grad_norm": 6.439064025878906, "kl": 0.042228489054832605, "learning_rate": 4.71740520627066e-06, "loss": 0.0018, "num_tokens": 27654126.0, "reward": 0.3125, "reward_std": 0.3035428076982498, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.3035428076982498, "sampling/importance_sampling_ratio/max": 1.2590969324111938, "sampling/importance_sampling_ratio/mean": 0.9998880386352539, "sampling/importance_sampling_ratio/min": 0.7882028698921204, "sampling/sampling_logp_difference/max": 0.25784212350845337, "sampling/sampling_logp_difference/mean": 0.02142437845468521, "step": 6370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 323.6, "completions/max_terminated_length": 304.6, "completions/mean_length": 222.675, "completions/mean_terminated_length": 217.1857162475586, "completions/min_length": 155.9, "completions/min_terminated_length": 155.9, "entropy": 1.1420228436589241, "epoch": 0.586558793785051, "frac_reward_zero_std": 0.4, "grad_norm": 0.10710560530424118, "kl": 0.061873108858708295, "learning_rate": 4.716233933046008e-06, "loss": 0.0032, "num_tokens": 27698020.0, "reward": -0.09375, "reward_std": 0.29333876073360443, "rewards/reward_fn/mean": -0.09375, "rewards/reward_fn/std": 0.2933387666940689, "sampling/importance_sampling_ratio/max": 1.2376533508300782, "sampling/importance_sampling_ratio/mean": 1.0000510334968566, "sampling/importance_sampling_ratio/min": 0.7760604858398438, "sampling/sampling_logp_difference/max": 0.2708033323287964, "sampling/sampling_logp_difference/mean": 0.021824695356190204, "step": 6380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.5, "completions/max_terminated_length": 310.5, "completions/mean_length": 226.7, "completions/mean_terminated_length": 226.7, "completions/min_length": 162.8, "completions/min_terminated_length": 162.8, "entropy": 1.1181060269474983, "epoch": 0.5874781649351843, "frac_reward_zero_std": 0.2, "grad_norm": 0.1111498549580574, "kl": 0.061604120791889726, "learning_rate": 4.715060383571454e-06, "loss": 0.0031, "num_tokens": 27745724.0, "reward": -0.19375, "reward_std": 0.3649082899093628, "rewards/reward_fn/mean": -0.19375, "rewards/reward_fn/std": 0.3649082899093628, "sampling/importance_sampling_ratio/max": 1.2603531360626221, "sampling/importance_sampling_ratio/mean": 0.9997588336467743, "sampling/importance_sampling_ratio/min": 0.791512793302536, "sampling/sampling_logp_difference/max": 0.25716852247714994, "sampling/sampling_logp_difference/mean": 0.021364770643413068, "step": 6390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.7, "completions/max_terminated_length": 288.7, "completions/mean_length": 205.95, "completions/mean_terminated_length": 205.95, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.1478823773562907, "epoch": 0.5883975360853176, "frac_reward_zero_std": 0.1, "grad_norm": 5.890495777130127, "kl": 0.06008486316422932, "learning_rate": 4.713884559052327e-06, "loss": 0.0032, "num_tokens": 27790392.0, "reward": -0.225, "reward_std": 0.37287638187408445, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.37287638187408445, "sampling/importance_sampling_ratio/max": 1.239621114730835, "sampling/importance_sampling_ratio/mean": 1.0000463724136353, "sampling/importance_sampling_ratio/min": 0.7828689455986023, "sampling/sampling_logp_difference/max": 0.2543454885482788, "sampling/sampling_logp_difference/mean": 0.02227793224155903, "step": 6400 }, { "epoch": 0.5883975360853176, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 301.18848167539267, "eval_completions/max_terminated_length": 299.4188481675393, "eval_completions/mean_length": 212.71684118673647, "eval_completions/mean_terminated_length": 212.20256814740299, "eval_completions/min_length": 140.72076788830717, "eval_completions/min_terminated_length": 140.72076788830717, "eval_entropy": 1.1127406142352971, "eval_frac_reward_zero_std": 0.42408376963350786, "eval_kl": 0.06194145265921144, "eval_loss": 0.003257643198594451, "eval_num_tokens": 27790392.0, "eval_reward": 0.05977312390924956, "eval_reward_std": 0.26429427164179287, "eval_rewards/reward_fn/mean": 0.05977312390924956, "eval_rewards/reward_fn/std": 0.2642942788193155, "eval_runtime": 1757.0686, "eval_samples_per_second": 0.326, "eval_sampling/importance_sampling_ratio/max": 1.2742895018992, "eval_sampling/importance_sampling_ratio/mean": 0.9999519474219277, "eval_sampling/importance_sampling_ratio/min": 0.767305584894217, "eval_sampling/sampling_logp_difference/max": 0.29101582737493265, "eval_sampling/sampling_logp_difference/mean": 0.021767915346794623, "eval_steps_per_second": 0.041, "step": 6400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.6, "completions/max_terminated_length": 281.6, "completions/mean_length": 199.4, "completions/mean_terminated_length": 199.4, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "entropy": 1.140552642941475, "epoch": 0.5893169072354509, "frac_reward_zero_std": 0.8, "grad_norm": 0.17851880192756653, "kl": 0.06977051523281261, "learning_rate": 4.712706460696292e-06, "loss": 0.0037, "num_tokens": 27831712.0, "reward": 0.1375, "reward_std": 0.0980459213256836, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.09804592728614807, "sampling/importance_sampling_ratio/max": 1.2868279099464417, "sampling/importance_sampling_ratio/mean": 1.0006411850452424, "sampling/importance_sampling_ratio/min": 0.7834126949310303, "sampling/sampling_logp_difference/max": 0.25857616364955904, "sampling/sampling_logp_difference/mean": 0.021910130232572555, "step": 6410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 391.3, "completions/max_terminated_length": 374.3, "completions/mean_length": 260.6, "completions/mean_terminated_length": 253.42083435058595, "completions/min_length": 171.9, "completions/min_terminated_length": 171.9, "entropy": 1.0753916244953872, "epoch": 0.5902362783855842, "frac_reward_zero_std": 0.6, "grad_norm": 0.11423100531101227, "kl": 0.06381710111163556, "learning_rate": 4.711526089713353e-06, "loss": 0.003, "num_tokens": 27880344.0, "reward": -0.0125, "reward_std": 0.18643819093704223, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.18643819093704223, "sampling/importance_sampling_ratio/max": 1.2943445324897767, "sampling/importance_sampling_ratio/mean": 1.0000058591365815, "sampling/importance_sampling_ratio/min": 0.7858851313591003, "sampling/sampling_logp_difference/max": 0.2742376565933228, "sampling/sampling_logp_difference/mean": 0.020468858629465105, "step": 6420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 311.0, "completions/max_terminated_length": 295.2, "completions/mean_length": 218.05, "completions/mean_terminated_length": 215.0375, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0088297475129366, "epoch": 0.5911556495357175, "frac_reward_zero_std": 0.3, "grad_norm": 6.517147541046143, "kl": 0.05601551482686773, "learning_rate": 4.710343447315845e-06, "loss": 0.0034, "num_tokens": 27923540.0, "reward": 0.05625, "reward_std": 0.34339634478092196, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.3433963626623154, "sampling/importance_sampling_ratio/max": 1.2495843410491942, "sampling/importance_sampling_ratio/mean": 0.9999912917613983, "sampling/importance_sampling_ratio/min": 0.7804013431072235, "sampling/sampling_logp_difference/max": 0.2558472752571106, "sampling/sampling_logp_difference/mean": 0.020548405684530736, "step": 6430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.8, "completions/max_terminated_length": 283.8, "completions/mean_length": 199.025, "completions/mean_terminated_length": 199.025, "completions/min_length": 134.1, "completions/min_terminated_length": 134.1, "entropy": 1.0597386829555036, "epoch": 0.5920750206858508, "frac_reward_zero_std": 0.3, "grad_norm": 6.834327220916748, "kl": 0.04844910238170996, "learning_rate": 4.709158534718435e-06, "loss": 0.0019, "num_tokens": 27962574.0, "reward": 0.225, "reward_std": 0.3984315723180771, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3984315901994705, "sampling/importance_sampling_ratio/max": 1.2805827975273132, "sampling/importance_sampling_ratio/mean": 1.0006895005702972, "sampling/importance_sampling_ratio/min": 0.790285712480545, "sampling/sampling_logp_difference/max": 0.2671658992767334, "sampling/sampling_logp_difference/mean": 0.02112551834434271, "step": 6440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.3, "completions/max_terminated_length": 273.3, "completions/mean_length": 193.925, "completions/mean_terminated_length": 193.925, "completions/min_length": 126.9, "completions/min_terminated_length": 126.9, "entropy": 1.1002034984529019, "epoch": 0.5929943918359842, "frac_reward_zero_std": 0.2, "grad_norm": 7.945777893066406, "kl": 0.06655430651153438, "learning_rate": 4.707971353138124e-06, "loss": 0.0033, "num_tokens": 28005328.0, "reward": 0.075, "reward_std": 0.42685834467411043, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.42685835659503935, "sampling/importance_sampling_ratio/max": 1.2863455653190612, "sampling/importance_sampling_ratio/mean": 1.000013881921768, "sampling/importance_sampling_ratio/min": 0.7873491227626801, "sampling/sampling_logp_difference/max": 0.28044370412826536, "sampling/sampling_logp_difference/mean": 0.02188140116631985, "step": 6450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.5, "completions/max_terminated_length": 292.5, "completions/mean_length": 210.1, "completions/mean_terminated_length": 210.1, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.0388106063008309, "epoch": 0.5939137629861175, "frac_reward_zero_std": 0.3, "grad_norm": 7.690207004547119, "kl": 0.049655258230632174, "learning_rate": 4.706781903794243e-06, "loss": 0.0018, "num_tokens": 28048352.0, "reward": -0.1, "reward_std": 0.34984527826309203, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.34984528422355654, "sampling/importance_sampling_ratio/max": 1.2694766402244568, "sampling/importance_sampling_ratio/mean": 0.9999480605125427, "sampling/importance_sampling_ratio/min": 0.7341262042522431, "sampling/sampling_logp_difference/max": 0.34055122137069704, "sampling/sampling_logp_difference/mean": 0.021230046078562735, "step": 6460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 348.6, "completions/max_terminated_length": 348.6, "completions/mean_length": 239.1625, "completions/mean_terminated_length": 239.1625, "completions/min_length": 155.2, "completions/min_terminated_length": 155.2, "entropy": 1.124115265160799, "epoch": 0.5948331341362508, "frac_reward_zero_std": 0.4, "grad_norm": 5.366975784301758, "kl": 0.04762445252854377, "learning_rate": 4.705590187908452e-06, "loss": 0.0022, "num_tokens": 28096581.0, "reward": -0.325, "reward_std": 0.2794355183839798, "rewards/reward_fn/mean": -0.325, "rewards/reward_fn/std": 0.2794355243444443, "sampling/importance_sampling_ratio/max": 1.260988676548004, "sampling/importance_sampling_ratio/mean": 1.0001127541065216, "sampling/importance_sampling_ratio/min": 0.7639222085475922, "sampling/sampling_logp_difference/max": 0.28994319438934324, "sampling/sampling_logp_difference/mean": 0.02184861060231924, "step": 6470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.3, "completions/max_terminated_length": 311.3, "completions/mean_length": 213.95, "completions/mean_terminated_length": 213.95, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 1.1191321179270743, "epoch": 0.5957525052863841, "frac_reward_zero_std": 0.4, "grad_norm": 0.16259022057056427, "kl": 0.06310149931232445, "learning_rate": 4.7043962067047376e-06, "loss": 0.0036, "num_tokens": 28140729.0, "reward": 0.0125, "reward_std": 0.2972315102815628, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.29723151624202726, "sampling/importance_sampling_ratio/max": 1.2911235809326171, "sampling/importance_sampling_ratio/mean": 1.0000535488128661, "sampling/importance_sampling_ratio/min": 0.7360935509204865, "sampling/sampling_logp_difference/max": 0.343064284324646, "sampling/sampling_logp_difference/mean": 0.02265188079327345, "step": 6480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 218.05, "completions/mean_terminated_length": 218.05, "completions/min_length": 143.7, "completions/min_terminated_length": 143.7, "entropy": 1.073278898000717, "epoch": 0.5966718764365174, "frac_reward_zero_std": 0.2, "grad_norm": 5.741918563842773, "kl": 0.05609490211354569, "learning_rate": 4.703199961409415e-06, "loss": 0.0029, "num_tokens": 28186085.0, "reward": -0.1, "reward_std": 0.3717166125774384, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.37171661853790283, "sampling/importance_sampling_ratio/max": 1.3422470450401307, "sampling/importance_sampling_ratio/mean": 1.0004152178764343, "sampling/importance_sampling_ratio/min": 0.7311745285987854, "sampling/sampling_logp_difference/max": 0.3469825744628906, "sampling/sampling_logp_difference/mean": 0.021227271854877473, "step": 6490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.9, "completions/max_terminated_length": 272.9, "completions/mean_length": 210.7125, "completions/mean_terminated_length": 210.7125, "completions/min_length": 154.5, "completions/min_terminated_length": 154.5, "entropy": 1.1236042480915784, "epoch": 0.5975912475866507, "frac_reward_zero_std": 0.4, "grad_norm": 0.10843749344348907, "kl": 0.06298108347691596, "learning_rate": 4.702001453251125e-06, "loss": 0.0027, "num_tokens": 28231518.0, "reward": -0.0125, "reward_std": 0.2857582807540894, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.28575828671455383, "sampling/importance_sampling_ratio/max": 1.259701633453369, "sampling/importance_sampling_ratio/mean": 0.9999745070934296, "sampling/importance_sampling_ratio/min": 0.7881865620613098, "sampling/sampling_logp_difference/max": 0.26497507095336914, "sampling/sampling_logp_difference/mean": 0.021484932489693166, "step": 6500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.0, "completions/max_terminated_length": 258.0, "completions/mean_length": 206.8875, "completions/mean_terminated_length": 206.8875, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.1007804326713084, "epoch": 0.598510618736784, "frac_reward_zero_std": 0.4, "grad_norm": 4.295722007751465, "kl": 0.05527350713382475, "learning_rate": 4.700800683460829e-06, "loss": 0.0028, "num_tokens": 28274269.0, "reward": 0.125, "reward_std": 0.2748226135969162, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.2748226195573807, "sampling/importance_sampling_ratio/max": 1.2431827545166017, "sampling/importance_sampling_ratio/mean": 0.9996847808361053, "sampling/importance_sampling_ratio/min": 0.7552955746650696, "sampling/sampling_logp_difference/max": 0.29806989431381226, "sampling/sampling_logp_difference/mean": 0.02180723175406456, "step": 6510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 346.4, "completions/max_terminated_length": 330.1, "completions/mean_length": 237.4, "completions/mean_terminated_length": 232.0, "completions/min_length": 155.8, "completions/min_terminated_length": 155.8, "entropy": 1.1116694692522286, "epoch": 0.5994299898869173, "frac_reward_zero_std": 0.2, "grad_norm": 0.1328957974910736, "kl": 0.0401865983963944, "learning_rate": 4.699597653271816e-06, "loss": 0.0017, "num_tokens": 28322285.0, "reward": 0.0125, "reward_std": 0.3564689576625824, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.35646896362304686, "sampling/importance_sampling_ratio/max": 1.32187876701355, "sampling/importance_sampling_ratio/mean": 1.00042262673378, "sampling/importance_sampling_ratio/min": 0.7949499130249024, "sampling/sampling_logp_difference/max": 0.2899640560150146, "sampling/sampling_logp_difference/mean": 0.021367712318897246, "step": 6520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 304.4, "completions/max_terminated_length": 296.8, "completions/mean_length": 214.7875, "completions/mean_terminated_length": 211.18035888671875, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0357746385037898, "epoch": 0.6003493610370506, "frac_reward_zero_std": 0.5, "grad_norm": 5.095525741577148, "kl": 0.0541783868917264, "learning_rate": 4.698392363919694e-06, "loss": 0.0029, "num_tokens": 28366844.0, "reward": 0.03125, "reward_std": 0.22814334630966188, "rewards/reward_fn/mean": 0.03125, "rewards/reward_fn/std": 0.22814335227012633, "sampling/importance_sampling_ratio/max": 1.2725078105926513, "sampling/importance_sampling_ratio/mean": 1.0002647578716277, "sampling/importance_sampling_ratio/min": 0.8044821918010712, "sampling/sampling_logp_difference/max": 0.25182098150253296, "sampling/sampling_logp_difference/mean": 0.02063723560422659, "step": 6530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 212.7, "completions/mean_terminated_length": 212.7, "completions/min_length": 143.8, "completions/min_terminated_length": 143.8, "entropy": 1.0229517601430416, "epoch": 0.601268732187184, "frac_reward_zero_std": 0.2, "grad_norm": 0.14948205649852753, "kl": 0.04433594692964107, "learning_rate": 4.697184816642392e-06, "loss": 0.0013, "num_tokens": 28409660.0, "reward": -0.1875, "reward_std": 0.45328023433685305, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.45328024625778196, "sampling/importance_sampling_ratio/max": 1.2769538044929505, "sampling/importance_sampling_ratio/mean": 0.9999709725379944, "sampling/importance_sampling_ratio/min": 0.7692527294158935, "sampling/sampling_logp_difference/max": 0.2807143568992615, "sampling/sampling_logp_difference/mean": 0.02112936656922102, "step": 6540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 300.2, "completions/max_terminated_length": 281.7, "completions/mean_length": 214.8875, "completions/mean_terminated_length": 211.7107147216797, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.0688552245497704, "epoch": 0.6021881033373173, "frac_reward_zero_std": 0.1, "grad_norm": 6.147977352142334, "kl": 0.06027426638174802, "learning_rate": 4.6959750126801565e-06, "loss": 0.0026, "num_tokens": 28455963.0, "reward": -0.16875, "reward_std": 0.5030868291854859, "rewards/reward_fn/mean": -0.16875, "rewards/reward_fn/std": 0.5030868351459503, "sampling/importance_sampling_ratio/max": 1.266992461681366, "sampling/importance_sampling_ratio/mean": 0.9999652683734894, "sampling/importance_sampling_ratio/min": 0.7717308163642883, "sampling/sampling_logp_difference/max": 0.2738988041877747, "sampling/sampling_logp_difference/mean": 0.021233023144304752, "step": 6550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.7, "completions/max_terminated_length": 289.7, "completions/mean_length": 197.625, "completions/mean_terminated_length": 197.625, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0125599063932895, "epoch": 0.6031074744874506, "frac_reward_zero_std": 0.3, "grad_norm": 6.874145030975342, "kl": 0.05518628921126947, "learning_rate": 4.694762953275554e-06, "loss": 0.003, "num_tokens": 28498717.0, "reward": -0.25, "reward_std": 0.3185652822256088, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.3185652881860733, "sampling/importance_sampling_ratio/max": 1.2456906676292419, "sampling/importance_sampling_ratio/mean": 1.0002852916717528, "sampling/importance_sampling_ratio/min": 0.7621093034744263, "sampling/sampling_logp_difference/max": 0.2888748586177826, "sampling/sampling_logp_difference/mean": 0.02088515181094408, "step": 6560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.6, "completions/max_terminated_length": 264.6, "completions/mean_length": 190.5, "completions/mean_terminated_length": 190.5, "completions/min_length": 119.2, "completions/min_terminated_length": 119.2, "entropy": 1.0812375828623773, "epoch": 0.6040268456375839, "frac_reward_zero_std": 0.3, "grad_norm": 0.15348157286643982, "kl": 0.054522328171879055, "learning_rate": 4.6935486396734666e-06, "loss": 0.0027, "num_tokens": 28542341.0, "reward": -0.175, "reward_std": 0.3193414121866226, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3193414181470871, "sampling/importance_sampling_ratio/max": 1.2910607695579528, "sampling/importance_sampling_ratio/mean": 1.0002699017524719, "sampling/importance_sampling_ratio/min": 0.767870819568634, "sampling/sampling_logp_difference/max": 0.3117747068405151, "sampling/sampling_logp_difference/mean": 0.021191845275461675, "step": 6570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.3, "completions/max_terminated_length": 293.3, "completions/mean_length": 204.025, "completions/mean_terminated_length": 204.025, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.1789320133626462, "epoch": 0.6049462167877172, "frac_reward_zero_std": 0.3, "grad_norm": 15.32407283782959, "kl": 0.0939450132776983, "learning_rate": 4.692332073121092e-06, "loss": 0.0045, "num_tokens": 28586503.0, "reward": 0.1125, "reward_std": 0.3664708971977234, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.36647090315818787, "sampling/importance_sampling_ratio/max": 1.2901464462280274, "sampling/importance_sampling_ratio/mean": 0.9993108451366425, "sampling/importance_sampling_ratio/min": 0.7882357776165009, "sampling/sampling_logp_difference/max": 0.28635857105255125, "sampling/sampling_logp_difference/mean": 0.022379349917173386, "step": 6580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 245.8, "completions/max_terminated_length": 245.8, "completions/mean_length": 183.625, "completions/mean_terminated_length": 183.625, "completions/min_length": 117.4, "completions/min_terminated_length": 117.4, "entropy": 0.9794266946613789, "epoch": 0.6058655879378505, "frac_reward_zero_std": 0.7, "grad_norm": 0.1575051099061966, "kl": 0.030132370861247183, "learning_rate": 4.691113254867942e-06, "loss": 0.0014, "num_tokens": 28626881.0, "reward": -0.075, "reward_std": 0.1498008370399475, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.14980084896087648, "sampling/importance_sampling_ratio/max": 1.2865644931793212, "sampling/importance_sampling_ratio/mean": 0.9994420886039734, "sampling/importance_sampling_ratio/min": 0.7910012304782867, "sampling/sampling_logp_difference/max": 0.266807496547699, "sampling/sampling_logp_difference/mean": 0.020635079033672808, "step": 6590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.9, "completions/max_terminated_length": 251.9, "completions/mean_length": 186.95, "completions/mean_terminated_length": 186.95, "completions/min_length": 111.1, "completions/min_terminated_length": 111.1, "entropy": 1.0408857516944408, "epoch": 0.6067849590879838, "frac_reward_zero_std": 0.3, "grad_norm": 8.724885940551758, "kl": 0.05112304209615104, "learning_rate": 4.68989218616584e-06, "loss": 0.0023, "num_tokens": 28665701.0, "reward": -0.1625, "reward_std": 0.32810779809951784, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.32810781002044676, "sampling/importance_sampling_ratio/max": 1.2828383922576905, "sampling/importance_sampling_ratio/mean": 1.0003514289855957, "sampling/importance_sampling_ratio/min": 0.7838038146495819, "sampling/sampling_logp_difference/max": 0.2821976900100708, "sampling/sampling_logp_difference/mean": 0.02131260298192501, "step": 6600 }, { "epoch": 0.6067849590879838, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0004363001745200698, "eval_completions/max_length": 286.8394415357766, "eval_completions/max_terminated_length": 286.5759162303665, "eval_completions/mean_length": 199.5519197207679, "eval_completions/mean_terminated_length": 199.44571181159162, "eval_completions/min_length": 127.96335078534031, "eval_completions/min_terminated_length": 127.96335078534031, "eval_entropy": 1.0793265205715339, "eval_frac_reward_zero_std": 0.30017452006980805, "eval_kl": 0.048713106779872485, "eval_loss": 0.002523747505620122, "eval_num_tokens": 28665701.0, "eval_reward": 0.0581369982547993, "eval_reward_std": 0.3411068114310659, "eval_rewards/reward_fn/mean": 0.0581369982547993, "eval_rewards/reward_fn/std": 0.34110681964880923, "eval_runtime": 1683.4142, "eval_samples_per_second": 0.34, "eval_sampling/importance_sampling_ratio/max": 1.2629436705957116, "eval_sampling/importance_sampling_ratio/mean": 0.9999684097463132, "eval_sampling/importance_sampling_ratio/min": 0.774613283795628, "eval_sampling/sampling_logp_difference/max": 0.2772146638151239, "eval_sampling/sampling_logp_difference/mean": 0.021778167501729093, "eval_steps_per_second": 0.043, "step": 6600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.2, "completions/max_terminated_length": 250.2, "completions/mean_length": 186.125, "completions/mean_terminated_length": 186.125, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0622225984930993, "epoch": 0.6077043302381171, "frac_reward_zero_std": 0.2, "grad_norm": 5.780598163604736, "kl": 0.05618832890177146, "learning_rate": 4.688668868268921e-06, "loss": 0.003, "num_tokens": 28709863.0, "reward": -0.0875, "reward_std": 0.3791164219379425, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.379116427898407, "sampling/importance_sampling_ratio/max": 1.2290721774101256, "sampling/importance_sampling_ratio/mean": 0.9998210966587067, "sampling/importance_sampling_ratio/min": 0.7747166216373443, "sampling/sampling_logp_difference/max": 0.27154449820518495, "sampling/sampling_logp_difference/mean": 0.020862479135394095, "step": 6610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 211.8875, "completions/mean_terminated_length": 211.8875, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0691601939499378, "epoch": 0.6086237013882504, "frac_reward_zero_std": 0.2, "grad_norm": 5.949253082275391, "kl": 0.042310426477342844, "learning_rate": 4.687443302433631e-06, "loss": 0.0024, "num_tokens": 28756574.0, "reward": -0.175, "reward_std": 0.3662448555231094, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.3662448674440384, "sampling/importance_sampling_ratio/max": 1.249899423122406, "sampling/importance_sampling_ratio/mean": 0.999539852142334, "sampling/importance_sampling_ratio/min": 0.7916143417358399, "sampling/sampling_logp_difference/max": 0.24048879146575927, "sampling/sampling_logp_difference/mean": 0.02073957398533821, "step": 6620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 195.925, "completions/mean_terminated_length": 195.925, "completions/min_length": 111.0, "completions/min_terminated_length": 111.0, "entropy": 1.0465519703924655, "epoch": 0.6095430725383837, "frac_reward_zero_std": 0.2, "grad_norm": 7.619245529174805, "kl": 0.036577882239362224, "learning_rate": 4.686215489918724e-06, "loss": 0.0022, "num_tokens": 28799064.0, "reward": 0.0, "reward_std": 0.45853824317455294, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.45853826105594636, "sampling/importance_sampling_ratio/max": 1.2395642876625061, "sampling/importance_sampling_ratio/mean": 1.0000780403614045, "sampling/importance_sampling_ratio/min": 0.7945998311042786, "sampling/sampling_logp_difference/max": 0.2371358633041382, "sampling/sampling_logp_difference/mean": 0.02094010226428509, "step": 6630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 198.6375, "completions/mean_terminated_length": 198.6375, "completions/min_length": 108.4, "completions/min_terminated_length": 108.4, "entropy": 1.0828476385679096, "epoch": 0.6104624436885171, "frac_reward_zero_std": 0.1, "grad_norm": 5.500866889953613, "kl": 0.05637252437882125, "learning_rate": 4.684985431985263e-06, "loss": 0.0017, "num_tokens": 28840331.0, "reward": 0.08125, "reward_std": 0.4347601145505905, "rewards/reward_fn/mean": 0.08125, "rewards/reward_fn/std": 0.434760120511055, "sampling/importance_sampling_ratio/max": 1.2804141521453858, "sampling/importance_sampling_ratio/mean": 0.9999092638492584, "sampling/importance_sampling_ratio/min": 0.7911666989326477, "sampling/sampling_logp_difference/max": 0.2663842558860779, "sampling/sampling_logp_difference/mean": 0.021877037733793257, "step": 6640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.7, "completions/max_terminated_length": 273.7, "completions/mean_length": 195.55, "completions/mean_terminated_length": 195.55, "completions/min_length": 122.1, "completions/min_terminated_length": 122.1, "entropy": 0.9815972581505775, "epoch": 0.6113818148386504, "frac_reward_zero_std": 0.2, "grad_norm": 6.434087753295898, "kl": 0.04146459153853357, "learning_rate": 4.6837531298966145e-06, "loss": 0.0021, "num_tokens": 28882591.0, "reward": 0.15, "reward_std": 0.36193286776542666, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.36193287968635557, "sampling/importance_sampling_ratio/max": 1.2819383144378662, "sampling/importance_sampling_ratio/mean": 1.0000441431999207, "sampling/importance_sampling_ratio/min": 0.7762354671955108, "sampling/sampling_logp_difference/max": 0.2784293174743652, "sampling/sampling_logp_difference/mean": 0.020076168701052666, "step": 6650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.1, "completions/max_terminated_length": 321.1, "completions/mean_length": 224.375, "completions/mean_terminated_length": 224.375, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.089103788137436, "epoch": 0.6123011859887837, "frac_reward_zero_std": 0.1, "grad_norm": 6.827288627624512, "kl": 0.04946248646592721, "learning_rate": 4.682518584918453e-06, "loss": 0.0025, "num_tokens": 28927725.0, "reward": -0.2125, "reward_std": 0.44064723551273344, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.4406472533941269, "sampling/importance_sampling_ratio/max": 1.3398911118507386, "sampling/importance_sampling_ratio/mean": 1.0000384390354156, "sampling/importance_sampling_ratio/min": 0.7956932306289672, "sampling/sampling_logp_difference/max": 0.30133056640625, "sampling/sampling_logp_difference/mean": 0.021049473993480207, "step": 6660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.3, "completions/max_terminated_length": 301.3, "completions/mean_length": 203.8375, "completions/mean_terminated_length": 203.8375, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.090546839684248, "epoch": 0.613220557138917, "frac_reward_zero_std": 0.3, "grad_norm": 5.501370429992676, "kl": 0.056024701485875995, "learning_rate": 4.681281798318754e-06, "loss": 0.0024, "num_tokens": 28973880.0, "reward": -0.0625, "reward_std": 0.3636886715888977, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.3636886954307556, "sampling/importance_sampling_ratio/max": 1.3274587988853455, "sampling/importance_sampling_ratio/mean": 1.0000105619430542, "sampling/importance_sampling_ratio/min": 0.7645586431026459, "sampling/sampling_logp_difference/max": 0.29791786670684817, "sampling/sampling_logp_difference/mean": 0.02171063907444477, "step": 6670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.9, "completions/max_terminated_length": 300.9, "completions/mean_length": 205.6625, "completions/mean_terminated_length": 205.6625, "completions/min_length": 129.8, "completions/min_terminated_length": 129.8, "entropy": 1.0911942847073077, "epoch": 0.6141399282890503, "frac_reward_zero_std": 0.4, "grad_norm": 5.215987205505371, "kl": 0.0499262303346768, "learning_rate": 4.680042771367797e-06, "loss": 0.0027, "num_tokens": 29015477.0, "reward": 0.175, "reward_std": 0.2739637613296509, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.27396377325057986, "sampling/importance_sampling_ratio/max": 1.288773810863495, "sampling/importance_sampling_ratio/mean": 1.0000105381011963, "sampling/importance_sampling_ratio/min": 0.7785434782505035, "sampling/sampling_logp_difference/max": 0.2994304418563843, "sampling/sampling_logp_difference/mean": 0.02125229239463806, "step": 6680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.8, "completions/max_terminated_length": 290.8, "completions/mean_length": 212.175, "completions/mean_terminated_length": 212.175, "completions/min_length": 146.4, "completions/min_terminated_length": 146.4, "entropy": 1.1108071878552437, "epoch": 0.6150592994391836, "frac_reward_zero_std": 0.5, "grad_norm": 5.469494342803955, "kl": 0.055718297022394836, "learning_rate": 4.678801505338163e-06, "loss": 0.0033, "num_tokens": 29061251.0, "reward": 0.25, "reward_std": 0.2276727557182312, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.22767276763916017, "sampling/importance_sampling_ratio/max": 1.2935654878616334, "sampling/importance_sampling_ratio/mean": 0.9994342505931855, "sampling/importance_sampling_ratio/min": 0.7713475346565246, "sampling/sampling_logp_difference/max": 0.2980908274650574, "sampling/sampling_logp_difference/mean": 0.021103793010115623, "step": 6690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.5, "completions/max_terminated_length": 318.5, "completions/mean_length": 227.425, "completions/mean_terminated_length": 227.425, "completions/min_length": 137.6, "completions/min_terminated_length": 137.6, "entropy": 1.0685371793806553, "epoch": 0.6159786705893169, "frac_reward_zero_std": 0.2, "grad_norm": 5.516569137573242, "kl": 0.05985648108180612, "learning_rate": 4.67755800150473e-06, "loss": 0.0028, "num_tokens": 29107701.0, "reward": -0.05, "reward_std": 0.39010659158229827, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3901066094636917, "sampling/importance_sampling_ratio/max": 1.3361659526824952, "sampling/importance_sampling_ratio/mean": 0.9999526917934418, "sampling/importance_sampling_ratio/min": 0.7891739666461944, "sampling/sampling_logp_difference/max": 0.29613181948661804, "sampling/sampling_logp_difference/mean": 0.020833785459399224, "step": 6700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 211.8875, "completions/mean_terminated_length": 211.8875, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.0137500926852225, "epoch": 0.6168980417394502, "frac_reward_zero_std": 0.1, "grad_norm": 6.433523178100586, "kl": 0.03887462035054341, "learning_rate": 4.6763122611446765e-06, "loss": 0.0019, "num_tokens": 29147516.0, "reward": -0.0125, "reward_std": 0.4538504630327225, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.45385047495365144, "sampling/importance_sampling_ratio/max": 1.285509753227234, "sampling/importance_sampling_ratio/mean": 1.0002160370349884, "sampling/importance_sampling_ratio/min": 0.7662584841251373, "sampling/sampling_logp_difference/max": 0.2902275621891022, "sampling/sampling_logp_difference/mean": 0.020672991313040255, "step": 6710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.9, "completions/max_terminated_length": 320.9, "completions/mean_length": 211.325, "completions/mean_terminated_length": 211.325, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.1308486185967923, "epoch": 0.6178174128895835, "frac_reward_zero_std": 0.0, "grad_norm": 6.557374000549316, "kl": 0.059669853863306345, "learning_rate": 4.675064285537479e-06, "loss": 0.0026, "num_tokens": 29191206.0, "reward": -0.025, "reward_std": 0.4958795368671417, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.4958795428276062, "sampling/importance_sampling_ratio/max": 1.4397983789443969, "sampling/importance_sampling_ratio/mean": 1.0003291189670562, "sampling/importance_sampling_ratio/min": 0.7238905787467956, "sampling/sampling_logp_difference/max": 0.4397250771522522, "sampling/sampling_logp_difference/mean": 0.02223335150629282, "step": 6720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.5, "completions/max_terminated_length": 269.5, "completions/mean_length": 197.6, "completions/mean_terminated_length": 197.6, "completions/min_length": 144.4, "completions/min_terminated_length": 144.4, "entropy": 1.011008794605732, "epoch": 0.6187367840397169, "frac_reward_zero_std": 0.2, "grad_norm": 6.798991680145264, "kl": 0.08817185126245022, "learning_rate": 4.6738140759649064e-06, "loss": 0.0042, "num_tokens": 29231878.0, "reward": -0.1625, "reward_std": 0.35154260993003844, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.35154260993003844, "sampling/importance_sampling_ratio/max": 1.3025209188461304, "sampling/importance_sampling_ratio/mean": 1.000456589460373, "sampling/importance_sampling_ratio/min": 0.7527783691883088, "sampling/sampling_logp_difference/max": 0.3251086175441742, "sampling/sampling_logp_difference/mean": 0.021010379120707513, "step": 6730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.3, "completions/max_terminated_length": 326.3, "completions/mean_length": 235.35, "completions/mean_terminated_length": 235.35, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0962674736976623, "epoch": 0.6196561551898502, "frac_reward_zero_std": 0.4, "grad_norm": 0.14123082160949707, "kl": 0.04932711883448064, "learning_rate": 4.672561633711026e-06, "loss": 0.002, "num_tokens": 29279858.0, "reward": 0.2625, "reward_std": 0.2394672751426697, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.23946728110313414, "sampling/importance_sampling_ratio/max": 1.3456030011177063, "sampling/importance_sampling_ratio/mean": 1.000491976737976, "sampling/importance_sampling_ratio/min": 0.8033839344978333, "sampling/sampling_logp_difference/max": 0.2943825364112854, "sampling/sampling_logp_difference/mean": 0.020889033563435078, "step": 6740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.7, "completions/max_terminated_length": 300.7, "completions/mean_length": 220.0, "completions/mean_terminated_length": 220.0, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.067647210508585, "epoch": 0.6205755263399835, "frac_reward_zero_std": 0.4, "grad_norm": 0.12703394889831543, "kl": 0.048374779312871395, "learning_rate": 4.671306960062195e-06, "loss": 0.0025, "num_tokens": 29326538.0, "reward": 0.2, "reward_std": 0.3153284192085266, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.31532842516899107, "sampling/importance_sampling_ratio/max": 1.2758236885070802, "sampling/importance_sampling_ratio/mean": 1.0001848340034485, "sampling/importance_sampling_ratio/min": 0.7751625001430511, "sampling/sampling_logp_difference/max": 0.27781530618667605, "sampling/sampling_logp_difference/mean": 0.021103197894990443, "step": 6750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 314.3, "completions/max_terminated_length": 310.8, "completions/mean_length": 220.2875, "completions/mean_terminated_length": 216.39583435058594, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 0.8945754617452621, "epoch": 0.6214948974901168, "frac_reward_zero_std": 0.1, "grad_norm": 0.18356207013130188, "kl": 0.05154978273203596, "learning_rate": 4.670050056307065e-06, "loss": 0.0025, "num_tokens": 29368777.0, "reward": -0.125, "reward_std": 0.500145035982132, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.500145035982132, "sampling/importance_sampling_ratio/max": 1.2708938002586365, "sampling/importance_sampling_ratio/mean": 0.9999270439147949, "sampling/importance_sampling_ratio/min": 0.7740569829940795, "sampling/sampling_logp_difference/max": 0.2744333386421204, "sampling/sampling_logp_difference/mean": 0.018797744810581208, "step": 6760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.5, "completions/max_terminated_length": 305.5, "completions/mean_length": 212.4625, "completions/mean_terminated_length": 212.4625, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.1364482261240483, "epoch": 0.6224142686402501, "frac_reward_zero_std": 0.2, "grad_norm": 5.612137794494629, "kl": 0.0330553645035252, "learning_rate": 4.668790923736577e-06, "loss": 0.0016, "num_tokens": 29416086.0, "reward": 0.1625, "reward_std": 0.45704681277275083, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.45704683661460876, "sampling/importance_sampling_ratio/max": 1.274204635620117, "sampling/importance_sampling_ratio/mean": 1.0006697297096252, "sampling/importance_sampling_ratio/min": 0.7953601777553558, "sampling/sampling_logp_difference/max": 0.28102970123291016, "sampling/sampling_logp_difference/mean": 0.021726584807038308, "step": 6770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 312.7, "completions/max_terminated_length": 298.2, "completions/mean_length": 206.5625, "completions/mean_terminated_length": 203.7482147216797, "completions/min_length": 114.0, "completions/min_terminated_length": 114.0, "entropy": 1.0693196848034858, "epoch": 0.6233336397903834, "frac_reward_zero_std": 0.3, "grad_norm": 6.7688751220703125, "kl": 0.04354674016358331, "learning_rate": 4.667529563643959e-06, "loss": 0.002, "num_tokens": 29461411.0, "reward": -0.1375, "reward_std": 0.4097943514585495, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.409794357419014, "sampling/importance_sampling_ratio/max": 1.2784115552902222, "sampling/importance_sampling_ratio/mean": 1.0000968337059022, "sampling/importance_sampling_ratio/min": 0.7785356223583222, "sampling/sampling_logp_difference/max": 0.27309916019439695, "sampling/sampling_logp_difference/mean": 0.021031028218567373, "step": 6780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.4, "completions/max_terminated_length": 300.4, "completions/mean_length": 195.8625, "completions/mean_terminated_length": 195.8625, "completions/min_length": 120.0, "completions/min_terminated_length": 120.0, "entropy": 1.098496875911951, "epoch": 0.6242530109405167, "frac_reward_zero_std": 0.4, "grad_norm": 0.12524226307868958, "kl": 0.03582521768403239, "learning_rate": 4.666265977324731e-06, "loss": 0.0017, "num_tokens": 29502872.0, "reward": 0.1875, "reward_std": 0.2777381867170334, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.27773819863796234, "sampling/importance_sampling_ratio/max": 1.2434162735939025, "sampling/importance_sampling_ratio/mean": 0.9999323487281799, "sampling/importance_sampling_ratio/min": 0.7974631726741791, "sampling/sampling_logp_difference/max": 0.24550031423568724, "sampling/sampling_logp_difference/mean": 0.021749641001224517, "step": 6790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.1, "completions/max_terminated_length": 290.1, "completions/mean_length": 213.825, "completions/mean_terminated_length": 213.825, "completions/min_length": 131.5, "completions/min_terminated_length": 131.5, "entropy": 1.1424120552837849, "epoch": 0.62517238209065, "frac_reward_zero_std": 0.1, "grad_norm": 6.539258003234863, "kl": 0.05649263090454042, "learning_rate": 4.665000166076697e-06, "loss": 0.0028, "num_tokens": 29545882.0, "reward": -0.1875, "reward_std": 0.4628865152597427, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.4628865510225296, "sampling/importance_sampling_ratio/max": 1.258629274368286, "sampling/importance_sampling_ratio/mean": 0.9995796322822571, "sampling/importance_sampling_ratio/min": 0.7820207178592682, "sampling/sampling_logp_difference/max": 0.2577647686004639, "sampling/sampling_logp_difference/mean": 0.022256471402943134, "step": 6800 }, { "epoch": 0.62517238209065, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.001963350785340314, "eval_completions/max_length": 307.2739965095986, "eval_completions/max_terminated_length": 304.59685863874347, "eval_completions/mean_length": 213.94349912739966, "eval_completions/mean_terminated_length": 213.39765661216322, "eval_completions/min_length": 137.63176265270505, "eval_completions/min_terminated_length": 137.63176265270505, "eval_entropy": 1.0944355514037047, "eval_frac_reward_zero_std": 0.2670157068062827, "eval_kl": 0.04171284452652848, "eval_loss": 0.0021813286002725363, "eval_num_tokens": 29545882.0, "eval_reward": 0.06119109947643979, "eval_reward_std": 0.3629122475522975, "eval_rewards/reward_fn/mean": 0.06119109947643979, "eval_rewards/reward_fn/std": 0.3629122584746146, "eval_runtime": 1786.6055, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.275020508241903, "eval_sampling/importance_sampling_ratio/mean": 1.0000621857443406, "eval_sampling/importance_sampling_ratio/min": 0.7714944661287737, "eval_sampling/sampling_logp_difference/max": 0.28686973405758126, "eval_sampling/sampling_logp_difference/mean": 0.021319506369713193, "eval_steps_per_second": 0.04, "step": 6800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.0, "completions/max_terminated_length": 324.0, "completions/mean_length": 215.1, "completions/mean_terminated_length": 215.1, "completions/min_length": 136.3, "completions/min_terminated_length": 136.3, "entropy": 1.1555157959461213, "epoch": 0.6260917532407833, "frac_reward_zero_std": 0.1, "grad_norm": 6.354942321777344, "kl": 0.04119767733500339, "learning_rate": 4.663732131199946e-06, "loss": 0.0023, "num_tokens": 29590578.0, "reward": 0.025, "reward_std": 0.4903250575065613, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4903250753879547, "sampling/importance_sampling_ratio/max": 1.243701958656311, "sampling/importance_sampling_ratio/mean": 0.999388587474823, "sampling/importance_sampling_ratio/min": 0.772141021490097, "sampling/sampling_logp_difference/max": 0.26975553035736083, "sampling/sampling_logp_difference/mean": 0.022179535403847695, "step": 6810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 314.5, "completions/max_terminated_length": 287.4, "completions/mean_length": 211.85, "completions/mean_terminated_length": 207.81964416503905, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.1244710437953471, "epoch": 0.6270111243909167, "frac_reward_zero_std": 0.3, "grad_norm": 0.13574959337711334, "kl": 0.04340720770414919, "learning_rate": 4.662461873996853e-06, "loss": 0.0021, "num_tokens": 29633278.0, "reward": 0.10625, "reward_std": 0.35747711062431337, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.35747711062431337, "sampling/importance_sampling_ratio/max": 1.3211257815361024, "sampling/importance_sampling_ratio/mean": 1.0002057731151581, "sampling/importance_sampling_ratio/min": 0.7779735743999481, "sampling/sampling_logp_difference/max": 0.2981942296028137, "sampling/sampling_logp_difference/mean": 0.021695419773459436, "step": 6820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.2, "completions/max_terminated_length": 283.2, "completions/mean_length": 205.275, "completions/mean_terminated_length": 205.275, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.1168083816766738, "epoch": 0.62793049554105, "frac_reward_zero_std": 0.4, "grad_norm": 0.14704281091690063, "kl": 0.04773867194307968, "learning_rate": 4.6611893957720715e-06, "loss": 0.0026, "num_tokens": 29681700.0, "reward": -0.0375, "reward_std": 0.3530599385499954, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.3530599445104599, "sampling/importance_sampling_ratio/max": 1.2660350322723388, "sampling/importance_sampling_ratio/mean": 1.0005456387996674, "sampling/importance_sampling_ratio/min": 0.771111911535263, "sampling/sampling_logp_difference/max": 0.2757743239402771, "sampling/sampling_logp_difference/mean": 0.02183311153203249, "step": 6830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 208.9625, "completions/mean_terminated_length": 208.9625, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0161209784448146, "epoch": 0.6288498666911833, "frac_reward_zero_std": 0.1, "grad_norm": 5.436909198760986, "kl": 0.06230252794921398, "learning_rate": 4.6599146978325405e-06, "loss": 0.0031, "num_tokens": 29726153.0, "reward": -0.1375, "reward_std": 0.4254619300365448, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.42546194791793823, "sampling/importance_sampling_ratio/max": 1.2658935189247131, "sampling/importance_sampling_ratio/mean": 0.9994531333446502, "sampling/importance_sampling_ratio/min": 0.7994739711284637, "sampling/sampling_logp_difference/max": 0.25241904258728026, "sampling/sampling_logp_difference/mean": 0.019927116483449935, "step": 6840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.7, "completions/max_terminated_length": 306.7, "completions/mean_length": 217.425, "completions/mean_terminated_length": 217.425, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.1072007894515992, "epoch": 0.6297692378413166, "frac_reward_zero_std": 0.3, "grad_norm": 0.14334885776042938, "kl": 0.055353190226014704, "learning_rate": 4.658637781487477e-06, "loss": 0.0024, "num_tokens": 29769859.0, "reward": 0.15, "reward_std": 0.3466688096523285, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.3466688334941864, "sampling/importance_sampling_ratio/max": 1.2791266441345215, "sampling/importance_sampling_ratio/mean": 1.0000653445720673, "sampling/importance_sampling_ratio/min": 0.7633672654628754, "sampling/sampling_logp_difference/max": 0.29541170597076416, "sampling/sampling_logp_difference/mean": 0.021611334197223185, "step": 6850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 323.6, "completions/max_terminated_length": 300.3, "completions/mean_length": 208.8125, "completions/mean_terminated_length": 205.14464416503907, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 1.1705016158521175, "epoch": 0.6306886089914499, "frac_reward_zero_std": 0.4, "grad_norm": 0.13951478898525238, "kl": 0.0459657346829772, "learning_rate": 4.657358648048375e-06, "loss": 0.0024, "num_tokens": 29812716.0, "reward": -0.00625, "reward_std": 0.2870673507452011, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.2870673507452011, "sampling/importance_sampling_ratio/max": 1.2458139300346374, "sampling/importance_sampling_ratio/mean": 0.9998774826526642, "sampling/importance_sampling_ratio/min": 0.7827580690383911, "sampling/sampling_logp_difference/max": 0.25504260063171386, "sampling/sampling_logp_difference/mean": 0.02203933708369732, "step": 6860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.9, "completions/max_terminated_length": 290.9, "completions/mean_length": 198.0, "completions/mean_terminated_length": 198.0, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 1.0373144365847111, "epoch": 0.6316079801415831, "frac_reward_zero_std": 0.5, "grad_norm": 0.13260096311569214, "kl": 0.03887657629675232, "learning_rate": 4.656077298829009e-06, "loss": 0.0018, "num_tokens": 29852348.0, "reward": 0.1, "reward_std": 0.2675303816795349, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.26753038763999937, "sampling/importance_sampling_ratio/max": 1.2509888768196107, "sampling/importance_sampling_ratio/mean": 1.000206995010376, "sampling/importance_sampling_ratio/min": 0.7825287640094757, "sampling/sampling_logp_difference/max": 0.2552851915359497, "sampling/sampling_logp_difference/mean": 0.02099158223718405, "step": 6870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 262.0, "completions/max_terminated_length": 262.0, "completions/mean_length": 194.6625, "completions/mean_terminated_length": 194.6625, "completions/min_length": 131.9, "completions/min_terminated_length": 131.9, "entropy": 1.127125496417284, "epoch": 0.6325273512917164, "frac_reward_zero_std": 0.3, "grad_norm": 6.785040378570557, "kl": 0.04155215995851904, "learning_rate": 4.654793735145425e-06, "loss": 0.0019, "num_tokens": 29896321.0, "reward": 0.175, "reward_std": 0.38308919668197633, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3830892026424408, "sampling/importance_sampling_ratio/max": 1.3237892270088196, "sampling/importance_sampling_ratio/mean": 0.9999879240989685, "sampling/importance_sampling_ratio/min": 0.7832675039768219, "sampling/sampling_logp_difference/max": 0.30809980630874634, "sampling/sampling_logp_difference/mean": 0.022603087686002256, "step": 6880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 323.3, "completions/max_terminated_length": 319.0, "completions/mean_length": 225.7, "completions/mean_terminated_length": 220.46666870117187, "completions/min_length": 140.5, "completions/min_terminated_length": 140.5, "entropy": 1.1387503877282144, "epoch": 0.6334467224418497, "frac_reward_zero_std": 0.2, "grad_norm": 8.038926124572754, "kl": 0.044994328450411555, "learning_rate": 4.653507958315948e-06, "loss": 0.0021, "num_tokens": 29941993.0, "reward": 0.0875, "reward_std": 0.40024718046188357, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.40024718046188357, "sampling/importance_sampling_ratio/max": 1.2955519795417785, "sampling/importance_sampling_ratio/mean": 1.0002415895462036, "sampling/importance_sampling_ratio/min": 0.7670924007892609, "sampling/sampling_logp_difference/max": 0.29513548612594603, "sampling/sampling_logp_difference/mean": 0.022347428277134894, "step": 6890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 195.175, "completions/mean_terminated_length": 195.175, "completions/min_length": 121.7, "completions/min_terminated_length": 121.7, "entropy": 1.0728932473808528, "epoch": 0.634366093591983, "frac_reward_zero_std": 0.6, "grad_norm": 5.686387538909912, "kl": 0.0415146428742446, "learning_rate": 4.652219969661174e-06, "loss": 0.002, "num_tokens": 29985351.0, "reward": 0.225, "reward_std": 0.18685350716114044, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.18685351312160492, "sampling/importance_sampling_ratio/max": 1.2598000288009643, "sampling/importance_sampling_ratio/mean": 1.000317108631134, "sampling/importance_sampling_ratio/min": 0.7552180290222168, "sampling/sampling_logp_difference/max": 0.3066049098968506, "sampling/sampling_logp_difference/mean": 0.021217455342411994, "step": 6900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.7, "completions/max_terminated_length": 288.7, "completions/mean_length": 202.825, "completions/mean_terminated_length": 202.825, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.0837183758616447, "epoch": 0.6352854647421164, "frac_reward_zero_std": 0.2, "grad_norm": 0.21113154292106628, "kl": 0.04661462446092628, "learning_rate": 4.650929770503968e-06, "loss": 0.0018, "num_tokens": 30030049.0, "reward": 0.1875, "reward_std": 0.4493387460708618, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.44933876395225525, "sampling/importance_sampling_ratio/max": 1.3036141753196717, "sampling/importance_sampling_ratio/mean": 1.0001380920410157, "sampling/importance_sampling_ratio/min": 0.7622316956520081, "sampling/sampling_logp_difference/max": 0.30969287157058717, "sampling/sampling_logp_difference/mean": 0.021134368795901536, "step": 6910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 325.1, "completions/max_terminated_length": 307.9, "completions/mean_length": 227.325, "completions/mean_terminated_length": 220.90178833007812, "completions/min_length": 154.3, "completions/min_terminated_length": 154.3, "entropy": 1.0782302975654603, "epoch": 0.6362048358922497, "frac_reward_zero_std": 0.4, "grad_norm": 8.446222305297852, "kl": 0.04903796419966966, "learning_rate": 4.64963736216947e-06, "loss": 0.0024, "num_tokens": 30075451.0, "reward": -0.10625, "reward_std": 0.31606110036373136, "rewards/reward_fn/mean": -0.10625, "rewards/reward_fn/std": 0.31606111228466033, "sampling/importance_sampling_ratio/max": 1.2200359940528869, "sampling/importance_sampling_ratio/mean": 0.9996735095977783, "sampling/importance_sampling_ratio/min": 0.7796869039535522, "sampling/sampling_logp_difference/max": 0.24958672523498535, "sampling/sampling_logp_difference/mean": 0.021077096182852983, "step": 6920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.5, "completions/max_terminated_length": 300.5, "completions/mean_length": 198.1875, "completions/mean_terminated_length": 198.1875, "completions/min_length": 108.2, "completions/min_terminated_length": 108.2, "entropy": 1.0618108380585909, "epoch": 0.637124207042383, "frac_reward_zero_std": 0.3, "grad_norm": 7.563666820526123, "kl": 0.03885188545100391, "learning_rate": 4.648342745985087e-06, "loss": 0.0018, "num_tokens": 30121498.0, "reward": 0.425, "reward_std": 0.3978420317173004, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.3978420555591583, "sampling/importance_sampling_ratio/max": 1.2341456890106202, "sampling/importance_sampling_ratio/mean": 1.0005105316638947, "sampling/importance_sampling_ratio/min": 0.7986541211605072, "sampling/sampling_logp_difference/max": 0.24060747623443604, "sampling/sampling_logp_difference/mean": 0.02072082869708538, "step": 6930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 201.175, "completions/mean_terminated_length": 201.175, "completions/min_length": 116.7, "completions/min_terminated_length": 116.7, "entropy": 1.1433960556983949, "epoch": 0.6380435781925163, "frac_reward_zero_std": 0.0, "grad_norm": 5.469820976257324, "kl": 0.04899288679007441, "learning_rate": 4.647045923280493e-06, "loss": 0.0026, "num_tokens": 30164136.0, "reward": -0.03125, "reward_std": 0.5548639625310898, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.5548639863729476, "sampling/importance_sampling_ratio/max": 1.2946020126342774, "sampling/importance_sampling_ratio/mean": 0.9998938620090485, "sampling/importance_sampling_ratio/min": 0.771307361125946, "sampling/sampling_logp_difference/max": 0.2983285427093506, "sampling/sampling_logp_difference/mean": 0.02288770768791437, "step": 6940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 256.1, "completions/max_terminated_length": 256.1, "completions/mean_length": 182.5625, "completions/mean_terminated_length": 182.5625, "completions/min_length": 121.5, "completions/min_terminated_length": 121.5, "entropy": 1.0454321503639221, "epoch": 0.6389629493426496, "frac_reward_zero_std": 0.1, "grad_norm": 5.0962419509887695, "kl": 0.03612101135076955, "learning_rate": 4.645746895387628e-06, "loss": 0.0017, "num_tokens": 30203525.0, "reward": 0.0125, "reward_std": 0.39266277551651, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.3926627814769745, "sampling/importance_sampling_ratio/max": 1.3027013540267944, "sampling/importance_sampling_ratio/mean": 1.000362628698349, "sampling/importance_sampling_ratio/min": 0.7952675461769104, "sampling/sampling_logp_difference/max": 0.26876828670501707, "sampling/sampling_logp_difference/mean": 0.02092335410416126, "step": 6950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.4, "completions/max_terminated_length": 309.4, "completions/mean_length": 196.6625, "completions/mean_terminated_length": 196.6625, "completions/min_length": 116.4, "completions/min_terminated_length": 116.4, "entropy": 1.0862117640674114, "epoch": 0.6398823204927829, "frac_reward_zero_std": 0.2, "grad_norm": 5.281369209289551, "kl": 0.046541214553872126, "learning_rate": 4.6444456636407e-06, "loss": 0.0023, "num_tokens": 30244858.0, "reward": 0.075, "reward_std": 0.36193286776542666, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.36193287968635557, "sampling/importance_sampling_ratio/max": 1.2642287731170654, "sampling/importance_sampling_ratio/mean": 0.999469417333603, "sampling/importance_sampling_ratio/min": 0.7357144683599472, "sampling/sampling_logp_difference/max": 0.32335125207901, "sampling/sampling_logp_difference/mean": 0.020689988601952793, "step": 6960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.7, "completions/max_terminated_length": 308.7, "completions/mean_length": 193.1875, "completions/mean_terminated_length": 193.1875, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.160147128254175, "epoch": 0.6408016916429162, "frac_reward_zero_std": 0.3, "grad_norm": 7.213428497314453, "kl": 0.06023726573912427, "learning_rate": 4.643142229376178e-06, "loss": 0.003, "num_tokens": 30289065.0, "reward": 0.275, "reward_std": 0.3438155859708786, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.34381560385227206, "sampling/importance_sampling_ratio/max": 1.241119384765625, "sampling/importance_sampling_ratio/mean": 1.0001249015331268, "sampling/importance_sampling_ratio/min": 0.7904399693012237, "sampling/sampling_logp_difference/max": 0.24830453395843505, "sampling/sampling_logp_difference/mean": 0.022625867277383804, "step": 6970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.4, "completions/max_terminated_length": 257.4, "completions/mean_length": 186.9375, "completions/mean_terminated_length": 186.9375, "completions/min_length": 116.6, "completions/min_terminated_length": 116.6, "entropy": 1.1095874726772308, "epoch": 0.6417210627930495, "frac_reward_zero_std": 0.0, "grad_norm": 6.596249103546143, "kl": 0.037959228648105636, "learning_rate": 4.64183659393279e-06, "loss": 0.0023, "num_tokens": 30327364.0, "reward": -0.2, "reward_std": 0.5650873184204102, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.5650873243808746, "sampling/importance_sampling_ratio/max": 1.2830547928810119, "sampling/importance_sampling_ratio/mean": 1.00035280585289, "sampling/importance_sampling_ratio/min": 0.7854923188686371, "sampling/sampling_logp_difference/max": 0.26614352464675906, "sampling/sampling_logp_difference/mean": 0.022300955839455128, "step": 6980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 202.4625, "completions/mean_terminated_length": 202.4625, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.1392049722373485, "epoch": 0.6426404339431828, "frac_reward_zero_std": 0.1, "grad_norm": 6.119923114776611, "kl": 0.04383351393043995, "learning_rate": 4.640528758651532e-06, "loss": 0.0021, "num_tokens": 30369929.0, "reward": 0.05, "reward_std": 0.5186047405004501, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.5186047583818436, "sampling/importance_sampling_ratio/max": 1.342177724838257, "sampling/importance_sampling_ratio/mean": 1.0002257466316222, "sampling/importance_sampling_ratio/min": 0.7912344515323639, "sampling/sampling_logp_difference/max": 0.30578161478042604, "sampling/sampling_logp_difference/mean": 0.021541558764874937, "step": 6990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 331.3, "completions/max_terminated_length": 324.0, "completions/mean_length": 225.4875, "completions/mean_terminated_length": 222.7125, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.0147764183580876, "epoch": 0.6435598050933161, "frac_reward_zero_std": 0.2, "grad_norm": 0.35733526945114136, "kl": 0.03853838726645335, "learning_rate": 4.639218724875655e-06, "loss": 0.002, "num_tokens": 30410288.0, "reward": -0.05625, "reward_std": 0.48227610886096955, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.48227610886096955, "sampling/importance_sampling_ratio/max": 1.3069356441497804, "sampling/importance_sampling_ratio/mean": 0.9995644807815551, "sampling/importance_sampling_ratio/min": 0.7649729728698731, "sampling/sampling_logp_difference/max": 0.31061495542526246, "sampling/sampling_logp_difference/mean": 0.02032538391649723, "step": 7000 }, { "epoch": 0.6435598050933161, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 299.3752181500873, "eval_completions/max_terminated_length": 297.4851657940663, "eval_completions/mean_length": 207.52028795811518, "eval_completions/mean_terminated_length": 207.1254987999615, "eval_completions/min_length": 132.5043630017452, "eval_completions/min_terminated_length": 132.5043630017452, "eval_entropy": 1.1399416261422488, "eval_frac_reward_zero_std": 0.2600349040139616, "eval_kl": 0.04126270575835703, "eval_loss": 0.002148049185052514, "eval_num_tokens": 30410288.0, "eval_reward": 0.03588568935427574, "eval_reward_std": 0.3895306445229116, "eval_rewards/reward_fn/mean": 0.03588568935427574, "eval_rewards/reward_fn/std": 0.38953065565327283, "eval_runtime": 1744.9092, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.2754711324631856, "eval_sampling/importance_sampling_ratio/mean": 1.000052079584378, "eval_sampling/importance_sampling_ratio/min": 0.7804716802080264, "eval_sampling/sampling_logp_difference/max": 0.27688996413199274, "eval_sampling/sampling_logp_difference/mean": 0.021683384202758782, "eval_steps_per_second": 0.041, "step": 7000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 361.9, "completions/max_terminated_length": 361.9, "completions/mean_length": 234.0875, "completions/mean_terminated_length": 234.0875, "completions/min_length": 157.4, "completions/min_terminated_length": 157.4, "entropy": 1.1878170371055603, "epoch": 0.6444791762434495, "frac_reward_zero_std": 0.3, "grad_norm": 0.17004451155662537, "kl": 0.04644565583439544, "learning_rate": 4.637906493950668e-06, "loss": 0.0022, "num_tokens": 30455367.0, "reward": -0.2625, "reward_std": 0.353536981344223, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.353536993265152, "sampling/importance_sampling_ratio/max": 1.2563496351242065, "sampling/importance_sampling_ratio/mean": 0.9998129367828369, "sampling/importance_sampling_ratio/min": 0.7519959330558776, "sampling/sampling_logp_difference/max": 0.3017697691917419, "sampling/sampling_logp_difference/mean": 0.022627398557960986, "step": 7010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 214.725, "completions/mean_terminated_length": 214.725, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.1484743781387805, "epoch": 0.6453985473935828, "frac_reward_zero_std": 0.4, "grad_norm": 0.1319764107465744, "kl": 0.03933565615443513, "learning_rate": 4.636592067224337e-06, "loss": 0.002, "num_tokens": 30499057.0, "reward": 0.075, "reward_std": 0.3130935251712799, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.3130935370922089, "sampling/importance_sampling_ratio/max": 1.261081600189209, "sampling/importance_sampling_ratio/mean": 0.9996011137962342, "sampling/importance_sampling_ratio/min": 0.7873663008213043, "sampling/sampling_logp_difference/max": 0.25997090339660645, "sampling/sampling_logp_difference/mean": 0.022218709625303746, "step": 7020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.9, "completions/max_terminated_length": 335.9, "completions/mean_length": 227.275, "completions/mean_terminated_length": 227.275, "completions/min_length": 149.6, "completions/min_terminated_length": 149.6, "entropy": 1.2338936746120452, "epoch": 0.6463179185437161, "frac_reward_zero_std": 0.2, "grad_norm": 0.18731720745563507, "kl": 0.05541989308549091, "learning_rate": 4.635275446046685e-06, "loss": 0.0027, "num_tokens": 30545191.0, "reward": -0.2125, "reward_std": 0.332049286365509, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.3320492923259735, "sampling/importance_sampling_ratio/max": 1.31266348361969, "sampling/importance_sampling_ratio/mean": 0.999973726272583, "sampling/importance_sampling_ratio/min": 0.7816827774047852, "sampling/sampling_logp_difference/max": 0.29988530874252317, "sampling/sampling_logp_difference/mean": 0.022703452967107296, "step": 7030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 305.7, "completions/max_terminated_length": 305.7, "completions/mean_length": 222.2875, "completions/mean_terminated_length": 222.2875, "completions/min_length": 150.7, "completions/min_terminated_length": 150.7, "entropy": 1.1482506163418293, "epoch": 0.6472372896938494, "frac_reward_zero_std": 0.2, "grad_norm": 4.908675670623779, "kl": 0.04748713891021907, "learning_rate": 4.633956631769986e-06, "loss": 0.0025, "num_tokens": 30591022.0, "reward": -0.125, "reward_std": 0.4153409838676453, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.41534098982810974, "sampling/importance_sampling_ratio/max": 1.2869335412979126, "sampling/importance_sampling_ratio/mean": 0.9994430124759675, "sampling/importance_sampling_ratio/min": 0.7627106249332428, "sampling/sampling_logp_difference/max": 0.31466342210769654, "sampling/sampling_logp_difference/mean": 0.02198565937578678, "step": 7040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 216.75, "completions/mean_terminated_length": 216.75, "completions/min_length": 150.5, "completions/min_terminated_length": 150.5, "entropy": 1.1612841099500657, "epoch": 0.6481566608439827, "frac_reward_zero_std": 0.3, "grad_norm": 6.825497627258301, "kl": 0.05818576728925109, "learning_rate": 4.632635625748768e-06, "loss": 0.0033, "num_tokens": 30632786.0, "reward": 0.175, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.3121665120124817, "sampling/importance_sampling_ratio/mean": 0.9999531984329224, "sampling/importance_sampling_ratio/min": 0.7557978451251983, "sampling/sampling_logp_difference/max": 0.32109909057617186, "sampling/sampling_logp_difference/mean": 0.02272858116775751, "step": 7050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.3, "completions/max_terminated_length": 306.3, "completions/mean_length": 208.4625, "completions/mean_terminated_length": 208.4625, "completions/min_length": 125.6, "completions/min_terminated_length": 125.6, "entropy": 1.0524587467312814, "epoch": 0.649076031994116, "frac_reward_zero_std": 0.4, "grad_norm": 4.919009208679199, "kl": 0.0453869518940337, "learning_rate": 4.631312429339811e-06, "loss": 0.0025, "num_tokens": 30674471.0, "reward": 0.2, "reward_std": 0.25587469935417173, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.25587469935417173, "sampling/importance_sampling_ratio/max": 1.2805373191833496, "sampling/importance_sampling_ratio/mean": 1.00029855966568, "sampling/importance_sampling_ratio/min": 0.7590613961219788, "sampling/sampling_logp_difference/max": 0.3037511229515076, "sampling/sampling_logp_difference/mean": 0.021444643288850783, "step": 7060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 318.2, "completions/max_terminated_length": 295.5, "completions/mean_length": 226.675, "completions/mean_terminated_length": 223.1125015258789, "completions/min_length": 147.9, "completions/min_terminated_length": 147.9, "entropy": 1.148937001824379, "epoch": 0.6499954031442493, "frac_reward_zero_std": 0.3, "grad_norm": 5.626952648162842, "kl": 0.056578563433140515, "learning_rate": 4.6299870439021446e-06, "loss": 0.0026, "num_tokens": 30719565.0, "reward": 0.03125, "reward_std": 0.33991776406764984, "rewards/reward_fn/mean": 0.03125, "rewards/reward_fn/std": 0.33991776406764984, "sampling/importance_sampling_ratio/max": 1.2652281999588013, "sampling/importance_sampling_ratio/mean": 1.0006783246994018, "sampling/importance_sampling_ratio/min": 0.7524769246578217, "sampling/sampling_logp_difference/max": 0.2974590539932251, "sampling/sampling_logp_difference/mean": 0.02250605672597885, "step": 7070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.9, "completions/max_terminated_length": 280.9, "completions/mean_length": 210.5875, "completions/mean_terminated_length": 210.5875, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.0896439947187901, "epoch": 0.6509147742943826, "frac_reward_zero_std": 0.6, "grad_norm": 7.971179485321045, "kl": 0.05408676060615107, "learning_rate": 4.628659470797044e-06, "loss": 0.0025, "num_tokens": 30762316.0, "reward": -0.0875, "reward_std": 0.2270910620689392, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.22709106802940368, "sampling/importance_sampling_ratio/max": 1.2525829911231994, "sampling/importance_sampling_ratio/mean": 1.0000336110591888, "sampling/importance_sampling_ratio/min": 0.7763023734092712, "sampling/sampling_logp_difference/max": 0.26648181676864624, "sampling/sampling_logp_difference/mean": 0.022121440246701242, "step": 7080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.2, "completions/max_terminated_length": 307.2, "completions/mean_length": 213.75, "completions/mean_terminated_length": 213.75, "completions/min_length": 138.4, "completions/min_terminated_length": 138.4, "entropy": 1.1037369690835477, "epoch": 0.6518341454445159, "frac_reward_zero_std": 0.4, "grad_norm": 0.10703787207603455, "kl": 0.04414446228765882, "learning_rate": 4.627329711388033e-06, "loss": 0.0026, "num_tokens": 30807432.0, "reward": 0.0125, "reward_std": 0.261338609457016, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.2613386154174805, "sampling/importance_sampling_ratio/max": 1.2566338777542114, "sampling/importance_sampling_ratio/mean": 1.0004343688488007, "sampling/importance_sampling_ratio/min": 0.7955641984939575, "sampling/sampling_logp_difference/max": 0.2435443878173828, "sampling/sampling_logp_difference/mean": 0.021285582892596723, "step": 7090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 194.5125, "completions/mean_terminated_length": 194.5125, "completions/min_length": 121.2, "completions/min_terminated_length": 121.2, "entropy": 1.1432061351835727, "epoch": 0.6527535165946493, "frac_reward_zero_std": 0.1, "grad_norm": 6.29132604598999, "kl": 0.05914498777128756, "learning_rate": 4.625997767040883e-06, "loss": 0.0031, "num_tokens": 30846545.0, "reward": 0.025, "reward_std": 0.5240216553211212, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.5240216672420501, "sampling/importance_sampling_ratio/max": 1.2507328391075134, "sampling/importance_sampling_ratio/mean": 0.9995620131492615, "sampling/importance_sampling_ratio/min": 0.7924042165279388, "sampling/sampling_logp_difference/max": 0.248442405462265, "sampling/sampling_logp_difference/mean": 0.02318373918533325, "step": 7100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.1, "completions/max_terminated_length": 316.1, "completions/mean_length": 211.9, "completions/mean_terminated_length": 211.9, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.1003998763859273, "epoch": 0.6536728877447826, "frac_reward_zero_std": 0.5, "grad_norm": 0.16145722568035126, "kl": 0.04716645225416869, "learning_rate": 4.624663639123607e-06, "loss": 0.0023, "num_tokens": 30892137.0, "reward": -0.025, "reward_std": 0.27549403011798856, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.27549404203891753, "sampling/importance_sampling_ratio/max": 1.2743995308876037, "sampling/importance_sampling_ratio/mean": 0.9994609892368317, "sampling/importance_sampling_ratio/min": 0.7787565529346466, "sampling/sampling_logp_difference/max": 0.28251625299453736, "sampling/sampling_logp_difference/mean": 0.02230153065174818, "step": 7110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 219.0625, "completions/mean_terminated_length": 219.0625, "completions/min_length": 153.7, "completions/min_terminated_length": 153.7, "entropy": 1.115867978334427, "epoch": 0.6545922588949159, "frac_reward_zero_std": 0.4, "grad_norm": 5.820727825164795, "kl": 0.05551559854648076, "learning_rate": 4.623327329006461e-06, "loss": 0.0028, "num_tokens": 30935966.0, "reward": -0.125, "reward_std": 0.25040294229984283, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.2504029482603073, "sampling/importance_sampling_ratio/max": 1.2652942061424255, "sampling/importance_sampling_ratio/mean": 1.000111174583435, "sampling/importance_sampling_ratio/min": 0.7491200685501098, "sampling/sampling_logp_difference/max": 0.2989262342453003, "sampling/sampling_logp_difference/mean": 0.021689656376838683, "step": 7120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.3, "completions/max_terminated_length": 304.3, "completions/mean_length": 214.1, "completions/mean_terminated_length": 214.1, "completions/min_length": 145.0, "completions/min_terminated_length": 145.0, "entropy": 1.1453317418694495, "epoch": 0.6555116300450492, "frac_reward_zero_std": 0.3, "grad_norm": 6.079078674316406, "kl": 0.05883527337573469, "learning_rate": 4.621988838061942e-06, "loss": 0.0029, "num_tokens": 30978470.0, "reward": -0.05, "reward_std": 0.3423954129219055, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3423954248428345, "sampling/importance_sampling_ratio/max": 1.2732753038406373, "sampling/importance_sampling_ratio/mean": 0.9997225821018219, "sampling/importance_sampling_ratio/min": 0.7677494645118713, "sampling/sampling_logp_difference/max": 0.28570563197135923, "sampling/sampling_logp_difference/mean": 0.021984077244997024, "step": 7130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.2, "completions/max_terminated_length": 304.2, "completions/mean_length": 219.525, "completions/mean_terminated_length": 219.525, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0643950961530209, "epoch": 0.6564310011951825, "frac_reward_zero_std": 0.3, "grad_norm": 4.963262557983398, "kl": 0.045676520210690796, "learning_rate": 4.620648167664789e-06, "loss": 0.0023, "num_tokens": 31020784.0, "reward": 0.0875, "reward_std": 0.3288124233484268, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.3288124293088913, "sampling/importance_sampling_ratio/max": 1.2813105940818788, "sampling/importance_sampling_ratio/mean": 0.9999165058135986, "sampling/importance_sampling_ratio/min": 0.7861571907997131, "sampling/sampling_logp_difference/max": 0.276343834400177, "sampling/sampling_logp_difference/mean": 0.021251189522445202, "step": 7140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.5, "completions/max_terminated_length": 255.5, "completions/mean_length": 186.8125, "completions/mean_terminated_length": 186.8125, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.1050178408622742, "epoch": 0.6573503723453158, "frac_reward_zero_std": 0.3, "grad_norm": 0.1518331915140152, "kl": 0.03278354926151224, "learning_rate": 4.619305319191975e-06, "loss": 0.002, "num_tokens": 31063193.0, "reward": 0.2, "reward_std": 0.40437457263469695, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.40437457859516146, "sampling/importance_sampling_ratio/max": 1.3489197373390198, "sampling/importance_sampling_ratio/mean": 1.0003912448883057, "sampling/importance_sampling_ratio/min": 0.7802765369415283, "sampling/sampling_logp_difference/max": 0.3191341161727905, "sampling/sampling_logp_difference/mean": 0.022332103736698628, "step": 7150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.8, "completions/max_terminated_length": 291.8, "completions/mean_length": 196.4, "completions/mean_terminated_length": 196.4, "completions/min_length": 114.2, "completions/min_terminated_length": 114.2, "entropy": 1.0893623948097229, "epoch": 0.6582697434954491, "frac_reward_zero_std": 0.6, "grad_norm": 0.1559673249721527, "kl": 0.05480624116025865, "learning_rate": 4.617960294022717e-06, "loss": 0.0026, "num_tokens": 31107273.0, "reward": 0.0125, "reward_std": 0.1974882572889328, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.19748826324939728, "sampling/importance_sampling_ratio/max": 1.2473576664924622, "sampling/importance_sampling_ratio/mean": 1.000143676996231, "sampling/importance_sampling_ratio/min": 0.7711097955703735, "sampling/sampling_logp_difference/max": 0.2689287900924683, "sampling/sampling_logp_difference/mean": 0.021457194723188876, "step": 7160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.9, "completions/max_terminated_length": 279.9, "completions/mean_length": 204.0875, "completions/mean_terminated_length": 204.0875, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.0858058348298072, "epoch": 0.6591891146455824, "frac_reward_zero_std": 0.3, "grad_norm": 0.12401717156171799, "kl": 0.04866970150033012, "learning_rate": 4.616613093538461e-06, "loss": 0.0025, "num_tokens": 31152624.0, "reward": 0.375, "reward_std": 0.3483739882707596, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.34837399423122406, "sampling/importance_sampling_ratio/max": 1.2619606018066407, "sampling/importance_sampling_ratio/mean": 1.0003080785274505, "sampling/importance_sampling_ratio/min": 0.7845691561698913, "sampling/sampling_logp_difference/max": 0.2728304147720337, "sampling/sampling_logp_difference/mean": 0.02141085732728243, "step": 7170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 264.8, "completions/max_terminated_length": 264.8, "completions/mean_length": 198.625, "completions/mean_terminated_length": 198.625, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0963338486850263, "epoch": 0.6601084857957157, "frac_reward_zero_std": 0.2, "grad_norm": 0.13831183314323425, "kl": 0.06449027534108609, "learning_rate": 4.615263719122891e-06, "loss": 0.0024, "num_tokens": 31194042.0, "reward": 0.05, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.2703498601913452, "sampling/importance_sampling_ratio/mean": 1.000065553188324, "sampling/importance_sampling_ratio/min": 0.7917282342910766, "sampling/sampling_logp_difference/max": 0.267667293548584, "sampling/sampling_logp_difference/mean": 0.021503900922834874, "step": 7180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.3, "completions/max_terminated_length": 282.3, "completions/mean_length": 190.1, "completions/mean_terminated_length": 190.1, "completions/min_length": 121.8, "completions/min_terminated_length": 121.8, "entropy": 1.0183037899434566, "epoch": 0.6610278569458491, "frac_reward_zero_std": 0.6, "grad_norm": 0.13874197006225586, "kl": 0.04413658081321046, "learning_rate": 4.613912172161923e-06, "loss": 0.0023, "num_tokens": 31235002.0, "reward": -0.1, "reward_std": 0.1414213538169861, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.1414213538169861, "sampling/importance_sampling_ratio/max": 1.2225879907608033, "sampling/importance_sampling_ratio/mean": 1.0004299223423003, "sampling/importance_sampling_ratio/min": 0.7879893839359283, "sampling/sampling_logp_difference/max": 0.2429264783859253, "sampling/sampling_logp_difference/mean": 0.021308378875255586, "step": 7190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.9, "completions/max_terminated_length": 274.9, "completions/mean_length": 193.475, "completions/mean_terminated_length": 193.475, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 0.9930923976004123, "epoch": 0.6619472280959824, "frac_reward_zero_std": 0.2, "grad_norm": 6.2710089683532715, "kl": 0.04803750627906993, "learning_rate": 4.612558454043702e-06, "loss": 0.0026, "num_tokens": 31277808.0, "reward": 0.05, "reward_std": 0.3820319950580597, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.38203200697898865, "sampling/importance_sampling_ratio/max": 1.261991024017334, "sampling/importance_sampling_ratio/mean": 1.0001124501228333, "sampling/importance_sampling_ratio/min": 0.783654797077179, "sampling/sampling_logp_difference/max": 0.2786588311195374, "sampling/sampling_logp_difference/mean": 0.020226932782679796, "step": 7200 }, { "epoch": 0.6619472280959824, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 284.48342059336824, "eval_completions/max_terminated_length": 283.14659685863876, "eval_completions/mean_length": 198.51221640488657, "eval_completions/mean_terminated_length": 198.1925643734491, "eval_completions/min_length": 127.1239092495637, "eval_completions/min_terminated_length": 127.1239092495637, "eval_entropy": 1.064619136990469, "eval_frac_reward_zero_std": 0.31239092495637, "eval_kl": 0.0498309459010656, "eval_loss": 0.0026732217520475388, "eval_num_tokens": 31277808.0, "eval_reward": 0.06489965095986039, "eval_reward_std": 0.3346925450141101, "eval_rewards/reward_fn/mean": 0.06489965095986039, "eval_rewards/reward_fn/std": 0.3346925534398976, "eval_runtime": 1667.7373, "eval_samples_per_second": 0.344, "eval_sampling/importance_sampling_ratio/max": 1.2635605285305003, "eval_sampling/importance_sampling_ratio/mean": 1.0000404038354365, "eval_sampling/importance_sampling_ratio/min": 0.7779044504773138, "eval_sampling/sampling_logp_difference/max": 0.27392785368165423, "eval_sampling/sampling_logp_difference/mean": 0.02119820735971549, "eval_steps_per_second": 0.043, "step": 7200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.6, "completions/max_terminated_length": 303.6, "completions/mean_length": 211.025, "completions/mean_terminated_length": 211.025, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0055617660284042, "epoch": 0.6628665992461157, "frac_reward_zero_std": 0.4, "grad_norm": 7.493412017822266, "kl": 0.05891411122283898, "learning_rate": 4.611202566158607e-06, "loss": 0.0029, "num_tokens": 31321546.0, "reward": 0.05, "reward_std": 0.29661122560501096, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.29661123752593993, "sampling/importance_sampling_ratio/max": 1.28914954662323, "sampling/importance_sampling_ratio/mean": 1.000070708990097, "sampling/importance_sampling_ratio/min": 0.7822191119194031, "sampling/sampling_logp_difference/max": 0.29209272265434266, "sampling/sampling_logp_difference/mean": 0.020206468179821968, "step": 7210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 209.875, "completions/mean_terminated_length": 209.875, "completions/min_length": 122.9, "completions/min_terminated_length": 122.9, "entropy": 1.0289199404418468, "epoch": 0.663785970396249, "frac_reward_zero_std": 0.2, "grad_norm": 5.5265278816223145, "kl": 0.04613122205482796, "learning_rate": 4.609844509899243e-06, "loss": 0.0026, "num_tokens": 31366680.0, "reward": 0.0, "reward_std": 0.3265853762626648, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3265853762626648, "sampling/importance_sampling_ratio/max": 1.2539484739303588, "sampling/importance_sampling_ratio/mean": 1.0000987112522126, "sampling/importance_sampling_ratio/min": 0.7771053314208984, "sampling/sampling_logp_difference/max": 0.26145464181900024, "sampling/sampling_logp_difference/mean": 0.02051695678383112, "step": 7220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 277.1, "completions/max_terminated_length": 267.1, "completions/mean_length": 197.225, "completions/mean_terminated_length": 194.93214416503906, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.0228578120470047, "epoch": 0.6647053415463823, "frac_reward_zero_std": 0.1, "grad_norm": 0.16433332860469818, "kl": 0.04508859519264661, "learning_rate": 4.6084842866604426e-06, "loss": 0.0024, "num_tokens": 31407482.0, "reward": 0.21875, "reward_std": 0.37037220001220705, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.3703722059726715, "sampling/importance_sampling_ratio/max": 1.2638112306594849, "sampling/importance_sampling_ratio/mean": 0.9997063279151917, "sampling/importance_sampling_ratio/min": 0.7383287847042084, "sampling/sampling_logp_difference/max": 0.3096863508224487, "sampling/sampling_logp_difference/mean": 0.021090848557651042, "step": 7230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 205.8, "completions/mean_terminated_length": 205.8, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0304714642465114, "epoch": 0.6656247126965156, "frac_reward_zero_std": 0.4, "grad_norm": 0.23519223928451538, "kl": 0.05572328233392909, "learning_rate": 4.607121897839264e-06, "loss": 0.003, "num_tokens": 31453018.0, "reward": 0.1125, "reward_std": 0.2756610929965973, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.27566109895706176, "sampling/importance_sampling_ratio/max": 1.2423957347869874, "sampling/importance_sampling_ratio/mean": 1.0003386080265044, "sampling/importance_sampling_ratio/min": 0.7729002416133881, "sampling/sampling_logp_difference/max": 0.2709189295768738, "sampling/sampling_logp_difference/mean": 0.02015252448618412, "step": 7240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.1, "completions/max_terminated_length": 275.1, "completions/mean_length": 204.5625, "completions/mean_terminated_length": 204.5625, "completions/min_length": 134.4, "completions/min_terminated_length": 134.4, "entropy": 1.0800440780818463, "epoch": 0.6665440838466489, "frac_reward_zero_std": 0.5, "grad_norm": 0.17569485306739807, "kl": 0.06364223085111007, "learning_rate": 4.605757344834989e-06, "loss": 0.003, "num_tokens": 31496543.0, "reward": 0.175, "reward_std": 0.20411193668842315, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.20411194264888763, "sampling/importance_sampling_ratio/max": 1.2897516369819642, "sampling/importance_sampling_ratio/mean": 1.0006681382656097, "sampling/importance_sampling_ratio/min": 0.7404118180274963, "sampling/sampling_logp_difference/max": 0.3246050179004669, "sampling/sampling_logp_difference/mean": 0.02191933020949364, "step": 7250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.9, "completions/max_terminated_length": 271.9, "completions/mean_length": 186.1875, "completions/mean_terminated_length": 186.1875, "completions/min_length": 120.4, "completions/min_terminated_length": 120.4, "entropy": 1.0374748148024082, "epoch": 0.6674634549967822, "frac_reward_zero_std": 0.4, "grad_norm": 6.727726936340332, "kl": 0.04095509689068422, "learning_rate": 4.604390629049125e-06, "loss": 0.0025, "num_tokens": 31540822.0, "reward": 0.075, "reward_std": 0.3282748609781265, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.328274866938591, "sampling/importance_sampling_ratio/max": 1.3150431394577027, "sampling/importance_sampling_ratio/mean": 1.0001801908016206, "sampling/importance_sampling_ratio/min": 0.7840251088142395, "sampling/sampling_logp_difference/max": 0.30605231523513793, "sampling/sampling_logp_difference/mean": 0.021714341826736927, "step": 7260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.5, "completions/max_terminated_length": 311.5, "completions/mean_length": 213.075, "completions/mean_terminated_length": 213.075, "completions/min_length": 130.8, "completions/min_terminated_length": 130.8, "entropy": 1.042024876177311, "epoch": 0.6683828261469155, "frac_reward_zero_std": 0.0, "grad_norm": 5.499321937561035, "kl": 0.04465296705020592, "learning_rate": 4.603021751885397e-06, "loss": 0.0022, "num_tokens": 31583180.0, "reward": -0.1125, "reward_std": 0.44357921183109283, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.4435792237520218, "sampling/importance_sampling_ratio/max": 1.2784564971923829, "sampling/importance_sampling_ratio/mean": 0.9999276459217071, "sampling/importance_sampling_ratio/min": 0.7687676250934601, "sampling/sampling_logp_difference/max": 0.2863020122051239, "sampling/sampling_logp_difference/mean": 0.021744346618652342, "step": 7270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.1, "completions/max_terminated_length": 280.1, "completions/mean_length": 195.0, "completions/mean_terminated_length": 195.0, "completions/min_length": 130.5, "completions/min_terminated_length": 130.5, "entropy": 0.9949001189321279, "epoch": 0.6693021972970489, "frac_reward_zero_std": 0.3, "grad_norm": 5.6927490234375, "kl": 0.055850255314726385, "learning_rate": 4.601650714749752e-06, "loss": 0.0027, "num_tokens": 31622796.0, "reward": -0.15, "reward_std": 0.42896814048290255, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.42896815240383146, "sampling/importance_sampling_ratio/max": 1.36483793258667, "sampling/importance_sampling_ratio/mean": 1.0003822922706604, "sampling/importance_sampling_ratio/min": 0.7705457389354706, "sampling/sampling_logp_difference/max": 0.327013099193573, "sampling/sampling_logp_difference/mean": 0.02087930031120777, "step": 7280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.6, "completions/max_terminated_length": 255.6, "completions/mean_length": 189.6375, "completions/mean_terminated_length": 189.6375, "completions/min_length": 123.9, "completions/min_terminated_length": 123.9, "entropy": 1.0203311927616596, "epoch": 0.6702215684471822, "frac_reward_zero_std": 0.4, "grad_norm": 0.1278441995382309, "kl": 0.04536830126889981, "learning_rate": 4.600277519050355e-06, "loss": 0.0019, "num_tokens": 31662711.0, "reward": 0.0, "reward_std": 0.27773033976554873, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.2777303636074066, "sampling/importance_sampling_ratio/max": 1.226729953289032, "sampling/importance_sampling_ratio/mean": 0.9999350786209107, "sampling/importance_sampling_ratio/min": 0.8006794214248657, "sampling/sampling_logp_difference/max": 0.23733015060424806, "sampling/sampling_logp_difference/mean": 0.020400801673531532, "step": 7290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 203.6375, "completions/mean_terminated_length": 203.6375, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.1139685682952405, "epoch": 0.6711409395973155, "frac_reward_zero_std": 0.4, "grad_norm": 0.13240955770015717, "kl": 0.051546607888303696, "learning_rate": 4.598902166197589e-06, "loss": 0.0025, "num_tokens": 31706122.0, "reward": -0.14375, "reward_std": 0.2735445201396942, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.2735445320606232, "sampling/importance_sampling_ratio/max": 1.3180628299713135, "sampling/importance_sampling_ratio/mean": 1.0000078141689301, "sampling/importance_sampling_ratio/min": 0.7910449624061584, "sampling/sampling_logp_difference/max": 0.2981088161468506, "sampling/sampling_logp_difference/mean": 0.021368534304201603, "step": 7300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 318.2, "completions/max_terminated_length": 296.1, "completions/mean_length": 210.575, "completions/mean_terminated_length": 206.87857208251953, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.1729208558797837, "epoch": 0.6720603107474488, "frac_reward_zero_std": 0.6, "grad_norm": 3.9134163856506348, "kl": 0.04998117060167715, "learning_rate": 4.597524657604049e-06, "loss": 0.0024, "num_tokens": 31753496.0, "reward": -0.01875, "reward_std": 0.1864342659711838, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.18643427193164824, "sampling/importance_sampling_ratio/max": 1.2764506816864014, "sampling/importance_sampling_ratio/mean": 1.0004078209400178, "sampling/importance_sampling_ratio/min": 0.7668221116065979, "sampling/sampling_logp_difference/max": 0.2980937361717224, "sampling/sampling_logp_difference/mean": 0.02186319902539253, "step": 7310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.9, "completions/max_terminated_length": 297.9, "completions/mean_length": 215.1625, "completions/mean_terminated_length": 215.1625, "completions/min_length": 153.2, "completions/min_terminated_length": 153.2, "entropy": 1.0700261563062667, "epoch": 0.6729796818975821, "frac_reward_zero_std": 0.4, "grad_norm": 0.11135904490947723, "kl": 0.05424491100711748, "learning_rate": 4.5961449946845495e-06, "loss": 0.002, "num_tokens": 31799517.0, "reward": 0.1625, "reward_std": 0.28516874015331267, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.28516875207424164, "sampling/importance_sampling_ratio/max": 1.2372430443763733, "sampling/importance_sampling_ratio/mean": 1.0000016927719115, "sampling/importance_sampling_ratio/min": 0.790747195482254, "sampling/sampling_logp_difference/max": 0.24547867774963378, "sampling/sampling_logp_difference/mean": 0.020578095875680446, "step": 7320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 318.1, "completions/max_terminated_length": 318.1, "completions/mean_length": 220.6625, "completions/mean_terminated_length": 220.6625, "completions/min_length": 134.7, "completions/min_terminated_length": 134.7, "entropy": 1.1330191358923911, "epoch": 0.6738990530477154, "frac_reward_zero_std": 0.4, "grad_norm": 0.12394562363624573, "kl": 0.04857685996685177, "learning_rate": 4.594763178856113e-06, "loss": 0.0023, "num_tokens": 31844370.0, "reward": 0.0, "reward_std": 0.25040294229984283, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.2504029482603073, "sampling/importance_sampling_ratio/max": 1.2333351254463196, "sampling/importance_sampling_ratio/mean": 1.0001760303974152, "sampling/importance_sampling_ratio/min": 0.7773239195346833, "sampling/sampling_logp_difference/max": 0.2579160749912262, "sampling/sampling_logp_difference/mean": 0.022077248059213163, "step": 7330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 320.6, "completions/max_terminated_length": 320.6, "completions/mean_length": 232.1125, "completions/mean_terminated_length": 232.1125, "completions/min_length": 159.4, "completions/min_terminated_length": 159.4, "entropy": 1.0977382034063339, "epoch": 0.6748184241978487, "frac_reward_zero_std": 0.6, "grad_norm": 6.417923450469971, "kl": 0.05118432566523552, "learning_rate": 4.593379211537977e-06, "loss": 0.0027, "num_tokens": 31889987.0, "reward": 0.05, "reward_std": 0.2292030245065689, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.22920303642749787, "sampling/importance_sampling_ratio/max": 1.2419498562812805, "sampling/importance_sampling_ratio/mean": 1.000518250465393, "sampling/importance_sampling_ratio/min": 0.7248602390289307, "sampling/sampling_logp_difference/max": 0.3369430899620056, "sampling/sampling_logp_difference/mean": 0.02117815352976322, "step": 7340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.2, "completions/max_terminated_length": 317.2, "completions/mean_length": 222.5, "completions/mean_terminated_length": 222.5, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 1.1368062272667885, "epoch": 0.675737795347982, "frac_reward_zero_std": 0.6, "grad_norm": 0.15781785547733307, "kl": 0.034471037273760885, "learning_rate": 4.591993094151586e-06, "loss": 0.0016, "num_tokens": 31937627.0, "reward": 0.3375, "reward_std": 0.1595182627439499, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.1595182627439499, "sampling/importance_sampling_ratio/max": 1.267795467376709, "sampling/importance_sampling_ratio/mean": 0.9997016847133636, "sampling/importance_sampling_ratio/min": 0.7713879704475403, "sampling/sampling_logp_difference/max": 0.2952385932207108, "sampling/sampling_logp_difference/mean": 0.02182802725583315, "step": 7350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.5, "completions/max_terminated_length": 294.5, "completions/mean_length": 215.725, "completions/mean_terminated_length": 215.725, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 1.0568891540169716, "epoch": 0.6766571664981152, "frac_reward_zero_std": 0.3, "grad_norm": 5.203394412994385, "kl": 0.05546131924493238, "learning_rate": 4.590604828120593e-06, "loss": 0.0027, "num_tokens": 31981757.0, "reward": -0.05, "reward_std": 0.3423954129219055, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3423954248428345, "sampling/importance_sampling_ratio/max": 1.2456037282943726, "sampling/importance_sampling_ratio/mean": 1.000145787000656, "sampling/importance_sampling_ratio/min": 0.795643550157547, "sampling/sampling_logp_difference/max": 0.23592472672462464, "sampling/sampling_logp_difference/mean": 0.020911729894578457, "step": 7360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.5, "completions/max_terminated_length": 273.5, "completions/mean_length": 211.875, "completions/mean_terminated_length": 211.875, "completions/min_length": 151.1, "completions/min_terminated_length": 151.1, "entropy": 0.9900574635714292, "epoch": 0.6775765376482485, "frac_reward_zero_std": 0.2, "grad_norm": 4.566556453704834, "kl": 0.049262395867845046, "learning_rate": 4.589214414870858e-06, "loss": 0.0027, "num_tokens": 32029371.0, "reward": 0.2125, "reward_std": 0.300939616560936, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.300939616560936, "sampling/importance_sampling_ratio/max": 1.2418309807777406, "sampling/importance_sampling_ratio/mean": 1.000444734096527, "sampling/importance_sampling_ratio/min": 0.7783786952495575, "sampling/sampling_logp_difference/max": 0.2679599463939667, "sampling/sampling_logp_difference/mean": 0.020084110647439958, "step": 7370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.9, "completions/max_terminated_length": 299.9, "completions/mean_length": 205.7875, "completions/mean_terminated_length": 205.7875, "completions/min_length": 128.4, "completions/min_terminated_length": 128.4, "entropy": 0.9073899272829294, "epoch": 0.678495908798382, "frac_reward_zero_std": 0.2, "grad_norm": 3.5518264770507812, "kl": 0.036938689762610014, "learning_rate": 4.58782185583045e-06, "loss": 0.0019, "num_tokens": 32072874.0, "reward": 0.125, "reward_std": 0.4173794865608215, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.41737951040267945, "sampling/importance_sampling_ratio/max": 1.2959476947784423, "sampling/importance_sampling_ratio/mean": 1.0002209484577178, "sampling/importance_sampling_ratio/min": 0.7928396224975586, "sampling/sampling_logp_difference/max": 0.2790764331817627, "sampling/sampling_logp_difference/mean": 0.01888665952719748, "step": 7380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 192.5625, "completions/mean_terminated_length": 192.5625, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0405104793608189, "epoch": 0.6794152799485152, "frac_reward_zero_std": 0.5, "grad_norm": 0.13286620378494263, "kl": 0.04277502251788974, "learning_rate": 4.5864271524296355e-06, "loss": 0.0021, "num_tokens": 32112607.0, "reward": 0.3375, "reward_std": 0.21504760384559632, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.2150476098060608, "sampling/importance_sampling_ratio/max": 1.2894740223884582, "sampling/importance_sampling_ratio/mean": 1.0004708468914032, "sampling/importance_sampling_ratio/min": 0.7762864589691162, "sampling/sampling_logp_difference/max": 0.2877599477767944, "sampling/sampling_logp_difference/mean": 0.020924269407987594, "step": 7390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.1, "completions/max_terminated_length": 312.1, "completions/mean_length": 202.5875, "completions/mean_terminated_length": 202.5875, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0661517322063445, "epoch": 0.6803346510986485, "frac_reward_zero_std": 0.6, "grad_norm": 0.13939781486988068, "kl": 0.06156819289317354, "learning_rate": 4.585030306100887e-06, "loss": 0.0031, "num_tokens": 32153334.0, "reward": 0.0, "reward_std": 0.1414213538169861, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.1414213538169861, "sampling/importance_sampling_ratio/max": 1.2577814698219298, "sampling/importance_sampling_ratio/mean": 0.9998873054981232, "sampling/importance_sampling_ratio/min": 0.7967040300369262, "sampling/sampling_logp_difference/max": 0.2475142478942871, "sampling/sampling_logp_difference/mean": 0.021554626151919366, "step": 7400 }, { "epoch": 0.6803346510986485, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 290.28970331588135, "eval_completions/max_terminated_length": 289.32111692844677, "eval_completions/mean_length": 205.44742582897032, "eval_completions/mean_terminated_length": 205.1254052500034, "eval_completions/min_length": 134.56020942408378, "eval_completions/min_terminated_length": 134.56020942408378, "eval_entropy": 1.0888234543342656, "eval_frac_reward_zero_std": 0.343804537521815, "eval_kl": 0.05033627688806719, "eval_loss": 0.0026131882332265377, "eval_num_tokens": 32153334.0, "eval_reward": 0.061845549738219895, "eval_reward_std": 0.31748389781665637, "eval_rewards/reward_fn/mean": 0.061845549738219895, "eval_rewards/reward_fn/std": 0.31748390780277486, "eval_runtime": 1698.8002, "eval_samples_per_second": 0.337, "eval_sampling/importance_sampling_ratio/max": 1.2657911800381192, "eval_sampling/importance_sampling_ratio/mean": 1.0000523089530373, "eval_sampling/importance_sampling_ratio/min": 0.7797359454902263, "eval_sampling/sampling_logp_difference/max": 0.273970496176009, "eval_sampling/sampling_logp_difference/mean": 0.02093242133852179, "eval_steps_per_second": 0.042, "step": 7400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.1, "completions/max_terminated_length": 268.1, "completions/mean_length": 201.4, "completions/mean_terminated_length": 201.4, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.1320892922580241, "epoch": 0.6812540222487818, "frac_reward_zero_std": 0.2, "grad_norm": 7.608371734619141, "kl": 0.05768209486268461, "learning_rate": 4.5836313182788804e-06, "loss": 0.0029, "num_tokens": 32194814.0, "reward": 0.2875, "reward_std": 0.4007413417100906, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.40074135959148405, "sampling/importance_sampling_ratio/max": 1.2715680241584777, "sampling/importance_sampling_ratio/mean": 0.9999514818191528, "sampling/importance_sampling_ratio/min": 0.7736750602722168, "sampling/sampling_logp_difference/max": 0.27415614724159243, "sampling/sampling_logp_difference/mean": 0.022923282533884048, "step": 7410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.9, "completions/max_terminated_length": 303.9, "completions/mean_length": 210.825, "completions/mean_terminated_length": 206.81964416503905, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.0314208410680294, "epoch": 0.6821733933989151, "frac_reward_zero_std": 0.4, "grad_norm": 0.2035738080739975, "kl": 0.06656519437674432, "learning_rate": 4.582230190400486e-06, "loss": 0.0035, "num_tokens": 32241760.0, "reward": -0.03125, "reward_std": 0.26260885298252107, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.26260886490345003, "sampling/importance_sampling_ratio/max": 1.26887948513031, "sampling/importance_sampling_ratio/mean": 1.0000440657138825, "sampling/importance_sampling_ratio/min": 0.7858240187168122, "sampling/sampling_logp_difference/max": 0.25771403312683105, "sampling/sampling_logp_difference/mean": 0.020971113443374635, "step": 7420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.3, "completions/max_terminated_length": 289.3, "completions/mean_length": 193.475, "completions/mean_terminated_length": 193.475, "completions/min_length": 115.1, "completions/min_terminated_length": 115.1, "entropy": 0.9514463262632489, "epoch": 0.6830927645490484, "frac_reward_zero_std": 0.3, "grad_norm": 5.9331817626953125, "kl": 0.06294347807997838, "learning_rate": 4.580826923904774e-06, "loss": 0.0034, "num_tokens": 32285414.0, "reward": 0.1125, "reward_std": 0.35585459768772126, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.35585459768772126, "sampling/importance_sampling_ratio/max": 1.2565185666084289, "sampling/importance_sampling_ratio/mean": 0.9996947526931763, "sampling/importance_sampling_ratio/min": 0.7628743946552277, "sampling/sampling_logp_difference/max": 0.28076925277709963, "sampling/sampling_logp_difference/mean": 0.01992936562746763, "step": 7430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 283.9, "completions/max_terminated_length": 266.1, "completions/mean_length": 185.7375, "completions/mean_terminated_length": 181.9607147216797, "completions/min_length": 92.1, "completions/min_terminated_length": 92.1, "entropy": 1.0744081519544124, "epoch": 0.6840121356991817, "frac_reward_zero_std": 0.6, "grad_norm": 8.312846183776855, "kl": 0.03791194562218152, "learning_rate": 4.579421520233013e-06, "loss": 0.0022, "num_tokens": 32328881.0, "reward": 0.04375, "reward_std": 0.21255126893520354, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.21255126893520354, "sampling/importance_sampling_ratio/max": 1.2457831263542176, "sampling/importance_sampling_ratio/mean": 1.0004271984100341, "sampling/importance_sampling_ratio/min": 0.80016610622406, "sampling/sampling_logp_difference/max": 0.23352012634277344, "sampling/sampling_logp_difference/mean": 0.0215287521481514, "step": 7440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 202.575, "completions/mean_terminated_length": 202.575, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.132238607108593, "epoch": 0.684931506849315, "frac_reward_zero_std": 0.3, "grad_norm": 7.909250736236572, "kl": 0.04190785058890469, "learning_rate": 4.578013980828664e-06, "loss": 0.0017, "num_tokens": 32376975.0, "reward": 0.1625, "reward_std": 0.48442131876945493, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.48442131876945493, "sampling/importance_sampling_ratio/max": 1.251918911933899, "sampling/importance_sampling_ratio/mean": 0.9994283258914948, "sampling/importance_sampling_ratio/min": 0.7680756390094757, "sampling/sampling_logp_difference/max": 0.295600438117981, "sampling/sampling_logp_difference/mean": 0.02196041587740183, "step": 7450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 338.1, "completions/max_terminated_length": 338.1, "completions/mean_length": 228.4, "completions/mean_terminated_length": 228.4, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.2107975255697965, "epoch": 0.6858508779994483, "frac_reward_zero_std": 0.1, "grad_norm": 0.11257170140743256, "kl": 0.05027083324966952, "learning_rate": 4.5766043071373845e-06, "loss": 0.0028, "num_tokens": 32421679.0, "reward": -0.025, "reward_std": 0.3745658665895462, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.37456587255001067, "sampling/importance_sampling_ratio/max": 1.3062814235687257, "sampling/importance_sampling_ratio/mean": 0.9996409952640534, "sampling/importance_sampling_ratio/min": 0.7797921299934387, "sampling/sampling_logp_difference/max": 0.2978152811527252, "sampling/sampling_logp_difference/mean": 0.023141766153275966, "step": 7460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 322.8, "completions/max_terminated_length": 319.2, "completions/mean_length": 205.775, "completions/mean_terminated_length": 202.9357147216797, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 1.064218432456255, "epoch": 0.6867702491495817, "frac_reward_zero_std": 0.3, "grad_norm": 5.38142204284668, "kl": 0.04115309783956036, "learning_rate": 4.57519250060702e-06, "loss": 0.0019, "num_tokens": 32464917.0, "reward": 0.03125, "reward_std": 0.34506991803646087, "rewards/reward_fn/mean": 0.03125, "rewards/reward_fn/std": 0.34506991803646087, "sampling/importance_sampling_ratio/max": 1.2449893236160279, "sampling/importance_sampling_ratio/mean": 1.000265634059906, "sampling/importance_sampling_ratio/min": 0.7682155489921569, "sampling/sampling_logp_difference/max": 0.2762560248374939, "sampling/sampling_logp_difference/mean": 0.021231625229120255, "step": 7470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.1, "completions/max_terminated_length": 295.1, "completions/mean_length": 198.3125, "completions/mean_terminated_length": 198.3125, "completions/min_length": 119.3, "completions/min_terminated_length": 119.3, "entropy": 1.0488322123885154, "epoch": 0.687689620299715, "frac_reward_zero_std": 0.1, "grad_norm": 7.424997806549072, "kl": 0.03883749302476645, "learning_rate": 4.573778562687612e-06, "loss": 0.0021, "num_tokens": 32506270.0, "reward": 0.3125, "reward_std": 0.44697387516498566, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.44697387516498566, "sampling/importance_sampling_ratio/max": 1.2687963485717773, "sampling/importance_sampling_ratio/mean": 0.9997938632965088, "sampling/importance_sampling_ratio/min": 0.7802180767059326, "sampling/sampling_logp_difference/max": 0.26483807563781736, "sampling/sampling_logp_difference/mean": 0.021511211432516573, "step": 7480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 219.975, "completions/mean_terminated_length": 219.975, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 1.1070709958672524, "epoch": 0.6886089914498483, "frac_reward_zero_std": 0.3, "grad_norm": 5.352287769317627, "kl": 0.04235333006363362, "learning_rate": 4.572362494831386e-06, "loss": 0.0021, "num_tokens": 32549756.0, "reward": -0.0625, "reward_std": 0.3093190997838974, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.30931911170482634, "sampling/importance_sampling_ratio/max": 1.2543426632881165, "sampling/importance_sampling_ratio/mean": 1.0003794074058532, "sampling/importance_sampling_ratio/min": 0.7820522606372833, "sampling/sampling_logp_difference/max": 0.26365768909454346, "sampling/sampling_logp_difference/mean": 0.021890545822679998, "step": 7490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.1, "completions/max_terminated_length": 273.1, "completions/mean_length": 202.05, "completions/mean_terminated_length": 202.05, "completions/min_length": 123.3, "completions/min_terminated_length": 123.3, "entropy": 1.1775474093854428, "epoch": 0.6895283625999816, "frac_reward_zero_std": 0.3, "grad_norm": 6.267361640930176, "kl": 0.058694390836171804, "learning_rate": 4.5709442984927564e-06, "loss": 0.0032, "num_tokens": 32592440.0, "reward": -0.0125, "reward_std": 0.4013115704059601, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4013115882873535, "sampling/importance_sampling_ratio/max": 1.2642083883285522, "sampling/importance_sampling_ratio/mean": 0.9999862372875213, "sampling/importance_sampling_ratio/min": 0.7694486200809478, "sampling/sampling_logp_difference/max": 0.2832889914512634, "sampling/sampling_logp_difference/mean": 0.022359587252140045, "step": 7500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 341.2, "completions/max_terminated_length": 341.2, "completions/mean_length": 227.1625, "completions/mean_terminated_length": 227.1625, "completions/min_length": 162.7, "completions/min_terminated_length": 162.7, "entropy": 1.1424855321645737, "epoch": 0.6904477337501149, "frac_reward_zero_std": 0.3, "grad_norm": 0.14225175976753235, "kl": 0.051977888692636046, "learning_rate": 4.5695239751283275e-06, "loss": 0.0021, "num_tokens": 32640413.0, "reward": -0.15, "reward_std": 0.3202547669410706, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3202547788619995, "sampling/importance_sampling_ratio/max": 1.2743274807929992, "sampling/importance_sampling_ratio/mean": 0.9995707333087921, "sampling/importance_sampling_ratio/min": 0.7844434559345246, "sampling/sampling_logp_difference/max": 0.2723142385482788, "sampling/sampling_logp_difference/mean": 0.021843813173472882, "step": 7510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.8, "completions/max_terminated_length": 306.8, "completions/mean_length": 217.2875, "completions/mean_terminated_length": 217.2875, "completions/min_length": 148.4, "completions/min_terminated_length": 148.4, "entropy": 1.1221212826669216, "epoch": 0.6913671049002482, "frac_reward_zero_std": 0.0, "grad_norm": 5.902036190032959, "kl": 0.04671697786543518, "learning_rate": 4.568101526196882e-06, "loss": 0.0017, "num_tokens": 32684084.0, "reward": 0.2625, "reward_std": 0.4355591177940369, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.4355591356754303, "sampling/importance_sampling_ratio/max": 1.2734981060028077, "sampling/importance_sampling_ratio/mean": 0.999754786491394, "sampling/importance_sampling_ratio/min": 0.7822802126407623, "sampling/sampling_logp_difference/max": 0.27042909860610964, "sampling/sampling_logp_difference/mean": 0.021833948232233523, "step": 7520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.1, "completions/max_terminated_length": 295.1, "completions/mean_length": 211.9, "completions/mean_terminated_length": 211.9, "completions/min_length": 149.4, "completions/min_terminated_length": 149.4, "entropy": 1.0731165552511812, "epoch": 0.6922864760503815, "frac_reward_zero_std": 0.3, "grad_norm": 6.176202297210693, "kl": 0.038268228416563944, "learning_rate": 4.566676953159392e-06, "loss": 0.0015, "num_tokens": 32729092.0, "reward": 0.0875, "reward_std": 0.3311904340982437, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.3311904460191727, "sampling/importance_sampling_ratio/max": 1.265696358680725, "sampling/importance_sampling_ratio/mean": 1.0003490507602693, "sampling/importance_sampling_ratio/min": 0.7816310703754425, "sampling/sampling_logp_difference/max": 0.26671168208122253, "sampling/sampling_logp_difference/mean": 0.020541245769709348, "step": 7530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.4, "completions/max_terminated_length": 303.4, "completions/mean_length": 225.2375, "completions/mean_terminated_length": 225.2375, "completions/min_length": 173.1, "completions/min_terminated_length": 173.1, "entropy": 1.1765621855854989, "epoch": 0.6932058472005148, "frac_reward_zero_std": 0.3, "grad_norm": 0.13530386984348297, "kl": 0.039450769958784805, "learning_rate": 4.565250257479004e-06, "loss": 0.0019, "num_tokens": 32774247.0, "reward": 0.25, "reward_std": 0.40760951340198515, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.40760951936244966, "sampling/importance_sampling_ratio/max": 1.2988221883773803, "sampling/importance_sampling_ratio/mean": 1.000485473871231, "sampling/importance_sampling_ratio/min": 0.7841191112995147, "sampling/sampling_logp_difference/max": 0.2745662212371826, "sampling/sampling_logp_difference/mean": 0.02200355865061283, "step": 7540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.7, "completions/max_terminated_length": 298.7, "completions/mean_length": 222.45, "completions/mean_terminated_length": 222.45, "completions/min_length": 164.0, "completions/min_terminated_length": 164.0, "entropy": 1.1780326135456562, "epoch": 0.6941252183506481, "frac_reward_zero_std": 0.4, "grad_norm": 0.1095915138721466, "kl": 0.05131513993255794, "learning_rate": 4.563821440621055e-06, "loss": 0.0024, "num_tokens": 32819875.0, "reward": -0.1375, "reward_std": 0.30439275205135347, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.3043927580118179, "sampling/importance_sampling_ratio/max": 1.2823341965675354, "sampling/importance_sampling_ratio/mean": 0.9999317348003387, "sampling/importance_sampling_ratio/min": 0.7296609789133072, "sampling/sampling_logp_difference/max": 0.3403995394706726, "sampling/sampling_logp_difference/mean": 0.022616959176957607, "step": 7550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 295.7, "completions/max_terminated_length": 268.4, "completions/mean_length": 199.9875, "completions/mean_terminated_length": 195.71428680419922, "completions/min_length": 133.5, "completions/min_terminated_length": 133.5, "entropy": 1.0845373101532458, "epoch": 0.6950445895007815, "frac_reward_zero_std": 0.2, "grad_norm": 0.13479961454868317, "kl": 0.03286644645268098, "learning_rate": 4.562390504053049e-06, "loss": 0.0016, "num_tokens": 32860586.0, "reward": -0.05625, "reward_std": 0.4166190758347511, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.4166190758347511, "sampling/importance_sampling_ratio/max": 1.2547454833984375, "sampling/importance_sampling_ratio/mean": 1.0001095533370972, "sampling/importance_sampling_ratio/min": 0.7857195138931274, "sampling/sampling_logp_difference/max": 0.2574007511138916, "sampling/sampling_logp_difference/mean": 0.02132561318576336, "step": 7560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 319.0, "completions/max_terminated_length": 301.9, "completions/mean_length": 227.9875, "completions/mean_terminated_length": 224.77678680419922, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.1534547120332719, "epoch": 0.6959639606509148, "frac_reward_zero_std": 0.4, "grad_norm": 5.957430362701416, "kl": 0.030418830597773193, "learning_rate": 4.560957449244675e-06, "loss": 0.0012, "num_tokens": 32906073.0, "reward": 0.10625, "reward_std": 0.40544959008693693, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.4054496079683304, "sampling/importance_sampling_ratio/max": 1.2866672992706298, "sampling/importance_sampling_ratio/mean": 0.9993079364299774, "sampling/importance_sampling_ratio/min": 0.777455598115921, "sampling/sampling_logp_difference/max": 0.2747867345809937, "sampling/sampling_logp_difference/mean": 0.022157854959368707, "step": 7570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 300.3, "completions/max_terminated_length": 300.3, "completions/mean_length": 206.8, "completions/mean_terminated_length": 206.8, "completions/min_length": 149.0, "completions/min_terminated_length": 149.0, "entropy": 1.144667898118496, "epoch": 0.6968833318010481, "frac_reward_zero_std": 0.3, "grad_norm": 4.517390727996826, "kl": 0.03396015546750277, "learning_rate": 4.5595222776677935e-06, "loss": 0.0015, "num_tokens": 32946865.0, "reward": 0.20625, "reward_std": 0.46173239052295684, "rewards/reward_fn/mean": 0.20625, "rewards/reward_fn/std": 0.46173239648342135, "sampling/importance_sampling_ratio/max": 1.2663776516914367, "sampling/importance_sampling_ratio/mean": 1.0000428795814513, "sampling/importance_sampling_ratio/min": 0.777834290266037, "sampling/sampling_logp_difference/max": 0.27124514877796174, "sampling/sampling_logp_difference/mean": 0.022485843859612942, "step": 7580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.3, "completions/max_terminated_length": 297.3, "completions/mean_length": 211.4625, "completions/mean_terminated_length": 211.4625, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.1465738616883754, "epoch": 0.6978027029511814, "frac_reward_zero_std": 0.3, "grad_norm": 6.040626525878906, "kl": 0.04534736567875371, "learning_rate": 4.5580849907964435e-06, "loss": 0.0023, "num_tokens": 32987982.0, "reward": 0.1875, "reward_std": 0.3973605424165726, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3973605424165726, "sampling/importance_sampling_ratio/max": 1.2816292643547058, "sampling/importance_sampling_ratio/mean": 0.9996797740459442, "sampling/importance_sampling_ratio/min": 0.7559710204601288, "sampling/sampling_logp_difference/max": 0.3027397871017456, "sampling/sampling_logp_difference/mean": 0.022260284051299095, "step": 7590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.1, "completions/max_terminated_length": 271.1, "completions/mean_length": 190.0625, "completions/mean_terminated_length": 190.0625, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.0846397157758474, "epoch": 0.6987220741013147, "frac_reward_zero_std": 0.2, "grad_norm": 6.175319194793701, "kl": 0.05151133330073208, "learning_rate": 4.5566455901068305e-06, "loss": 0.0027, "num_tokens": 33029787.0, "reward": 0.4125, "reward_std": 0.38550153374671936, "rewards/reward_fn/mean": 0.4125, "rewards/reward_fn/std": 0.3855015397071838, "sampling/importance_sampling_ratio/max": 1.2769267559051514, "sampling/importance_sampling_ratio/mean": 0.9996841490268707, "sampling/importance_sampling_ratio/min": 0.8086808145046234, "sampling/sampling_logp_difference/max": 0.2563469409942627, "sampling/sampling_logp_difference/mean": 0.02165211960673332, "step": 7600 }, { "epoch": 0.6987220741013147, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0006544502617801048, "eval_completions/max_length": 282.239092495637, "eval_completions/max_terminated_length": 281.80279232111695, "eval_completions/mean_length": 200.5185427574171, "eval_completions/mean_terminated_length": 200.34199705007396, "eval_completions/min_length": 132.00523560209425, "eval_completions/min_terminated_length": 132.00523560209425, "eval_entropy": 1.1149459109880537, "eval_frac_reward_zero_std": 0.29668411867364747, "eval_kl": 0.038804631088624655, "eval_loss": 0.0019907415844500065, "eval_num_tokens": 33029787.0, "eval_reward": 0.056391797556719025, "eval_reward_std": 0.36392435577111304, "eval_rewards/reward_fn/mean": 0.056391797556719025, "eval_rewards/reward_fn/std": 0.36392436700549635, "eval_runtime": 1657.7012, "eval_samples_per_second": 0.346, "eval_sampling/importance_sampling_ratio/max": 1.2669749332764177, "eval_sampling/importance_sampling_ratio/mean": 1.000005006582117, "eval_sampling/importance_sampling_ratio/min": 0.7869546036341636, "eval_sampling/sampling_logp_difference/max": 0.26855901086517653, "eval_sampling/sampling_logp_difference/mean": 0.020919934141061174, "eval_steps_per_second": 0.043, "step": 7600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.4, "completions/max_terminated_length": 328.4, "completions/mean_length": 217.9, "completions/mean_terminated_length": 217.9, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.1236586943268776, "epoch": 0.699641445251448, "frac_reward_zero_std": 0.3, "grad_norm": 0.13727843761444092, "kl": 0.045946736662881446, "learning_rate": 4.555204077077338e-06, "loss": 0.0023, "num_tokens": 33073835.0, "reward": -0.275, "reward_std": 0.2874556124210358, "rewards/reward_fn/mean": -0.275, "rewards/reward_fn/std": 0.2874556124210358, "sampling/importance_sampling_ratio/max": 1.253422498703003, "sampling/importance_sampling_ratio/mean": 0.9996615648269653, "sampling/importance_sampling_ratio/min": 0.789066880941391, "sampling/sampling_logp_difference/max": 0.2551787614822388, "sampling/sampling_logp_difference/mean": 0.021304538100957872, "step": 7610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 205.725, "completions/mean_terminated_length": 205.725, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 1.130693106353283, "epoch": 0.7005608164015813, "frac_reward_zero_std": 0.4, "grad_norm": 0.1048576831817627, "kl": 0.0387039344292134, "learning_rate": 4.553760453188513e-06, "loss": 0.002, "num_tokens": 33116901.0, "reward": 0.1125, "reward_std": 0.3929676622152328, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.3929676741361618, "sampling/importance_sampling_ratio/max": 1.256409728527069, "sampling/importance_sampling_ratio/mean": 0.9999854505062103, "sampling/importance_sampling_ratio/min": 0.770319128036499, "sampling/sampling_logp_difference/max": 0.2739866256713867, "sampling/sampling_logp_difference/mean": 0.022002885490655898, "step": 7620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 250.7, "completions/max_terminated_length": 250.7, "completions/mean_length": 182.9375, "completions/mean_terminated_length": 182.9375, "completions/min_length": 123.5, "completions/min_terminated_length": 123.5, "entropy": 1.094175136089325, "epoch": 0.7014801875517146, "frac_reward_zero_std": 0.2, "grad_norm": 6.98690938949585, "kl": 0.039000157522968946, "learning_rate": 4.552314719923073e-06, "loss": 0.0021, "num_tokens": 33157824.0, "reward": -0.1375, "reward_std": 0.3824188977479935, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.38241890370845794, "sampling/importance_sampling_ratio/max": 1.2645862817764282, "sampling/importance_sampling_ratio/mean": 1.0000009000301362, "sampling/importance_sampling_ratio/min": 0.772594565153122, "sampling/sampling_logp_difference/max": 0.28966230154037476, "sampling/sampling_logp_difference/mean": 0.02176764216274023, "step": 7630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.5, "completions/max_terminated_length": 281.5, "completions/mean_length": 201.7, "completions/mean_terminated_length": 201.7, "completions/min_length": 124.5, "completions/min_terminated_length": 124.5, "entropy": 1.1148397870361806, "epoch": 0.7023995587018479, "frac_reward_zero_std": 0.5, "grad_norm": 6.051836967468262, "kl": 0.039943625044543296, "learning_rate": 4.550866878765903e-06, "loss": 0.0019, "num_tokens": 33202368.0, "reward": -0.00625, "reward_std": 0.27346964478492736, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.27346965074539187, "sampling/importance_sampling_ratio/max": 1.2642905831336975, "sampling/importance_sampling_ratio/mean": 1.000321626663208, "sampling/importance_sampling_ratio/min": 0.7478184580802918, "sampling/sampling_logp_difference/max": 0.3064985632896423, "sampling/sampling_logp_difference/mean": 0.021074975468218328, "step": 7640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.5, "completions/max_terminated_length": 292.5, "completions/mean_length": 204.55, "completions/mean_terminated_length": 204.55, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.09742097184062, "epoch": 0.7033189298519812, "frac_reward_zero_std": 0.4, "grad_norm": 5.594697952270508, "kl": 0.041919013380538675, "learning_rate": 4.549416931204051e-06, "loss": 0.002, "num_tokens": 33245476.0, "reward": 0.4375, "reward_std": 0.3049629807472229, "rewards/reward_fn/mean": 0.4375, "rewards/reward_fn/std": 0.3049629867076874, "sampling/importance_sampling_ratio/max": 1.2665457367897033, "sampling/importance_sampling_ratio/mean": 1.0000653088092804, "sampling/importance_sampling_ratio/min": 0.7821203589439392, "sampling/sampling_logp_difference/max": 0.26658855676651, "sampling/sampling_logp_difference/mean": 0.021193957701325417, "step": 7650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 202.2375, "completions/mean_terminated_length": 202.2375, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.1349547736346721, "epoch": 0.7042383010021146, "frac_reward_zero_std": 0.2, "grad_norm": 5.6268391609191895, "kl": 0.040196143608773126, "learning_rate": 4.547964878726729e-06, "loss": 0.0017, "num_tokens": 33287247.0, "reward": 0.075, "reward_std": 0.5108078241348266, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.5108078360557556, "sampling/importance_sampling_ratio/max": 1.3117812991142273, "sampling/importance_sampling_ratio/mean": 1.0000822067260742, "sampling/importance_sampling_ratio/min": 0.7875050127506256, "sampling/sampling_logp_difference/max": 0.28344073295593264, "sampling/sampling_logp_difference/mean": 0.022066364251077174, "step": 7660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.2, "completions/max_terminated_length": 258.2, "completions/mean_length": 192.1375, "completions/mean_terminated_length": 192.1375, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.1447755262255668, "epoch": 0.7051576721522479, "frac_reward_zero_std": 0.3, "grad_norm": 6.412333965301514, "kl": 0.05213334017898887, "learning_rate": 4.5465107228253135e-06, "loss": 0.0028, "num_tokens": 33327450.0, "reward": 0.18125, "reward_std": 0.3508347451686859, "rewards/reward_fn/mean": 0.18125, "rewards/reward_fn/std": 0.3508347511291504, "sampling/importance_sampling_ratio/max": 1.2710581183433534, "sampling/importance_sampling_ratio/mean": 0.9999002158641815, "sampling/importance_sampling_ratio/min": 0.7738643288612366, "sampling/sampling_logp_difference/max": 0.29165835976600646, "sampling/sampling_logp_difference/mean": 0.022472849115729332, "step": 7670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.2, "completions/max_terminated_length": 298.2, "completions/mean_length": 209.2625, "completions/mean_terminated_length": 209.2625, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 0.9848915636539459, "epoch": 0.7060770433023812, "frac_reward_zero_std": 0.3, "grad_norm": 0.09794360399246216, "kl": 0.0333785520051606, "learning_rate": 4.5450544649933375e-06, "loss": 0.002, "num_tokens": 33368159.0, "reward": 0.025, "reward_std": 0.3349648594856262, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3349648714065552, "sampling/importance_sampling_ratio/max": 1.2900812029838562, "sampling/importance_sampling_ratio/mean": 0.9998264491558075, "sampling/importance_sampling_ratio/min": 0.7895427644252777, "sampling/sampling_logp_difference/max": 0.2809017479419708, "sampling/sampling_logp_difference/mean": 0.02042638398706913, "step": 7680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 198.4, "completions/mean_terminated_length": 198.4, "completions/min_length": 140.7, "completions/min_terminated_length": 140.7, "entropy": 1.0890538729727268, "epoch": 0.7069964144525145, "frac_reward_zero_std": 0.2, "grad_norm": 0.16050505638122559, "kl": 0.043837835127487776, "learning_rate": 4.543596106726494e-06, "loss": 0.0022, "num_tokens": 33408623.0, "reward": -0.1, "reward_std": 0.42010811567306516, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4201081216335297, "sampling/importance_sampling_ratio/max": 1.35472332239151, "sampling/importance_sampling_ratio/mean": 1.0003214716911315, "sampling/importance_sampling_ratio/min": 0.7823984801769257, "sampling/sampling_logp_difference/max": 0.33009341955184934, "sampling/sampling_logp_difference/mean": 0.021598795801401137, "step": 7690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 202.9625, "completions/mean_terminated_length": 202.9625, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.0056064058095218, "epoch": 0.7079157856026478, "frac_reward_zero_std": 0.1, "grad_norm": 5.50482702255249, "kl": 0.04954571602283977, "learning_rate": 4.5421356495226356e-06, "loss": 0.0018, "num_tokens": 33451972.0, "reward": -0.225, "reward_std": 0.48828638792037965, "rewards/reward_fn/mean": -0.225, "rewards/reward_fn/std": 0.4882863938808441, "sampling/importance_sampling_ratio/max": 1.271443784236908, "sampling/importance_sampling_ratio/mean": 0.9991957306861877, "sampling/importance_sampling_ratio/min": 0.7375916063785553, "sampling/sampling_logp_difference/max": 0.3386650800704956, "sampling/sampling_logp_difference/mean": 0.0205399002879858, "step": 7700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.7, "completions/max_terminated_length": 297.7, "completions/mean_length": 205.3, "completions/mean_terminated_length": 205.3, "completions/min_length": 135.2, "completions/min_terminated_length": 135.2, "entropy": 1.1315946891903876, "epoch": 0.7088351567527811, "frac_reward_zero_std": 0.3, "grad_norm": 0.1856188327074051, "kl": 0.040633356000762434, "learning_rate": 4.540673094881768e-06, "loss": 0.0018, "num_tokens": 33494220.0, "reward": 0.075, "reward_std": 0.2966939479112625, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.296693953871727, "sampling/importance_sampling_ratio/max": 1.234983253479004, "sampling/importance_sampling_ratio/mean": 1.0000481128692627, "sampling/importance_sampling_ratio/min": 0.7976975739002228, "sampling/sampling_logp_difference/max": 0.23871080875396727, "sampling/sampling_logp_difference/mean": 0.02253729533404112, "step": 7710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 316.6, "completions/max_terminated_length": 302.6, "completions/mean_length": 227.625, "completions/mean_terminated_length": 224.5, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.1352139122784137, "epoch": 0.7097545279029144, "frac_reward_zero_std": 0.4, "grad_norm": 5.784977436065674, "kl": 0.04954952227417379, "learning_rate": 4.539208444306051e-06, "loss": 0.0024, "num_tokens": 33540222.0, "reward": 0.09375, "reward_std": 0.3108975887298584, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.3108976066112518, "sampling/importance_sampling_ratio/max": 1.271518075466156, "sampling/importance_sampling_ratio/mean": 0.9999210596084595, "sampling/importance_sampling_ratio/min": 0.7494085669517517, "sampling/sampling_logp_difference/max": 0.306148886680603, "sampling/sampling_logp_difference/mean": 0.022181358188390732, "step": 7720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.4, "completions/max_terminated_length": 315.4, "completions/mean_length": 226.4125, "completions/mean_terminated_length": 226.4125, "completions/min_length": 160.9, "completions/min_terminated_length": 160.9, "entropy": 1.0955932341516017, "epoch": 0.7106738990530477, "frac_reward_zero_std": 0.3, "grad_norm": 7.577101230621338, "kl": 0.03685650514089502, "learning_rate": 4.5377416992998e-06, "loss": 0.0015, "num_tokens": 33585631.0, "reward": -0.13125, "reward_std": 0.38051809966564176, "rewards/reward_fn/mean": -0.13125, "rewards/reward_fn/std": 0.3805181056261063, "sampling/importance_sampling_ratio/max": 1.2924562692642212, "sampling/importance_sampling_ratio/mean": 1.0002309203147888, "sampling/importance_sampling_ratio/min": 0.7845674693584442, "sampling/sampling_logp_difference/max": 0.2886636972427368, "sampling/sampling_logp_difference/mean": 0.021633667312562464, "step": 7730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.7, "completions/max_terminated_length": 282.7, "completions/mean_length": 207.9625, "completions/mean_terminated_length": 207.9625, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.0437857307493688, "epoch": 0.711593270203181, "frac_reward_zero_std": 0.3, "grad_norm": 4.8421807289123535, "kl": 0.03807074510259554, "learning_rate": 4.536272861369478e-06, "loss": 0.0017, "num_tokens": 33628524.0, "reward": -0.2, "reward_std": 0.3202547669410706, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3202547788619995, "sampling/importance_sampling_ratio/max": 1.3327083826065063, "sampling/importance_sampling_ratio/mean": 0.9998747706413269, "sampling/importance_sampling_ratio/min": 0.7640017390251159, "sampling/sampling_logp_difference/max": 0.316588830947876, "sampling/sampling_logp_difference/mean": 0.021363392099738122, "step": 7740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 307.9, "completions/max_terminated_length": 285.7, "completions/mean_length": 211.275, "completions/mean_terminated_length": 207.51428680419923, "completions/min_length": 125.6, "completions/min_terminated_length": 125.6, "entropy": 1.1353821109980344, "epoch": 0.7125126413533144, "frac_reward_zero_std": 0.3, "grad_norm": 0.20360620319843292, "kl": 0.03745500167715363, "learning_rate": 4.534801932023701e-06, "loss": 0.0018, "num_tokens": 33672482.0, "reward": 0.31875, "reward_std": 0.3897308617830276, "rewards/reward_fn/mean": 0.31875, "rewards/reward_fn/std": 0.38973086774349214, "sampling/importance_sampling_ratio/max": 1.265360164642334, "sampling/importance_sampling_ratio/mean": 1.000350195169449, "sampling/importance_sampling_ratio/min": 0.7864132761955261, "sampling/sampling_logp_difference/max": 0.2651828408241272, "sampling/sampling_logp_difference/mean": 0.022775939479470254, "step": 7750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 318.5, "completions/max_terminated_length": 306.1, "completions/mean_length": 210.2625, "completions/mean_terminated_length": 207.0464294433594, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0103291809558868, "epoch": 0.7134320125034477, "frac_reward_zero_std": 0.3, "grad_norm": 0.13381251692771912, "kl": 0.0372956077917479, "learning_rate": 4.53332891277323e-06, "loss": 0.0019, "num_tokens": 33715023.0, "reward": 0.19375, "reward_std": 0.3609550654888153, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.36095508337020876, "sampling/importance_sampling_ratio/max": 1.2696126341819762, "sampling/importance_sampling_ratio/mean": 1.0006945371627807, "sampling/importance_sampling_ratio/min": 0.7741854786872864, "sampling/sampling_logp_difference/max": 0.2791248619556427, "sampling/sampling_logp_difference/mean": 0.020971699617803097, "step": 7760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.1, "completions/max_terminated_length": 319.1, "completions/mean_length": 224.025, "completions/mean_terminated_length": 224.025, "completions/min_length": 159.2, "completions/min_terminated_length": 159.2, "entropy": 1.1375169530510902, "epoch": 0.714351383653581, "frac_reward_zero_std": 0.3, "grad_norm": 0.12078677117824554, "kl": 0.037285058957058935, "learning_rate": 4.531853805130975e-06, "loss": 0.0018, "num_tokens": 33761481.0, "reward": 0.05, "reward_std": 0.3038551896810532, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.30385519564151764, "sampling/importance_sampling_ratio/max": 1.221822166442871, "sampling/importance_sampling_ratio/mean": 0.9995638489723205, "sampling/importance_sampling_ratio/min": 0.7730231702327728, "sampling/sampling_logp_difference/max": 0.2626751184463501, "sampling/sampling_logp_difference/mean": 0.022068896889686586, "step": 7770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.0, "completions/max_terminated_length": 252.0, "completions/mean_length": 185.9625, "completions/mean_terminated_length": 185.9625, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0630664438009263, "epoch": 0.7152707548037143, "frac_reward_zero_std": 0.5, "grad_norm": 6.958263874053955, "kl": 0.031935215438716115, "learning_rate": 4.530376610611988e-06, "loss": 0.0019, "num_tokens": 33801398.0, "reward": 0.1625, "reward_std": 0.2877043843269348, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.28770439624786376, "sampling/importance_sampling_ratio/max": 1.2999430894851685, "sampling/importance_sampling_ratio/mean": 1.000385844707489, "sampling/importance_sampling_ratio/min": 0.7916532039642334, "sampling/sampling_logp_difference/max": 0.27481067180633545, "sampling/sampling_logp_difference/mean": 0.02188342474400997, "step": 7780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.6, "completions/max_terminated_length": 309.6, "completions/mean_length": 211.5875, "completions/mean_terminated_length": 211.5875, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.0215931819751858, "epoch": 0.7161901259538476, "frac_reward_zero_std": 0.2, "grad_norm": 6.65397310256958, "kl": 0.039828912253142335, "learning_rate": 4.528897330733467e-06, "loss": 0.002, "num_tokens": 33845397.0, "reward": -0.175, "reward_std": 0.44539301097393036, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.44539301097393036, "sampling/importance_sampling_ratio/max": 1.2443268537521361, "sampling/importance_sampling_ratio/mean": 0.999433183670044, "sampling/importance_sampling_ratio/min": 0.806364780664444, "sampling/sampling_logp_difference/max": 0.24003589153289795, "sampling/sampling_logp_difference/mean": 0.019792910479009152, "step": 7790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.4, "completions/max_terminated_length": 261.4, "completions/mean_length": 190.15, "completions/mean_terminated_length": 190.15, "completions/min_length": 135.3, "completions/min_terminated_length": 135.3, "entropy": 0.9830567568540574, "epoch": 0.7171094971039809, "frac_reward_zero_std": 0.3, "grad_norm": 5.937559604644775, "kl": 0.0322166383266449, "learning_rate": 4.527415967014752e-06, "loss": 0.0014, "num_tokens": 33887569.0, "reward": 0.125, "reward_std": 0.3504155069589615, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.35041551291942596, "sampling/importance_sampling_ratio/max": 1.2904535412788392, "sampling/importance_sampling_ratio/mean": 0.9998431980609894, "sampling/importance_sampling_ratio/min": 0.7544145703315734, "sampling/sampling_logp_difference/max": 0.324628746509552, "sampling/sampling_logp_difference/mean": 0.020097553543746472, "step": 7800 }, { "epoch": 0.7171094971039809, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 307.7190226876091, "eval_completions/max_terminated_length": 305.66841186736474, "eval_completions/mean_length": 219.8651832460733, "eval_completions/mean_terminated_length": 219.30843325310352, "eval_completions/min_length": 146.96684118673647, "eval_completions/min_terminated_length": 146.96684118673647, "eval_entropy": 1.0932212774666192, "eval_frac_reward_zero_std": 0.30715532286212915, "eval_kl": 0.042738341207877714, "eval_loss": 0.0021742191165685654, "eval_num_tokens": 33887569.0, "eval_reward": 0.05279232111692845, "eval_reward_std": 0.3492573535047068, "eval_rewards/reward_fn/mean": 0.05279232111692845, "eval_rewards/reward_fn/std": 0.3492573633868033, "eval_runtime": 1787.3842, "eval_samples_per_second": 0.321, "eval_sampling/importance_sampling_ratio/max": 1.2752930914128222, "eval_sampling/importance_sampling_ratio/mean": 0.9999584233157506, "eval_sampling/importance_sampling_ratio/min": 0.7807589556638275, "eval_sampling/sampling_logp_difference/max": 0.27663113673528034, "eval_sampling/sampling_logp_difference/mean": 0.020761492777531268, "eval_steps_per_second": 0.04, "step": 7800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 342.1, "completions/max_terminated_length": 342.1, "completions/mean_length": 242.25, "completions/mean_terminated_length": 242.25, "completions/min_length": 161.1, "completions/min_terminated_length": 161.1, "entropy": 1.139425777643919, "epoch": 0.7180288682541142, "frac_reward_zero_std": 0.3, "grad_norm": 4.486752510070801, "kl": 0.05300962534965947, "learning_rate": 4.525932520977322e-06, "loss": 0.0022, "num_tokens": 33937949.0, "reward": 0.0625, "reward_std": 0.404693940281868, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.404693940281868, "sampling/importance_sampling_ratio/max": 1.2692065238952637, "sampling/importance_sampling_ratio/mean": 1.000042188167572, "sampling/importance_sampling_ratio/min": 0.7585910797119141, "sampling/sampling_logp_difference/max": 0.2811714351177216, "sampling/sampling_logp_difference/mean": 0.021924970485270024, "step": 7810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 198.9375, "completions/mean_terminated_length": 198.9375, "completions/min_length": 144.8, "completions/min_terminated_length": 144.8, "entropy": 1.0535544984042644, "epoch": 0.7189482394042475, "frac_reward_zero_std": 0.3, "grad_norm": 0.12694041430950165, "kl": 0.046225569950183856, "learning_rate": 4.524446994144795e-06, "loss": 0.0026, "num_tokens": 33981160.0, "reward": 0.2625, "reward_std": 0.361081862449646, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.3610818684101105, "sampling/importance_sampling_ratio/max": 1.252843940258026, "sampling/importance_sampling_ratio/mean": 1.0002496123313904, "sampling/importance_sampling_ratio/min": 0.7583630740642547, "sampling/sampling_logp_difference/max": 0.28971938490867616, "sampling/sampling_logp_difference/mean": 0.021731074154376983, "step": 7820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.8, "completions/max_terminated_length": 295.8, "completions/mean_length": 216.9625, "completions/mean_terminated_length": 216.9625, "completions/min_length": 152.7, "completions/min_terminated_length": 152.7, "entropy": 1.0459695369005204, "epoch": 0.7198676105543808, "frac_reward_zero_std": 0.2, "grad_norm": 5.909083366394043, "kl": 0.042244491196470335, "learning_rate": 4.5229593880429245e-06, "loss": 0.0017, "num_tokens": 34026213.0, "reward": 0.05, "reward_std": 0.38372147977352145, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.38372149765491487, "sampling/importance_sampling_ratio/max": 1.2656212329864502, "sampling/importance_sampling_ratio/mean": 0.999660336971283, "sampling/importance_sampling_ratio/min": 0.7756404161453248, "sampling/sampling_logp_difference/max": 0.275792670249939, "sampling/sampling_logp_difference/mean": 0.021000822819769383, "step": 7830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.6, "completions/max_terminated_length": 325.6, "completions/mean_length": 233.5625, "completions/mean_terminated_length": 233.5625, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.1200813628733157, "epoch": 0.7207869817045142, "frac_reward_zero_std": 0.5, "grad_norm": 0.11171316355466843, "kl": 0.047881312703248115, "learning_rate": 4.521469704199603e-06, "loss": 0.0026, "num_tokens": 34071514.0, "reward": -0.125, "reward_std": 0.25032021999359133, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.25032023191452024, "sampling/importance_sampling_ratio/max": 1.3082908630371093, "sampling/importance_sampling_ratio/mean": 1.0002647995948792, "sampling/importance_sampling_ratio/min": 0.7753558576107025, "sampling/sampling_logp_difference/max": 0.2913394570350647, "sampling/sampling_logp_difference/mean": 0.021616406552493573, "step": 7840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.8, "completions/max_terminated_length": 308.8, "completions/mean_length": 214.175, "completions/mean_terminated_length": 214.175, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.0277395989745854, "epoch": 0.7217063528546475, "frac_reward_zero_std": 0.2, "grad_norm": 4.850182056427002, "kl": 0.052500088210217656, "learning_rate": 4.519977944144854e-06, "loss": 0.003, "num_tokens": 34114304.0, "reward": 0.0125, "reward_std": 0.4219646632671356, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.4219646692276001, "sampling/importance_sampling_ratio/max": 1.2588640332221985, "sampling/importance_sampling_ratio/mean": 1.0002581179141998, "sampling/importance_sampling_ratio/min": 0.7745973169803619, "sampling/sampling_logp_difference/max": 0.2695298671722412, "sampling/sampling_logp_difference/mean": 0.02054786393418908, "step": 7850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.3, "completions/max_terminated_length": 303.3, "completions/mean_length": 214.575, "completions/mean_terminated_length": 214.575, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.1137353211641312, "epoch": 0.7226257240047808, "frac_reward_zero_std": 0.2, "grad_norm": 0.13138894736766815, "kl": 0.050362446304643524, "learning_rate": 4.518484109410837e-06, "loss": 0.0023, "num_tokens": 34158102.0, "reward": 0.05, "reward_std": 0.38927595913410185, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.38927596509456636, "sampling/importance_sampling_ratio/max": 1.262539768218994, "sampling/importance_sampling_ratio/mean": 0.9997757256031037, "sampling/importance_sampling_ratio/min": 0.8053169488906861, "sampling/sampling_logp_difference/max": 0.24541983604431153, "sampling/sampling_logp_difference/mean": 0.021543565765023233, "step": 7860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.9, "completions/max_terminated_length": 309.9, "completions/mean_length": 212.8125, "completions/mean_terminated_length": 212.8125, "completions/min_length": 139.8, "completions/min_terminated_length": 139.8, "entropy": 1.0954293794929981, "epoch": 0.723545095154914, "frac_reward_zero_std": 0.3, "grad_norm": 0.1322997808456421, "kl": 0.03421082017011941, "learning_rate": 4.516988201531837e-06, "loss": 0.0017, "num_tokens": 34202567.0, "reward": 0.025, "reward_std": 0.3918117702007294, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3918117702007294, "sampling/importance_sampling_ratio/max": 1.2883691906929016, "sampling/importance_sampling_ratio/mean": 0.9999663412570954, "sampling/importance_sampling_ratio/min": 0.7708441138267517, "sampling/sampling_logp_difference/max": 0.31276854276657107, "sampling/sampling_logp_difference/mean": 0.021195939183235167, "step": 7870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.2, "completions/max_terminated_length": 297.2, "completions/mean_length": 219.45, "completions/mean_terminated_length": 219.45, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.072926290333271, "epoch": 0.7244644663050473, "frac_reward_zero_std": 0.3, "grad_norm": 6.371822834014893, "kl": 0.04007494014222175, "learning_rate": 4.515490222044273e-06, "loss": 0.0028, "num_tokens": 34249091.0, "reward": 0.3125, "reward_std": 0.2857582807540894, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.28575828671455383, "sampling/importance_sampling_ratio/max": 1.2246232509613038, "sampling/importance_sampling_ratio/mean": 0.9997083663940429, "sampling/importance_sampling_ratio/min": 0.7805475413799285, "sampling/sampling_logp_difference/max": 0.25602915287017824, "sampling/sampling_logp_difference/mean": 0.020623759552836417, "step": 7880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 208.4875, "completions/mean_terminated_length": 208.4875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0243792064487933, "epoch": 0.7253838374551806, "frac_reward_zero_std": 0.6, "grad_norm": 5.473594665527344, "kl": 0.04635031825164333, "learning_rate": 4.513990172486689e-06, "loss": 0.0026, "num_tokens": 34292122.0, "reward": -0.0875, "reward_std": 0.23144718110561371, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.23144719302654265, "sampling/importance_sampling_ratio/max": 1.30718754529953, "sampling/importance_sampling_ratio/mean": 1.0002975821495057, "sampling/importance_sampling_ratio/min": 0.7781687617301941, "sampling/sampling_logp_difference/max": 0.29209108352661134, "sampling/sampling_logp_difference/mean": 0.021396419778466225, "step": 7890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.1, "completions/max_terminated_length": 337.1, "completions/mean_length": 219.1, "completions/mean_terminated_length": 219.1, "completions/min_length": 142.0, "completions/min_terminated_length": 142.0, "entropy": 1.0985589452087878, "epoch": 0.7263032086053139, "frac_reward_zero_std": 0.3, "grad_norm": 0.08928769826889038, "kl": 0.04869061856297776, "learning_rate": 4.512488054399758e-06, "loss": 0.0024, "num_tokens": 34338882.0, "reward": 0.0625, "reward_std": 0.34927949607372283, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.34927951991558076, "sampling/importance_sampling_ratio/max": 1.2863481521606446, "sampling/importance_sampling_ratio/mean": 0.9998274266719818, "sampling/importance_sampling_ratio/min": 0.7827123999595642, "sampling/sampling_logp_difference/max": 0.2661070704460144, "sampling/sampling_logp_difference/mean": 0.021367667987942696, "step": 7900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.2, "completions/max_terminated_length": 288.2, "completions/mean_length": 197.4375, "completions/mean_terminated_length": 197.4375, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0174917988479137, "epoch": 0.7272225797554472, "frac_reward_zero_std": 0.2, "grad_norm": 0.1163519099354744, "kl": 0.04199837712221779, "learning_rate": 4.510983869326273e-06, "loss": 0.0022, "num_tokens": 34379477.0, "reward": 0.0, "reward_std": 0.42832300066947937, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.42832301259040834, "sampling/importance_sampling_ratio/max": 1.2799532771110536, "sampling/importance_sampling_ratio/mean": 0.9994352340698243, "sampling/importance_sampling_ratio/min": 0.7756819188594818, "sampling/sampling_logp_difference/max": 0.28632594347000123, "sampling/sampling_logp_difference/mean": 0.02054649069905281, "step": 7910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.8, "completions/max_terminated_length": 275.8, "completions/mean_length": 191.9, "completions/mean_terminated_length": 191.9, "completions/min_length": 128.8, "completions/min_terminated_length": 128.8, "entropy": 1.0416158333420753, "epoch": 0.7281419509055805, "frac_reward_zero_std": 0.2, "grad_norm": 7.041605472564697, "kl": 0.03358384228777141, "learning_rate": 4.509477618811156e-06, "loss": 0.0012, "num_tokens": 34421453.0, "reward": 0.0875, "reward_std": 0.4782218664884567, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.4782218724489212, "sampling/importance_sampling_ratio/max": 1.2627132177352904, "sampling/importance_sampling_ratio/mean": 0.9999583303928375, "sampling/importance_sampling_ratio/min": 0.7706750512123108, "sampling/sampling_logp_difference/max": 0.275869745016098, "sampling/sampling_logp_difference/mean": 0.02102560121566057, "step": 7920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.7, "completions/max_terminated_length": 319.2, "completions/mean_length": 217.325, "completions/mean_terminated_length": 214.61964416503906, "completions/min_length": 143.1, "completions/min_terminated_length": 143.1, "entropy": 1.090885204076767, "epoch": 0.729061322055714, "frac_reward_zero_std": 0.5, "grad_norm": 5.933417797088623, "kl": 0.031383454659953715, "learning_rate": 4.507969304401443e-06, "loss": 0.0016, "num_tokens": 34466215.0, "reward": 0.06875, "reward_std": 0.285294908285141, "rewards/reward_fn/mean": 0.06875, "rewards/reward_fn/std": 0.285294908285141, "sampling/importance_sampling_ratio/max": 1.285138475894928, "sampling/importance_sampling_ratio/mean": 0.9998807489871979, "sampling/importance_sampling_ratio/min": 0.7618761599063874, "sampling/sampling_logp_difference/max": 0.3024583339691162, "sampling/sampling_logp_difference/mean": 0.021107158064842223, "step": 7930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.1, "completions/max_terminated_length": 283.1, "completions/mean_length": 207.875, "completions/mean_terminated_length": 207.875, "completions/min_length": 137.4, "completions/min_terminated_length": 137.4, "entropy": 0.9704998046159744, "epoch": 0.7299806932058472, "frac_reward_zero_std": 0.4, "grad_norm": 0.12281299382448196, "kl": 0.030406964721623807, "learning_rate": 4.506458927646297e-06, "loss": 0.0015, "num_tokens": 34506877.0, "reward": 0.3875, "reward_std": 0.26849985122680664, "rewards/reward_fn/mean": 0.3875, "rewards/reward_fn/std": 0.2684998571872711, "sampling/importance_sampling_ratio/max": 1.2835866928100585, "sampling/importance_sampling_ratio/mean": 0.9997747421264649, "sampling/importance_sampling_ratio/min": 0.7811548173427582, "sampling/sampling_logp_difference/max": 0.28658034801483157, "sampling/sampling_logp_difference/mean": 0.020425513759255408, "step": 7940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 343.8, "completions/max_terminated_length": 343.8, "completions/mean_length": 230.8875, "completions/mean_terminated_length": 230.8875, "completions/min_length": 147.4, "completions/min_terminated_length": 147.4, "entropy": 1.1622434698045254, "epoch": 0.7309000643559805, "frac_reward_zero_std": 0.2, "grad_norm": 5.867983341217041, "kl": 0.03975532315671444, "learning_rate": 4.504946490096995e-06, "loss": 0.0021, "num_tokens": 34549692.0, "reward": -0.0375, "reward_std": 0.340069380402565, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.340069380402565, "sampling/importance_sampling_ratio/max": 1.280290699005127, "sampling/importance_sampling_ratio/mean": 1.0001557052135468, "sampling/importance_sampling_ratio/min": 0.7522724390029907, "sampling/sampling_logp_difference/max": 0.32168265283107755, "sampling/sampling_logp_difference/mean": 0.022287780605256557, "step": 7950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.6, "completions/max_terminated_length": 328.6, "completions/mean_length": 212.525, "completions/mean_terminated_length": 212.525, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.0840055771172046, "epoch": 0.7318194355061138, "frac_reward_zero_std": 0.0, "grad_norm": 6.8784050941467285, "kl": 0.05069787713000551, "learning_rate": 4.503431993306931e-06, "loss": 0.0024, "num_tokens": 34591006.0, "reward": 0.0875, "reward_std": 0.5297319799661636, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.5297319859266281, "sampling/importance_sampling_ratio/max": 1.2535661220550538, "sampling/importance_sampling_ratio/mean": 1.0000344336032867, "sampling/importance_sampling_ratio/min": 0.805215185880661, "sampling/sampling_logp_difference/max": 0.2535577893257141, "sampling/sampling_logp_difference/mean": 0.02068595476448536, "step": 7960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 214.95, "completions/mean_terminated_length": 214.95, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.1212335646152496, "epoch": 0.7327388066562471, "frac_reward_zero_std": 0.4, "grad_norm": 6.346487522125244, "kl": 0.04603611533530057, "learning_rate": 4.5019154388316165e-06, "loss": 0.0023, "num_tokens": 34636938.0, "reward": -0.1, "reward_std": 0.2937580019235611, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.29375800788402556, "sampling/importance_sampling_ratio/max": 1.2788280606269837, "sampling/importance_sampling_ratio/mean": 1.000061458349228, "sampling/importance_sampling_ratio/min": 0.7785823583602905, "sampling/sampling_logp_difference/max": 0.27463321685791015, "sampling/sampling_logp_difference/mean": 0.021402878873050212, "step": 7970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 299.1, "completions/max_terminated_length": 275.4, "completions/mean_length": 202.575, "completions/mean_terminated_length": 198.7732147216797, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.0417906403541566, "epoch": 0.7336581778063804, "frac_reward_zero_std": 0.2, "grad_norm": 8.332976341247559, "kl": 0.044533792143920434, "learning_rate": 4.500396828228673e-06, "loss": 0.0028, "num_tokens": 34681400.0, "reward": 0.14375, "reward_std": 0.34977678656578065, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.34977678656578065, "sampling/importance_sampling_ratio/max": 1.2818006515502929, "sampling/importance_sampling_ratio/mean": 0.9997791945934296, "sampling/importance_sampling_ratio/min": 0.732440397143364, "sampling/sampling_logp_difference/max": 0.3559589147567749, "sampling/sampling_logp_difference/mean": 0.020886536128818987, "step": 7980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.8, "completions/max_terminated_length": 287.8, "completions/mean_length": 198.0375, "completions/mean_terminated_length": 198.0375, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.085494488477707, "epoch": 0.7345775489565137, "frac_reward_zero_std": 0.5, "grad_norm": 6.626796722412109, "kl": 0.026610825723037124, "learning_rate": 4.498876163057834e-06, "loss": 0.0015, "num_tokens": 34725099.0, "reward": 0.375, "reward_std": 0.29642280638217927, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.29642280638217927, "sampling/importance_sampling_ratio/max": 1.2888177275657653, "sampling/importance_sampling_ratio/mean": 1.000968623161316, "sampling/importance_sampling_ratio/min": 0.7828558683395386, "sampling/sampling_logp_difference/max": 0.26327439546585085, "sampling/sampling_logp_difference/mean": 0.021558471396565436, "step": 7990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.4, "completions/max_terminated_length": 280.4, "completions/mean_length": 193.5125, "completions/mean_terminated_length": 193.5125, "completions/min_length": 128.6, "completions/min_terminated_length": 128.6, "entropy": 1.0552413880825042, "epoch": 0.735496920106647, "frac_reward_zero_std": 0.3, "grad_norm": 0.10022339224815369, "kl": 0.053475710132624955, "learning_rate": 4.497353444880946e-06, "loss": 0.0022, "num_tokens": 34764924.0, "reward": 0.2375, "reward_std": 0.35827490985393523, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.35827491581439974, "sampling/importance_sampling_ratio/max": 1.2680016875267028, "sampling/importance_sampling_ratio/mean": 0.999761426448822, "sampling/importance_sampling_ratio/min": 0.7820928394794464, "sampling/sampling_logp_difference/max": 0.277970677614212, "sampling/sampling_logp_difference/mean": 0.02173848431557417, "step": 8000 }, { "epoch": 0.735496920106647, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0010907504363001745, "eval_completions/max_length": 283.7993019197208, "eval_completions/max_terminated_length": 283.2547993019197, "eval_completions/mean_length": 197.25501745200697, "eval_completions/mean_terminated_length": 197.0399526759176, "eval_completions/min_length": 126.37347294938918, "eval_completions/min_terminated_length": 126.37347294938918, "eval_entropy": 1.0924774579456755, "eval_frac_reward_zero_std": 0.29668411867364747, "eval_kl": 0.0382567643696673, "eval_loss": 0.001952486694790423, "eval_num_tokens": 34764924.0, "eval_reward": 0.03468586387434555, "eval_reward_std": 0.37787579387478387, "eval_rewards/reward_fn/mean": 0.03468586387434555, "eval_rewards/reward_fn/std": 0.37787580521318925, "eval_runtime": 1665.7685, "eval_samples_per_second": 0.344, "eval_sampling/importance_sampling_ratio/max": 1.2756718395060063, "eval_sampling/importance_sampling_ratio/mean": 0.9999773400408227, "eval_sampling/importance_sampling_ratio/min": 0.7776078785173139, "eval_sampling/sampling_logp_difference/max": 0.28330271042246263, "eval_sampling/sampling_logp_difference/mean": 0.021186423596859395, "eval_steps_per_second": 0.043, "step": 8000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.6, "completions/max_terminated_length": 274.6, "completions/mean_length": 189.9625, "completions/mean_terminated_length": 189.9625, "completions/min_length": 117.9, "completions/min_terminated_length": 117.9, "entropy": 1.1164085283875465, "epoch": 0.7364162912567803, "frac_reward_zero_std": 0.4, "grad_norm": 6.712962627410889, "kl": 0.03859089035540819, "learning_rate": 4.495828675261964e-06, "loss": 0.002, "num_tokens": 34809193.0, "reward": -0.15, "reward_std": 0.31875187158584595, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3187518835067749, "sampling/importance_sampling_ratio/max": 1.226724374294281, "sampling/importance_sampling_ratio/mean": 0.9996237277984619, "sampling/importance_sampling_ratio/min": 0.7882050752639771, "sampling/sampling_logp_difference/max": 0.2427889585494995, "sampling/sampling_logp_difference/mean": 0.021147720888257025, "step": 8010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.5, "completions/max_terminated_length": 287.5, "completions/mean_length": 185.85, "completions/mean_terminated_length": 185.85, "completions/min_length": 99.5, "completions/min_terminated_length": 99.5, "entropy": 1.1098272435367107, "epoch": 0.7373356624069136, "frac_reward_zero_std": 0.5, "grad_norm": 0.14185519516468048, "kl": 0.053213424660498276, "learning_rate": 4.494301855766947e-06, "loss": 0.0026, "num_tokens": 34849221.0, "reward": -0.01875, "reward_std": 0.25121486783027647, "rewards/reward_fn/mean": -0.01875, "rewards/reward_fn/std": 0.25121487975120543, "sampling/importance_sampling_ratio/max": 1.2421876192092896, "sampling/importance_sampling_ratio/mean": 0.9999228894710541, "sampling/importance_sampling_ratio/min": 0.7679965972900391, "sampling/sampling_logp_difference/max": 0.280995237827301, "sampling/sampling_logp_difference/mean": 0.0220765283331275, "step": 8020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 327.9, "completions/max_terminated_length": 327.9, "completions/mean_length": 221.9, "completions/mean_terminated_length": 221.9, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.1154138021171094, "epoch": 0.738255033557047, "frac_reward_zero_std": 0.2, "grad_norm": 5.695866584777832, "kl": 0.04086888821911998, "learning_rate": 4.492772987964061e-06, "loss": 0.0024, "num_tokens": 34890997.0, "reward": 0.0375, "reward_std": 0.4650593549013138, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.4650593727827072, "sampling/importance_sampling_ratio/max": 1.3206546783447266, "sampling/importance_sampling_ratio/mean": 1.0002088248729706, "sampling/importance_sampling_ratio/min": 0.7338766843080521, "sampling/sampling_logp_difference/max": 0.3303542137145996, "sampling/sampling_logp_difference/mean": 0.02125283405184746, "step": 8030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.1, "completions/max_terminated_length": 302.1, "completions/mean_length": 205.1625, "completions/mean_terminated_length": 205.1625, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.1147445015609265, "epoch": 0.7391744047071803, "frac_reward_zero_std": 0.1, "grad_norm": 7.414944648742676, "kl": 0.04698902390664443, "learning_rate": 4.491242073423575e-06, "loss": 0.0022, "num_tokens": 34934050.0, "reward": -0.2125, "reward_std": 0.4944003999233246, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.494400417804718, "sampling/importance_sampling_ratio/max": 1.3245744705200195, "sampling/importance_sampling_ratio/mean": 1.0002626717090606, "sampling/importance_sampling_ratio/min": 0.7732381999492646, "sampling/sampling_logp_difference/max": 0.31245502829551697, "sampling/sampling_logp_difference/mean": 0.02212924640625715, "step": 8040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.5, "completions/max_terminated_length": 309.5, "completions/mean_length": 216.9, "completions/mean_terminated_length": 216.9, "completions/min_length": 153.9, "completions/min_terminated_length": 153.9, "entropy": 1.084584641456604, "epoch": 0.7400937758573136, "frac_reward_zero_std": 0.4, "grad_norm": 6.194960117340088, "kl": 0.045161233912222085, "learning_rate": 4.489709113717864e-06, "loss": 0.0019, "num_tokens": 34977570.0, "reward": -0.3375, "reward_std": 0.2558668524026871, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.25586686432361605, "sampling/importance_sampling_ratio/max": 1.263705599308014, "sampling/importance_sampling_ratio/mean": 0.9999563097953796, "sampling/importance_sampling_ratio/min": 0.784296065568924, "sampling/sampling_logp_difference/max": 0.2550477385520935, "sampling/sampling_logp_difference/mean": 0.02131497785449028, "step": 8050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.2, "completions/max_terminated_length": 314.2, "completions/mean_length": 222.075, "completions/mean_terminated_length": 222.075, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.117838954180479, "epoch": 0.7410131470074469, "frac_reward_zero_std": 0.3, "grad_norm": 0.2061675488948822, "kl": 0.04983030980220064, "learning_rate": 4.488174110421397e-06, "loss": 0.0026, "num_tokens": 35023424.0, "reward": -0.1125, "reward_std": 0.3746744871139526, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.3746744990348816, "sampling/importance_sampling_ratio/max": 1.245806872844696, "sampling/importance_sampling_ratio/mean": 0.9993916511535644, "sampling/importance_sampling_ratio/min": 0.778225439786911, "sampling/sampling_logp_difference/max": 0.2540948510169983, "sampling/sampling_logp_difference/mean": 0.021714299358427523, "step": 8060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.0, "completions/max_terminated_length": 287.0, "completions/mean_length": 203.0375, "completions/mean_terminated_length": 203.0375, "completions/min_length": 143.9, "completions/min_terminated_length": 143.9, "entropy": 1.1539097726345062, "epoch": 0.7419325181575802, "frac_reward_zero_std": 0.3, "grad_norm": 0.1783839911222458, "kl": 0.03680320898420177, "learning_rate": 4.4866370651107475e-06, "loss": 0.0017, "num_tokens": 35065795.0, "reward": 0.1375, "reward_std": 0.32402919232845306, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.32402920424938203, "sampling/importance_sampling_ratio/max": 1.2686956644058227, "sampling/importance_sampling_ratio/mean": 1.0004583716392517, "sampling/importance_sampling_ratio/min": 0.7861164212226868, "sampling/sampling_logp_difference/max": 0.27084324359893797, "sampling/sampling_logp_difference/mean": 0.02113088108599186, "step": 8070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.8, "completions/max_terminated_length": 313.8, "completions/mean_length": 226.0875, "completions/mean_terminated_length": 226.0875, "completions/min_length": 148.1, "completions/min_terminated_length": 148.1, "entropy": 1.1743130065500735, "epoch": 0.7428518893077135, "frac_reward_zero_std": 0.1, "grad_norm": 0.1916358768939972, "kl": 0.04504839995643124, "learning_rate": 4.485097979364583e-06, "loss": 0.0018, "num_tokens": 35113226.0, "reward": -0.1375, "reward_std": 0.40190111100673676, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.4019011229276657, "sampling/importance_sampling_ratio/max": 1.2551014065742492, "sampling/importance_sampling_ratio/mean": 0.9998955011367798, "sampling/importance_sampling_ratio/min": 0.7922015309333801, "sampling/sampling_logp_difference/max": 0.2449549913406372, "sampling/sampling_logp_difference/mean": 0.02202475965023041, "step": 8080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 353.6, "completions/max_terminated_length": 353.6, "completions/mean_length": 236.7125, "completions/mean_terminated_length": 236.7125, "completions/min_length": 149.1, "completions/min_terminated_length": 149.1, "entropy": 1.1358940482139588, "epoch": 0.7437712604578468, "frac_reward_zero_std": 0.3, "grad_norm": 6.6515960693359375, "kl": 0.04084088265663013, "learning_rate": 4.48355685476367e-06, "loss": 0.0028, "num_tokens": 35159739.0, "reward": -0.2125, "reward_std": 0.31855743527412417, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.3185574531555176, "sampling/importance_sampling_ratio/max": 1.2688104152679442, "sampling/importance_sampling_ratio/mean": 0.9995947360992432, "sampling/importance_sampling_ratio/min": 0.7758252382278442, "sampling/sampling_logp_difference/max": 0.2809697389602661, "sampling/sampling_logp_difference/mean": 0.02172373775392771, "step": 8090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.0, "completions/max_terminated_length": 274.0, "completions/mean_length": 196.1375, "completions/mean_terminated_length": 196.1375, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.01872980594635, "epoch": 0.7446906316079801, "frac_reward_zero_std": 0.1, "grad_norm": 5.545493125915527, "kl": 0.05117638137890026, "learning_rate": 4.482013692890864e-06, "loss": 0.0029, "num_tokens": 35199878.0, "reward": 0.1625, "reward_std": 0.4064438194036484, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.4064438432455063, "sampling/importance_sampling_ratio/max": 1.2991944909095765, "sampling/importance_sampling_ratio/mean": 0.9996060609817505, "sampling/importance_sampling_ratio/min": 0.7771979153156281, "sampling/sampling_logp_difference/max": 0.27851874828338624, "sampling/sampling_logp_difference/mean": 0.020979069732129575, "step": 8100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 328.7, "completions/max_terminated_length": 310.4, "completions/mean_length": 227.5375, "completions/mean_terminated_length": 223.9232147216797, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.1326582320034504, "epoch": 0.7456100027581134, "frac_reward_zero_std": 0.1, "grad_norm": 8.607165336608887, "kl": 0.0513146938290447, "learning_rate": 4.480468495331118e-06, "loss": 0.0027, "num_tokens": 35246241.0, "reward": -0.08125, "reward_std": 0.4195474088191986, "rewards/reward_fn/mean": -0.08125, "rewards/reward_fn/std": 0.4195474207401276, "sampling/importance_sampling_ratio/max": 1.2275462746620178, "sampling/importance_sampling_ratio/mean": 0.9999100089073181, "sampling/importance_sampling_ratio/min": 0.7939382374286652, "sampling/sampling_logp_difference/max": 0.23643859028816222, "sampling/sampling_logp_difference/mean": 0.021156668290495874, "step": 8110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.6, "completions/max_terminated_length": 288.6, "completions/mean_length": 210.7375, "completions/mean_terminated_length": 210.7375, "completions/min_length": 138.7, "completions/min_terminated_length": 138.7, "entropy": 1.0426415480673312, "epoch": 0.7465293739082468, "frac_reward_zero_std": 0.3, "grad_norm": 0.14802688360214233, "kl": 0.057675226614810525, "learning_rate": 4.4789212636714745e-06, "loss": 0.0029, "num_tokens": 35290980.0, "reward": -0.0625, "reward_std": 0.3502548158168793, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.35025482773780825, "sampling/importance_sampling_ratio/max": 1.2510496973991394, "sampling/importance_sampling_ratio/mean": 1.0000108242034913, "sampling/importance_sampling_ratio/min": 0.7605468928813934, "sampling/sampling_logp_difference/max": 0.2885816752910614, "sampling/sampling_logp_difference/mean": 0.020778967626392843, "step": 8120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 320.4, "completions/max_terminated_length": 314.9, "completions/mean_length": 225.5125, "completions/mean_terminated_length": 223.22678833007814, "completions/min_length": 151.6, "completions/min_terminated_length": 151.6, "entropy": 1.0744621112942696, "epoch": 0.7474487450583801, "frac_reward_zero_std": 0.4, "grad_norm": 0.13565051555633545, "kl": 0.0431557557429187, "learning_rate": 4.477371999501063e-06, "loss": 0.0021, "num_tokens": 35338181.0, "reward": 0.25625, "reward_std": 0.3614010751247406, "rewards/reward_fn/mean": 0.25625, "rewards/reward_fn/std": 0.36140108704566953, "sampling/importance_sampling_ratio/max": 1.2702150106430055, "sampling/importance_sampling_ratio/mean": 0.999653285741806, "sampling/importance_sampling_ratio/min": 0.7316789954900742, "sampling/sampling_logp_difference/max": 0.34203912019729615, "sampling/sampling_logp_difference/mean": 0.021399160940200092, "step": 8130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 322.9, "completions/max_terminated_length": 318.5, "completions/mean_length": 235.1125, "completions/mean_terminated_length": 231.27916870117187, "completions/min_length": 163.6, "completions/min_terminated_length": 163.6, "entropy": 1.1023086324334144, "epoch": 0.7483681162085134, "frac_reward_zero_std": 0.2, "grad_norm": 5.571066856384277, "kl": 0.05457422552863136, "learning_rate": 4.475820704411102e-06, "loss": 0.0029, "num_tokens": 35384046.0, "reward": 0.0125, "reward_std": 0.38550153374671936, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.3855015397071838, "sampling/importance_sampling_ratio/max": 1.3074329614639282, "sampling/importance_sampling_ratio/mean": 0.9999190926551819, "sampling/importance_sampling_ratio/min": 0.7737188220024109, "sampling/sampling_logp_difference/max": 0.2918028891086578, "sampling/sampling_logp_difference/mean": 0.022051671892404555, "step": 8140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.8, "completions/max_terminated_length": 310.8, "completions/mean_length": 210.05, "completions/mean_terminated_length": 210.05, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 0.9659396767616272, "epoch": 0.7492874873586467, "frac_reward_zero_std": 0.7, "grad_norm": 0.13347794115543365, "kl": 0.04572153322515078, "learning_rate": 4.474267379994895e-06, "loss": 0.0024, "num_tokens": 35425834.0, "reward": 0.1, "reward_std": 0.17422050833702088, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.17422052025794982, "sampling/importance_sampling_ratio/max": 1.2609246611595153, "sampling/importance_sampling_ratio/mean": 0.9993985891342163, "sampling/importance_sampling_ratio/min": 0.7660029411315918, "sampling/sampling_logp_difference/max": 0.28041499853134155, "sampling/sampling_logp_difference/mean": 0.01976697351783514, "step": 8150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.4, "completions/max_terminated_length": 287.4, "completions/mean_length": 197.0875, "completions/mean_terminated_length": 197.0875, "completions/min_length": 125.7, "completions/min_terminated_length": 125.7, "entropy": 1.0424822442233563, "epoch": 0.75020685850878, "frac_reward_zero_std": 0.3, "grad_norm": 0.1294274628162384, "kl": 0.053996150474995376, "learning_rate": 4.4727120278478335e-06, "loss": 0.0026, "num_tokens": 35472529.0, "reward": 0.25, "reward_std": 0.3302692323923111, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.3302692502737045, "sampling/importance_sampling_ratio/max": 1.25041024684906, "sampling/importance_sampling_ratio/mean": 1.000388777256012, "sampling/importance_sampling_ratio/min": 0.7827749192714691, "sampling/sampling_logp_difference/max": 0.2536800980567932, "sampling/sampling_logp_difference/mean": 0.020770432986319066, "step": 8160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.0, "completions/max_terminated_length": 285.0, "completions/mean_length": 218.45, "completions/mean_terminated_length": 218.45, "completions/min_length": 155.9, "completions/min_terminated_length": 155.9, "entropy": 1.1214228980243206, "epoch": 0.7511262296589133, "frac_reward_zero_std": 0.7, "grad_norm": 0.1323217749595642, "kl": 0.04653022048296407, "learning_rate": 4.471154649567386e-06, "loss": 0.0019, "num_tokens": 35518901.0, "reward": 0.2625, "reward_std": 0.15235702097415924, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.15235702097415924, "sampling/importance_sampling_ratio/max": 1.2770470976829529, "sampling/importance_sampling_ratio/mean": 0.9999829590320587, "sampling/importance_sampling_ratio/min": 0.7996057629585266, "sampling/sampling_logp_difference/max": 0.24931695461273193, "sampling/sampling_logp_difference/mean": 0.02149878665804863, "step": 8170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 207.8625, "completions/mean_terminated_length": 207.8625, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.0842328920960427, "epoch": 0.7520456008090466, "frac_reward_zero_std": 0.5, "grad_norm": 6.630114555358887, "kl": 0.045263630733825264, "learning_rate": 4.469595246753107e-06, "loss": 0.0024, "num_tokens": 35563930.0, "reward": 0.1875, "reward_std": 0.3311904340982437, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3311904460191727, "sampling/importance_sampling_ratio/max": 1.2654990434646607, "sampling/importance_sampling_ratio/mean": 1.0001943409442902, "sampling/importance_sampling_ratio/min": 0.79794762134552, "sampling/sampling_logp_difference/max": 0.2562193155288696, "sampling/sampling_logp_difference/mean": 0.021051904559135436, "step": 8180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.5, "completions/max_terminated_length": 307.5, "completions/mean_length": 227.0125, "completions/mean_terminated_length": 227.0125, "completions/min_length": 135.6, "completions/min_terminated_length": 135.6, "entropy": 1.1410873807966708, "epoch": 0.7529649719591799, "frac_reward_zero_std": 0.4, "grad_norm": 5.307127475738525, "kl": 0.0383632734650746, "learning_rate": 4.4680338210066255e-06, "loss": 0.002, "num_tokens": 35607331.0, "reward": 0.1125, "reward_std": 0.26819893419742585, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.2681989401578903, "sampling/importance_sampling_ratio/max": 1.2538456916809082, "sampling/importance_sampling_ratio/mean": 1.000129634141922, "sampling/importance_sampling_ratio/min": 0.7794063210487365, "sampling/sampling_logp_difference/max": 0.2568260908126831, "sampling/sampling_logp_difference/mean": 0.021609990578144788, "step": 8190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.2, "completions/max_terminated_length": 292.2, "completions/mean_length": 221.9, "completions/mean_terminated_length": 221.9, "completions/min_length": 161.2, "completions/min_terminated_length": 161.2, "entropy": 1.1074596807360648, "epoch": 0.7538843431093132, "frac_reward_zero_std": 0.3, "grad_norm": 0.14383909106254578, "kl": 0.03385354392230511, "learning_rate": 4.466470373931654e-06, "loss": 0.002, "num_tokens": 35651603.0, "reward": -0.0375, "reward_std": 0.39694315791130064, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.39694316387176515, "sampling/importance_sampling_ratio/max": 1.2885071754455566, "sampling/importance_sampling_ratio/mean": 1.0000363349914552, "sampling/importance_sampling_ratio/min": 0.740203732252121, "sampling/sampling_logp_difference/max": 0.31600978374481203, "sampling/sampling_logp_difference/mean": 0.021144175529479982, "step": 8200 }, { "epoch": 0.7538843431093132, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 293.4066317626527, "eval_completions/max_terminated_length": 292.9738219895288, "eval_completions/mean_length": 206.5868237347295, "eval_completions/mean_terminated_length": 206.39631644843138, "eval_completions/min_length": 136.28097731239092, "eval_completions/min_terminated_length": 136.28097731239092, "eval_entropy": 1.0882980898932846, "eval_frac_reward_zero_std": 0.3263525305410122, "eval_kl": 0.041748599560498985, "eval_loss": 0.0021079054567962885, "eval_num_tokens": 35651603.0, "eval_reward": 0.060536649214659684, "eval_reward_std": 0.34079768567185126, "eval_rewards/reward_fn/mean": 0.060536649214659684, "eval_rewards/reward_fn/std": 0.34079769690623457, "eval_runtime": 1714.2684, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2691617898291943, "eval_sampling/importance_sampling_ratio/mean": 0.9999526114988078, "eval_sampling/importance_sampling_ratio/min": 0.775991971086041, "eval_sampling/sampling_logp_difference/max": 0.28232999689499955, "eval_sampling/sampling_logp_difference/mean": 0.020957368091203882, "eval_steps_per_second": 0.042, "step": 8200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 335.0, "completions/max_terminated_length": 319.3, "completions/mean_length": 223.9375, "completions/mean_terminated_length": 221.0017883300781, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.1713350355625152, "epoch": 0.7548037142594466, "frac_reward_zero_std": 0.3, "grad_norm": 0.09596233814954758, "kl": 0.0412192597577814, "learning_rate": 4.464904907133976e-06, "loss": 0.0025, "num_tokens": 35698430.0, "reward": -0.03125, "reward_std": 0.30431793332099916, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.3043179452419281, "sampling/importance_sampling_ratio/max": 1.2394259333610536, "sampling/importance_sampling_ratio/mean": 0.9992289543151855, "sampling/importance_sampling_ratio/min": 0.753474485874176, "sampling/sampling_logp_difference/max": 0.28635513484478, "sampling/sampling_logp_difference/mean": 0.02260472234338522, "step": 8210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.5, "completions/max_terminated_length": 289.5, "completions/mean_length": 211.5125, "completions/mean_terminated_length": 211.5125, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.1306281983852386, "epoch": 0.7557230854095799, "frac_reward_zero_std": 0.2, "grad_norm": 7.317781925201416, "kl": 0.04723062192206271, "learning_rate": 4.463337422221452e-06, "loss": 0.0022, "num_tokens": 35741655.0, "reward": -0.3375, "reward_std": 0.4089031368494034, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.40890314280986784, "sampling/importance_sampling_ratio/max": 1.2793290138244628, "sampling/importance_sampling_ratio/mean": 0.9998090744018555, "sampling/importance_sampling_ratio/min": 0.7788514912128448, "sampling/sampling_logp_difference/max": 0.3048470139503479, "sampling/sampling_logp_difference/mean": 0.02157476041465998, "step": 8220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 211.8125, "completions/mean_terminated_length": 211.8125, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.1112032204866409, "epoch": 0.7566424565597132, "frac_reward_zero_std": 0.5, "grad_norm": 0.12457574903964996, "kl": 0.04086814378388226, "learning_rate": 4.461767920804015e-06, "loss": 0.002, "num_tokens": 35784832.0, "reward": 0.1125, "reward_std": 0.24485630989074708, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.24485631585121154, "sampling/importance_sampling_ratio/max": 1.2734048962593079, "sampling/importance_sampling_ratio/mean": 1.0002968907356262, "sampling/importance_sampling_ratio/min": 0.7815337657928467, "sampling/sampling_logp_difference/max": 0.2746634900569916, "sampling/sampling_logp_difference/mean": 0.02131683360785246, "step": 8230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.8, "completions/max_terminated_length": 268.8, "completions/mean_length": 191.975, "completions/mean_terminated_length": 191.975, "completions/min_length": 129.6, "completions/min_terminated_length": 129.6, "entropy": 1.027645231038332, "epoch": 0.7575618277098465, "frac_reward_zero_std": 0.3, "grad_norm": 6.389520168304443, "kl": 0.06212011110037565, "learning_rate": 4.460196404493669e-06, "loss": 0.0031, "num_tokens": 35829398.0, "reward": -0.15, "reward_std": 0.33751319646835326, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.3375132083892822, "sampling/importance_sampling_ratio/max": 1.2784918308258058, "sampling/importance_sampling_ratio/mean": 1.0001433610916137, "sampling/importance_sampling_ratio/min": 0.7732639849185944, "sampling/sampling_logp_difference/max": 0.2856587052345276, "sampling/sampling_logp_difference/mean": 0.020899677462875844, "step": 8240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.9, "completions/max_terminated_length": 294.9, "completions/mean_length": 206.475, "completions/mean_terminated_length": 206.475, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.0497943833470345, "epoch": 0.7584811988599798, "frac_reward_zero_std": 0.6, "grad_norm": 0.15488189458847046, "kl": 0.05923373339464888, "learning_rate": 4.458622874904486e-06, "loss": 0.0028, "num_tokens": 35871420.0, "reward": -0.0625, "reward_std": 0.15235702097415924, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.15235702097415924, "sampling/importance_sampling_ratio/max": 1.2728855848312377, "sampling/importance_sampling_ratio/mean": 1.0003418028354645, "sampling/importance_sampling_ratio/min": 0.7701029598712921, "sampling/sampling_logp_difference/max": 0.2737884342670441, "sampling/sampling_logp_difference/mean": 0.02055224124342203, "step": 8250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.9, "completions/max_terminated_length": 281.9, "completions/mean_length": 197.1125, "completions/mean_terminated_length": 197.1125, "completions/min_length": 117.3, "completions/min_terminated_length": 117.3, "entropy": 1.1788907170295715, "epoch": 0.7594005700101131, "frac_reward_zero_std": 0.3, "grad_norm": 6.557623863220215, "kl": 0.050654699525330214, "learning_rate": 4.457047333652607e-06, "loss": 0.0025, "num_tokens": 35916149.0, "reward": -0.1, "reward_std": 0.3554430425167084, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3554430603981018, "sampling/importance_sampling_ratio/max": 1.2642478227615357, "sampling/importance_sampling_ratio/mean": 1.0001820981502534, "sampling/importance_sampling_ratio/min": 0.7880287289619445, "sampling/sampling_logp_difference/max": 0.2631549179553986, "sampling/sampling_logp_difference/mean": 0.023391783982515336, "step": 8260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.3, "completions/max_terminated_length": 255.3, "completions/mean_length": 178.975, "completions/mean_terminated_length": 178.975, "completions/min_length": 112.8, "completions/min_terminated_length": 112.8, "entropy": 0.984667555987835, "epoch": 0.7603199411602464, "frac_reward_zero_std": 0.3, "grad_norm": 6.660867691040039, "kl": 0.05467489522416145, "learning_rate": 4.455469782356241e-06, "loss": 0.0026, "num_tokens": 35955395.0, "reward": -0.175, "reward_std": 0.37809050679206846, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.37809052467346194, "sampling/importance_sampling_ratio/max": 1.2664636731147767, "sampling/importance_sampling_ratio/mean": 1.0007215559482574, "sampling/importance_sampling_ratio/min": 0.792838740348816, "sampling/sampling_logp_difference/max": 0.2565943658351898, "sampling/sampling_logp_difference/mean": 0.020006229542195796, "step": 8270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 278.1, "completions/max_terminated_length": 258.0, "completions/mean_length": 185.1, "completions/mean_terminated_length": 181.7964294433594, "completions/min_length": 115.7, "completions/min_terminated_length": 115.7, "entropy": 1.017738151550293, "epoch": 0.7612393123103797, "frac_reward_zero_std": 0.2, "grad_norm": 5.6502814292907715, "kl": 0.05798605147283524, "learning_rate": 4.453890222635657e-06, "loss": 0.0025, "num_tokens": 35992907.0, "reward": 0.09375, "reward_std": 0.4846843808889389, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.4846844047307968, "sampling/importance_sampling_ratio/max": 1.3162721157073975, "sampling/importance_sampling_ratio/mean": 1.0002390563488006, "sampling/importance_sampling_ratio/min": 0.7624096393585205, "sampling/sampling_logp_difference/max": 0.3455509454011917, "sampling/sampling_logp_difference/mean": 0.020331501960754395, "step": 8280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.6, "completions/max_terminated_length": 284.6, "completions/mean_length": 205.85, "completions/mean_terminated_length": 205.85, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.132000409066677, "epoch": 0.762158683460513, "frac_reward_zero_std": 0.5, "grad_norm": 6.36728048324585, "kl": 0.05303168012178503, "learning_rate": 4.452308656113191e-06, "loss": 0.0028, "num_tokens": 36037607.0, "reward": 0.075, "reward_std": 0.1948736011981964, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.1948736011981964, "sampling/importance_sampling_ratio/max": 1.2345200657844544, "sampling/importance_sampling_ratio/mean": 1.0002072334289551, "sampling/importance_sampling_ratio/min": 0.7656309545040131, "sampling/sampling_logp_difference/max": 0.27644087076187135, "sampling/sampling_logp_difference/mean": 0.021915152855217456, "step": 8290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.7, "completions/max_terminated_length": 281.7, "completions/mean_length": 195.025, "completions/mean_terminated_length": 195.025, "completions/min_length": 127.0, "completions/min_terminated_length": 127.0, "entropy": 1.1078555762767792, "epoch": 0.7630780546106464, "frac_reward_zero_std": 0.4, "grad_norm": 7.462594509124756, "kl": 0.05632182351546362, "learning_rate": 4.45072508441324e-06, "loss": 0.0031, "num_tokens": 36079249.0, "reward": -0.15, "reward_std": 0.2903633385896683, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.2903633564710617, "sampling/importance_sampling_ratio/max": 1.2656308650970458, "sampling/importance_sampling_ratio/mean": 0.9998750507831573, "sampling/importance_sampling_ratio/min": 0.777303683757782, "sampling/sampling_logp_difference/max": 0.28412245512008666, "sampling/sampling_logp_difference/mean": 0.02181570455431938, "step": 8300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.3, "completions/max_terminated_length": 298.3, "completions/mean_length": 203.3, "completions/mean_terminated_length": 203.3, "completions/min_length": 129.9, "completions/min_terminated_length": 129.9, "entropy": 1.2150594726204873, "epoch": 0.7639974257607797, "frac_reward_zero_std": 0.3, "grad_norm": 6.037957668304443, "kl": 0.062250671011861415, "learning_rate": 4.449139509162257e-06, "loss": 0.0029, "num_tokens": 36122201.0, "reward": 0.1, "reward_std": 0.2584230363368988, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.2584230363368988, "sampling/importance_sampling_ratio/max": 1.2610936641693116, "sampling/importance_sampling_ratio/mean": 1.0001720666885376, "sampling/importance_sampling_ratio/min": 0.7996842503547669, "sampling/sampling_logp_difference/max": 0.25071250200271605, "sampling/sampling_logp_difference/mean": 0.022956776432693005, "step": 8310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 268.7, "completions/max_terminated_length": 268.7, "completions/mean_length": 203.0375, "completions/mean_terminated_length": 203.0375, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.085725224763155, "epoch": 0.764916796910913, "frac_reward_zero_std": 0.4, "grad_norm": 0.1267528533935547, "kl": 0.04891264250036329, "learning_rate": 4.447551931988755e-06, "loss": 0.0017, "num_tokens": 36162444.0, "reward": 0.01875, "reward_std": 0.22178960591554642, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.2217896118760109, "sampling/importance_sampling_ratio/max": 1.3326180219650268, "sampling/importance_sampling_ratio/mean": 1.0002044022083283, "sampling/importance_sampling_ratio/min": 0.7759069800376892, "sampling/sampling_logp_difference/max": 0.3022372007369995, "sampling/sampling_logp_difference/mean": 0.021860329993069173, "step": 8320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 335.2, "completions/max_terminated_length": 335.2, "completions/mean_length": 224.9375, "completions/mean_terminated_length": 224.9375, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.1673907034099102, "epoch": 0.7658361680610463, "frac_reward_zero_std": 0.2, "grad_norm": 6.223412036895752, "kl": 0.06102771374862641, "learning_rate": 4.445962354523304e-06, "loss": 0.0032, "num_tokens": 36207863.0, "reward": 0.125, "reward_std": 0.3955783486366272, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.39557836055755613, "sampling/importance_sampling_ratio/max": 1.2773509860038756, "sampling/importance_sampling_ratio/mean": 0.9999799132347107, "sampling/importance_sampling_ratio/min": 0.7693881154060364, "sampling/sampling_logp_difference/max": 0.2882683277130127, "sampling/sampling_logp_difference/mean": 0.0226906631141901, "step": 8330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.3, "completions/max_terminated_length": 321.3, "completions/mean_length": 222.9125, "completions/mean_terminated_length": 222.9125, "completions/min_length": 139.2, "completions/min_terminated_length": 139.2, "entropy": 1.0983840510249139, "epoch": 0.7667555392111796, "frac_reward_zero_std": 0.5, "grad_norm": 5.316318511962891, "kl": 0.03722487888298929, "learning_rate": 4.4443707783985255e-06, "loss": 0.0016, "num_tokens": 36250808.0, "reward": 0.1875, "reward_std": 0.2592615157365799, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.2592615157365799, "sampling/importance_sampling_ratio/max": 1.2922040939331054, "sampling/importance_sampling_ratio/mean": 0.9997156381607055, "sampling/importance_sampling_ratio/min": 0.7839018762111664, "sampling/sampling_logp_difference/max": 0.2833583474159241, "sampling/sampling_logp_difference/mean": 0.021531708538532257, "step": 8340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 287.6, "completions/max_terminated_length": 287.6, "completions/mean_length": 215.275, "completions/mean_terminated_length": 215.275, "completions/min_length": 149.2, "completions/min_terminated_length": 149.2, "entropy": 1.0162071652710438, "epoch": 0.7676749103613129, "frac_reward_zero_std": 0.4, "grad_norm": 0.12623989582061768, "kl": 0.03177037722198293, "learning_rate": 4.442777205249098e-06, "loss": 0.0018, "num_tokens": 36294742.0, "reward": 0.125, "reward_std": 0.2958350956439972, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.29583510756492615, "sampling/importance_sampling_ratio/max": 1.249613606929779, "sampling/importance_sampling_ratio/mean": 1.0001385152339934, "sampling/importance_sampling_ratio/min": 0.7501398980617523, "sampling/sampling_logp_difference/max": 0.2947612404823303, "sampling/sampling_logp_difference/mean": 0.01980518698692322, "step": 8350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.9, "completions/max_terminated_length": 339.9, "completions/mean_length": 216.625, "completions/mean_terminated_length": 216.625, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.0463447362184524, "epoch": 0.7685942815114462, "frac_reward_zero_std": 0.3, "grad_norm": 5.614830493927002, "kl": 0.050177161960164085, "learning_rate": 4.441181636711749e-06, "loss": 0.0022, "num_tokens": 36338264.0, "reward": 0.3125, "reward_std": 0.3055525213479996, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.3055525213479996, "sampling/importance_sampling_ratio/max": 1.316318929195404, "sampling/importance_sampling_ratio/mean": 0.9998241186141967, "sampling/importance_sampling_ratio/min": 0.784780102968216, "sampling/sampling_logp_difference/max": 0.3033287763595581, "sampling/sampling_logp_difference/mean": 0.020822701789438725, "step": 8360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 326.7, "completions/max_terminated_length": 316.6, "completions/mean_length": 219.575, "completions/mean_terminated_length": 216.75, "completions/min_length": 112.9, "completions/min_terminated_length": 112.9, "entropy": 1.100361604243517, "epoch": 0.7695136526615795, "frac_reward_zero_std": 0.2, "grad_norm": 6.0107035636901855, "kl": 0.05771958347177133, "learning_rate": 4.439584074425253e-06, "loss": 0.003, "num_tokens": 36382454.0, "reward": -0.04375, "reward_std": 0.38596427738666533, "rewards/reward_fn/mean": -0.04375, "rewards/reward_fn/std": 0.3859642893075943, "sampling/importance_sampling_ratio/max": 1.3737538933753968, "sampling/importance_sampling_ratio/mean": 1.000322026014328, "sampling/importance_sampling_ratio/min": 0.7813075542449951, "sampling/sampling_logp_difference/max": 0.32148070335388185, "sampling/sampling_logp_difference/mean": 0.02222872506827116, "step": 8370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 342.7, "completions/max_terminated_length": 330.1, "completions/mean_length": 248.0125, "completions/mean_terminated_length": 246.0232147216797, "completions/min_length": 180.3, "completions/min_terminated_length": 180.3, "entropy": 1.088571298867464, "epoch": 0.7704330238117127, "frac_reward_zero_std": 0.1, "grad_norm": 5.949433326721191, "kl": 0.057909214548999444, "learning_rate": 4.437984520030437e-06, "loss": 0.0028, "num_tokens": 36427879.0, "reward": -0.00625, "reward_std": 0.3686827152967453, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.3686827152967453, "sampling/importance_sampling_ratio/max": 1.2526766061782837, "sampling/importance_sampling_ratio/mean": 1.0005345821380616, "sampling/importance_sampling_ratio/min": 0.7728150308132171, "sampling/sampling_logp_difference/max": 0.2652811765670776, "sampling/sampling_logp_difference/mean": 0.021244097501039505, "step": 8380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.7, "completions/max_terminated_length": 319.7, "completions/mean_length": 214.3125, "completions/mean_terminated_length": 214.3125, "completions/min_length": 141.9, "completions/min_terminated_length": 141.9, "entropy": 1.1342267915606499, "epoch": 0.771352394961846, "frac_reward_zero_std": 0.4, "grad_norm": 8.30804443359375, "kl": 0.049994938942836596, "learning_rate": 4.43638297517017e-06, "loss": 0.0029, "num_tokens": 36472184.0, "reward": 0.15, "reward_std": 0.2797048300504684, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.27970483601093293, "sampling/importance_sampling_ratio/max": 1.3075689911842345, "sampling/importance_sampling_ratio/mean": 0.9997446179389954, "sampling/importance_sampling_ratio/min": 0.7942805528640747, "sampling/sampling_logp_difference/max": 0.2780471324920654, "sampling/sampling_logp_difference/mean": 0.022112540528178216, "step": 8390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 341.8, "completions/max_terminated_length": 341.8, "completions/mean_length": 240.4875, "completions/mean_terminated_length": 240.4875, "completions/min_length": 154.8, "completions/min_terminated_length": 154.8, "entropy": 1.1278797470033168, "epoch": 0.7722717661119795, "frac_reward_zero_std": 0.4, "grad_norm": 7.3170485496521, "kl": 0.04881764885503799, "learning_rate": 4.434779441489367e-06, "loss": 0.0026, "num_tokens": 36519879.0, "reward": 0.05, "reward_std": 0.28029437065124513, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.28029437065124513, "sampling/importance_sampling_ratio/max": 1.3026459097862244, "sampling/importance_sampling_ratio/mean": 1.000385767221451, "sampling/importance_sampling_ratio/min": 0.7353048473596573, "sampling/sampling_logp_difference/max": 0.35137550830841063, "sampling/sampling_logp_difference/mean": 0.02200442273169756, "step": 8400 }, { "epoch": 0.7722717661119795, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.005671902268760907, "eval_completions/max_length": 330.49912739965094, "eval_completions/max_terminated_length": 326.1064572425829, "eval_completions/mean_length": 234.5556282722513, "eval_completions/mean_terminated_length": 233.35154417880128, "eval_completions/min_length": 158.89877835951134, "eval_completions/min_terminated_length": 158.89877835951134, "eval_entropy": 1.11524294308967, "eval_frac_reward_zero_std": 0.31413612565445026, "eval_kl": 0.04713486554832515, "eval_loss": 0.0024906403850764036, "eval_num_tokens": 36519879.0, "eval_reward": 0.053773996509598604, "eval_reward_std": 0.3388520177018164, "eval_rewards/reward_fn/mean": 0.053773996509598604, "eval_rewards/reward_fn/std": 0.33885202560749356, "eval_runtime": 1904.3541, "eval_samples_per_second": 0.301, "eval_sampling/importance_sampling_ratio/max": 1.276251434863758, "eval_sampling/importance_sampling_ratio/mean": 0.9999885669345423, "eval_sampling/importance_sampling_ratio/min": 0.7712784995167251, "eval_sampling/sampling_logp_difference/max": 0.28693026538294647, "eval_sampling/sampling_logp_difference/mean": 0.021595409305775977, "eval_steps_per_second": 0.038, "step": 8400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 342.1, "completions/max_terminated_length": 342.1, "completions/mean_length": 231.85, "completions/mean_terminated_length": 231.85, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.112854889780283, "epoch": 0.7731911372621127, "frac_reward_zero_std": 0.4, "grad_norm": 6.385634899139404, "kl": 0.05270067517412826, "learning_rate": 4.433173920634986e-06, "loss": 0.0028, "num_tokens": 36566155.0, "reward": -0.0375, "reward_std": 0.2972315102815628, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.29723151624202726, "sampling/importance_sampling_ratio/max": 1.294772219657898, "sampling/importance_sampling_ratio/mean": 0.9998943567276001, "sampling/importance_sampling_ratio/min": 0.7979962587356567, "sampling/sampling_logp_difference/max": 0.2656439423561096, "sampling/sampling_logp_difference/mean": 0.021819342859089375, "step": 8410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 323.2, "completions/max_terminated_length": 323.2, "completions/mean_length": 223.35, "completions/mean_terminated_length": 223.35, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.0673673093318938, "epoch": 0.774110508412246, "frac_reward_zero_std": 0.5, "grad_norm": 6.54659366607666, "kl": 0.03772438338492066, "learning_rate": 4.431566414256026e-06, "loss": 0.0018, "num_tokens": 36611103.0, "reward": 0.3125, "reward_std": 0.2687691628932953, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.2687691688537598, "sampling/importance_sampling_ratio/max": 1.267129933834076, "sampling/importance_sampling_ratio/mean": 1.0002500474452973, "sampling/importance_sampling_ratio/min": 0.7844542920589447, "sampling/sampling_logp_difference/max": 0.25771968364715575, "sampling/sampling_logp_difference/mean": 0.0217557854950428, "step": 8420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.1, "completions/max_terminated_length": 307.1, "completions/mean_length": 221.9875, "completions/mean_terminated_length": 221.9875, "completions/min_length": 149.3, "completions/min_terminated_length": 149.3, "entropy": 1.18791157156229, "epoch": 0.7750298795623793, "frac_reward_zero_std": 0.3, "grad_norm": 6.5652642250061035, "kl": 0.05099186827428639, "learning_rate": 4.4299569240035245e-06, "loss": 0.0022, "num_tokens": 36659958.0, "reward": 0.0, "reward_std": 0.3742649495601654, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3742649555206299, "sampling/importance_sampling_ratio/max": 1.2980244398117065, "sampling/importance_sampling_ratio/mean": 1.0004709303379058, "sampling/importance_sampling_ratio/min": 0.7510793507099152, "sampling/sampling_logp_difference/max": 0.31801933646202085, "sampling/sampling_logp_difference/mean": 0.023117323778569697, "step": 8430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 306.6, "completions/max_terminated_length": 287.7, "completions/mean_length": 219.3625, "completions/mean_terminated_length": 216.1732147216797, "completions/min_length": 150.2, "completions/min_terminated_length": 150.2, "entropy": 1.1362251348793506, "epoch": 0.7759492507125126, "frac_reward_zero_std": 0.4, "grad_norm": 7.763079643249512, "kl": 0.0665880466811359, "learning_rate": 4.428345451530557e-06, "loss": 0.0033, "num_tokens": 36704003.0, "reward": -0.16875, "reward_std": 0.2571449428796768, "rewards/reward_fn/mean": -0.16875, "rewards/reward_fn/std": 0.2571449488401413, "sampling/importance_sampling_ratio/max": 1.2639767050743103, "sampling/importance_sampling_ratio/mean": 1.0002585172653198, "sampling/importance_sampling_ratio/min": 0.7485855638980865, "sampling/sampling_logp_difference/max": 0.29659217596054077, "sampling/sampling_logp_difference/mean": 0.022258792631328107, "step": 8440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.2, "completions/max_terminated_length": 272.2, "completions/mean_length": 192.9625, "completions/mean_terminated_length": 192.9625, "completions/min_length": 132.7, "completions/min_terminated_length": 132.7, "entropy": 0.9923920854926109, "epoch": 0.7768686218626459, "frac_reward_zero_std": 0.7, "grad_norm": 0.36787649989128113, "kl": 0.05050960383960046, "learning_rate": 4.426731998492236e-06, "loss": 0.0024, "num_tokens": 36741600.0, "reward": 0.3125, "reward_std": 0.12793734967708587, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.12793734967708587, "sampling/importance_sampling_ratio/max": 1.259957730770111, "sampling/importance_sampling_ratio/mean": 0.9999345541000366, "sampling/importance_sampling_ratio/min": 0.7655860543251037, "sampling/sampling_logp_difference/max": 0.29106817245483396, "sampling/sampling_logp_difference/mean": 0.021167710050940515, "step": 8450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 290.1, "completions/max_terminated_length": 279.8, "completions/mean_length": 202.25, "completions/mean_terminated_length": 199.81785888671874, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.0233232654631137, "epoch": 0.7777879930127792, "frac_reward_zero_std": 0.1, "grad_norm": 6.527560710906982, "kl": 0.048779568687314166, "learning_rate": 4.425116566545706e-06, "loss": 0.0025, "num_tokens": 36786372.0, "reward": -0.06875, "reward_std": 0.45328661799430847, "rewards/reward_fn/mean": -0.06875, "rewards/reward_fn/std": 0.45328662991523744, "sampling/importance_sampling_ratio/max": 1.330122721195221, "sampling/importance_sampling_ratio/mean": 1.0000096380710601, "sampling/importance_sampling_ratio/min": 0.7576094567775726, "sampling/sampling_logp_difference/max": 0.3212560206651688, "sampling/sampling_logp_difference/mean": 0.021010516956448556, "step": 8460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.5, "completions/max_terminated_length": 303.7, "completions/mean_length": 222.3, "completions/mean_terminated_length": 219.8, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.1069677479565143, "epoch": 0.7787073641629125, "frac_reward_zero_std": 0.2, "grad_norm": 6.056061267852783, "kl": 0.05582822843571193, "learning_rate": 4.4234991573501426e-06, "loss": 0.0029, "num_tokens": 36832700.0, "reward": 0.01875, "reward_std": 0.40446448028087617, "rewards/reward_fn/mean": 0.01875, "rewards/reward_fn/std": 0.4044644981622696, "sampling/importance_sampling_ratio/max": 1.2818525791168214, "sampling/importance_sampling_ratio/mean": 1.0005426943302154, "sampling/importance_sampling_ratio/min": 0.7787148594856262, "sampling/sampling_logp_difference/max": 0.2689772129058838, "sampling/sampling_logp_difference/mean": 0.021798519976437093, "step": 8470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 213.5, "completions/mean_terminated_length": 213.5, "completions/min_length": 123.8, "completions/min_terminated_length": 123.8, "entropy": 1.1976394198834897, "epoch": 0.7796267353130458, "frac_reward_zero_std": 0.4, "grad_norm": 6.758187294006348, "kl": 0.04055676264688372, "learning_rate": 4.42187977256676e-06, "loss": 0.0018, "num_tokens": 36876428.0, "reward": -0.075, "reward_std": 0.3095962584018707, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.3095962584018707, "sampling/importance_sampling_ratio/max": 1.3068729877471923, "sampling/importance_sampling_ratio/mean": 0.9999541759490966, "sampling/importance_sampling_ratio/min": 0.7789978623390198, "sampling/sampling_logp_difference/max": 0.303797447681427, "sampling/sampling_logp_difference/mean": 0.022847374342381953, "step": 8480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.2, "completions/max_terminated_length": 315.2, "completions/mean_length": 204.15, "completions/mean_terminated_length": 204.15, "completions/min_length": 144.5, "completions/min_terminated_length": 144.5, "entropy": 1.0322167932987214, "epoch": 0.7805461064631792, "frac_reward_zero_std": 0.5, "grad_norm": 0.10412130504846573, "kl": 0.040058744698762895, "learning_rate": 4.420258413858792e-06, "loss": 0.0015, "num_tokens": 36916408.0, "reward": 0.2, "reward_std": 0.22051151394844054, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.2205115258693695, "sampling/importance_sampling_ratio/max": 1.312811589241028, "sampling/importance_sampling_ratio/mean": 1.0000422835350036, "sampling/importance_sampling_ratio/min": 0.7942772030830383, "sampling/sampling_logp_difference/max": 0.277325439453125, "sampling/sampling_logp_difference/mean": 0.02104961033910513, "step": 8490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 221.175, "completions/mean_terminated_length": 221.175, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.1144550062716008, "epoch": 0.7814654776133125, "frac_reward_zero_std": 0.4, "grad_norm": 0.13495837152004242, "kl": 0.04637873565661721, "learning_rate": 4.418635082891506e-06, "loss": 0.0019, "num_tokens": 36962110.0, "reward": -0.1625, "reward_std": 0.2765199452638626, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.2765199452638626, "sampling/importance_sampling_ratio/max": 1.2671054244041442, "sampling/importance_sampling_ratio/mean": 0.9998886525630951, "sampling/importance_sampling_ratio/min": 0.7846600711345673, "sampling/sampling_logp_difference/max": 0.2706998348236084, "sampling/sampling_logp_difference/mean": 0.021386586502194403, "step": 8500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.0, "completions/max_terminated_length": 317.0, "completions/mean_length": 213.8, "completions/mean_terminated_length": 213.8, "completions/min_length": 142.2, "completions/min_terminated_length": 142.2, "entropy": 1.0674708582460881, "epoch": 0.7823848487634458, "frac_reward_zero_std": 0.3, "grad_norm": 7.907252311706543, "kl": 0.04561225168872625, "learning_rate": 4.417009781332192e-06, "loss": 0.0023, "num_tokens": 37007134.0, "reward": 0.225, "reward_std": 0.35502841174602506, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3550284177064896, "sampling/importance_sampling_ratio/max": 1.2598554491996765, "sampling/importance_sampling_ratio/mean": 0.9996153533458709, "sampling/importance_sampling_ratio/min": 0.7693885743618012, "sampling/sampling_logp_difference/max": 0.2738891065120697, "sampling/sampling_logp_difference/mean": 0.021134657226502894, "step": 8510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.0, "completions/max_terminated_length": 297.0, "completions/mean_length": 213.0375, "completions/mean_terminated_length": 213.0375, "completions/min_length": 134.5, "completions/min_terminated_length": 134.5, "entropy": 1.042586360499263, "epoch": 0.7833042199135791, "frac_reward_zero_std": 0.5, "grad_norm": 6.600070953369141, "kl": 0.04060280580306426, "learning_rate": 4.4153825108501665e-06, "loss": 0.0021, "num_tokens": 37047113.0, "reward": 0.3375, "reward_std": 0.25210027396678925, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.25210027396678925, "sampling/importance_sampling_ratio/max": 1.2477716207504272, "sampling/importance_sampling_ratio/mean": 0.9993491470813751, "sampling/importance_sampling_ratio/min": 0.782102108001709, "sampling/sampling_logp_difference/max": 0.258186274766922, "sampling/sampling_logp_difference/mean": 0.02058903593569994, "step": 8520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 352.2, "completions/max_terminated_length": 328.6, "completions/mean_length": 227.3625, "completions/mean_terminated_length": 220.95178680419923, "completions/min_length": 150.3, "completions/min_terminated_length": 150.3, "entropy": 1.085213840007782, "epoch": 0.7842235910637124, "frac_reward_zero_std": 0.3, "grad_norm": 4.411600112915039, "kl": 0.04907072086352855, "learning_rate": 4.4137532731167646e-06, "loss": 0.0019, "num_tokens": 37094414.0, "reward": 0.125, "reward_std": 0.3953360438346863, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3953360617160797, "sampling/importance_sampling_ratio/max": 1.3134648680686951, "sampling/importance_sampling_ratio/mean": 1.0001985907554627, "sampling/importance_sampling_ratio/min": 0.7660737335681915, "sampling/sampling_logp_difference/max": 0.32343039512634275, "sampling/sampling_logp_difference/mean": 0.021011572517454623, "step": 8530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.3, "completions/max_terminated_length": 308.3, "completions/mean_length": 223.225, "completions/mean_terminated_length": 223.225, "completions/min_length": 165.0, "completions/min_terminated_length": 165.0, "entropy": 1.0233835220336913, "epoch": 0.7851429622138457, "frac_reward_zero_std": 0.2, "grad_norm": 0.15121251344680786, "kl": 0.061595835560001436, "learning_rate": 4.412122069805345e-06, "loss": 0.0028, "num_tokens": 37139128.0, "reward": 0.125, "reward_std": 0.31187528371810913, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.31187528371810913, "sampling/importance_sampling_ratio/max": 1.2777536392211915, "sampling/importance_sampling_ratio/mean": 0.9998096585273742, "sampling/importance_sampling_ratio/min": 0.7663146078586578, "sampling/sampling_logp_difference/max": 0.2899179458618164, "sampling/sampling_logp_difference/mean": 0.02097825426608324, "step": 8540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.4, "completions/max_terminated_length": 283.4, "completions/mean_length": 201.075, "completions/mean_terminated_length": 201.075, "completions/min_length": 122.8, "completions/min_terminated_length": 122.8, "entropy": 1.0402743242681027, "epoch": 0.786062333363979, "frac_reward_zero_std": 0.4, "grad_norm": 0.13226866722106934, "kl": 0.05446178841521032, "learning_rate": 4.410488902591286e-06, "loss": 0.003, "num_tokens": 37182806.0, "reward": -0.025, "reward_std": 0.28210032284259795, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.28210032284259795, "sampling/importance_sampling_ratio/max": 1.2301283121109008, "sampling/importance_sampling_ratio/mean": 0.9996185660362243, "sampling/importance_sampling_ratio/min": 0.7971519410610199, "sampling/sampling_logp_difference/max": 0.2421970248222351, "sampling/sampling_logp_difference/mean": 0.020698485244065524, "step": 8550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.4, "completions/max_terminated_length": 279.4, "completions/mean_length": 200.75, "completions/mean_terminated_length": 200.75, "completions/min_length": 127.1, "completions/min_terminated_length": 127.1, "entropy": 1.077260223031044, "epoch": 0.7869817045141123, "frac_reward_zero_std": 0.3, "grad_norm": 0.15907880663871765, "kl": 0.03778332668007352, "learning_rate": 4.408853773151977e-06, "loss": 0.0019, "num_tokens": 37224002.0, "reward": 0.2875, "reward_std": 0.2912300378084183, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.2912300378084183, "sampling/importance_sampling_ratio/max": 1.2608644247055054, "sampling/importance_sampling_ratio/mean": 0.9998204052448273, "sampling/importance_sampling_ratio/min": 0.7914727687835693, "sampling/sampling_logp_difference/max": 0.24903409481048583, "sampling/sampling_logp_difference/mean": 0.021366760693490507, "step": 8560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.7, "completions/max_terminated_length": 290.7, "completions/mean_length": 204.7625, "completions/mean_terminated_length": 204.7625, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.0591071791946889, "epoch": 0.7879010756642456, "frac_reward_zero_std": 0.5, "grad_norm": 5.243019104003906, "kl": 0.039074685383820904, "learning_rate": 4.407216683166832e-06, "loss": 0.002, "num_tokens": 37266903.0, "reward": 0.0875, "reward_std": 0.2692277133464813, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.2692277133464813, "sampling/importance_sampling_ratio/max": 1.292909562587738, "sampling/importance_sampling_ratio/mean": 1.0000833988189697, "sampling/importance_sampling_ratio/min": 0.7738879501819611, "sampling/sampling_logp_difference/max": 0.2781393527984619, "sampling/sampling_logp_difference/mean": 0.021012044697999954, "step": 8570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.6, "completions/max_terminated_length": 316.6, "completions/mean_length": 236.5625, "completions/mean_terminated_length": 236.5625, "completions/min_length": 155.1, "completions/min_terminated_length": 155.1, "entropy": 1.0437770657241345, "epoch": 0.788820446814379, "frac_reward_zero_std": 0.2, "grad_norm": 5.6960649490356445, "kl": 0.03853895569918677, "learning_rate": 4.40557763431727e-06, "loss": 0.0023, "num_tokens": 37314996.0, "reward": -0.0875, "reward_std": 0.41183285415172577, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.4118328779935837, "sampling/importance_sampling_ratio/max": 1.269897961616516, "sampling/importance_sampling_ratio/mean": 0.9999617874622345, "sampling/importance_sampling_ratio/min": 0.7577203273773193, "sampling/sampling_logp_difference/max": 0.287166428565979, "sampling/sampling_logp_difference/mean": 0.020373632572591303, "step": 8580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 213.375, "completions/mean_terminated_length": 213.375, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.085709285736084, "epoch": 0.7897398179645123, "frac_reward_zero_std": 0.2, "grad_norm": 6.100148677825928, "kl": 0.04313471909845248, "learning_rate": 4.403936628286727e-06, "loss": 0.003, "num_tokens": 37358770.0, "reward": 0.025, "reward_std": 0.3734060972929001, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.37340610921382905, "sampling/importance_sampling_ratio/max": 1.2621408581733704, "sampling/importance_sampling_ratio/mean": 0.9998152911663055, "sampling/importance_sampling_ratio/min": 0.7815590083599091, "sampling/sampling_logp_difference/max": 0.2683415412902832, "sampling/sampling_logp_difference/mean": 0.021751171723008157, "step": 8590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.0, "completions/max_terminated_length": 277.0, "completions/mean_length": 189.1875, "completions/mean_terminated_length": 189.1875, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.159361157566309, "epoch": 0.7906591891146456, "frac_reward_zero_std": 0.2, "grad_norm": 0.11463578790426254, "kl": 0.030379978602286428, "learning_rate": 4.402293666760648e-06, "loss": 0.0025, "num_tokens": 37400577.0, "reward": 0.4125, "reward_std": 0.4978660613298416, "rewards/reward_fn/mean": 0.4125, "rewards/reward_fn/std": 0.497866091132164, "sampling/importance_sampling_ratio/max": 1.2752498269081116, "sampling/importance_sampling_ratio/mean": 0.9999164402484894, "sampling/importance_sampling_ratio/min": 0.793759036064148, "sampling/sampling_logp_difference/max": 0.27189621925354, "sampling/sampling_logp_difference/mean": 0.022642544098198415, "step": 8600 }, { "epoch": 0.7906591891146456, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002399650959860384, "eval_completions/max_length": 296.16579406631763, "eval_completions/max_terminated_length": 293.4694589877836, "eval_completions/mean_length": 208.75239965095986, "eval_completions/mean_terminated_length": 208.05810917027125, "eval_completions/min_length": 135.22513089005236, "eval_completions/min_terminated_length": 135.22513089005236, "eval_entropy": 1.089236964574035, "eval_frac_reward_zero_std": 0.28970331588132636, "eval_kl": 0.03935647829001799, "eval_loss": 0.001974169397726655, "eval_num_tokens": 37400577.0, "eval_reward": 0.054864746945898775, "eval_reward_std": 0.35963230924664574, "eval_rewards/reward_fn/mean": 0.054864746945898775, "eval_rewards/reward_fn/std": 0.35963231985689664, "eval_runtime": 1731.2316, "eval_samples_per_second": 0.331, "eval_sampling/importance_sampling_ratio/max": 1.2659988542709883, "eval_sampling/importance_sampling_ratio/mean": 0.9999380022235358, "eval_sampling/importance_sampling_ratio/min": 0.7737978625568004, "eval_sampling/sampling_logp_difference/max": 0.28144922056747357, "eval_sampling/sampling_logp_difference/mean": 0.02118139997950347, "eval_steps_per_second": 0.042, "step": 8600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.1, "completions/max_terminated_length": 301.1, "completions/mean_length": 198.225, "completions/mean_terminated_length": 198.225, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.118923868983984, "epoch": 0.7915785602647789, "frac_reward_zero_std": 0.3, "grad_norm": 6.592360973358154, "kl": 0.04915940761566162, "learning_rate": 4.4006487514264855e-06, "loss": 0.0023, "num_tokens": 37444339.0, "reward": 0.125, "reward_std": 0.30762176811695097, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.30762178599834444, "sampling/importance_sampling_ratio/max": 1.2808159470558167, "sampling/importance_sampling_ratio/mean": 0.9999868094921112, "sampling/importance_sampling_ratio/min": 0.7918745994567871, "sampling/sampling_logp_difference/max": 0.26720079183578493, "sampling/sampling_logp_difference/mean": 0.022386537864804268, "step": 8610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.9, "completions/max_terminated_length": 322.9, "completions/mean_length": 230.6375, "completions/mean_terminated_length": 230.6375, "completions/min_length": 151.9, "completions/min_terminated_length": 151.9, "entropy": 1.0901392698287964, "epoch": 0.7924979314149122, "frac_reward_zero_std": 0.6, "grad_norm": 0.12871846556663513, "kl": 0.036224769288674, "learning_rate": 4.399001883973699e-06, "loss": 0.002, "num_tokens": 37494582.0, "reward": 0.0875, "reward_std": 0.18515617549419403, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.185156187415123, "sampling/importance_sampling_ratio/max": 1.2872913479804993, "sampling/importance_sampling_ratio/mean": 1.000032478570938, "sampling/importance_sampling_ratio/min": 0.765818428993225, "sampling/sampling_logp_difference/max": 0.286179518699646, "sampling/sampling_logp_difference/mean": 0.02144911829382181, "step": 8620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.3, "completions/max_terminated_length": 258.3, "completions/mean_length": 197.9375, "completions/mean_terminated_length": 197.9375, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0698160007596016, "epoch": 0.7934173025650455, "frac_reward_zero_std": 0.1, "grad_norm": 4.682737827301025, "kl": 0.03345448941690847, "learning_rate": 4.397353066093755e-06, "loss": 0.002, "num_tokens": 37538097.0, "reward": -0.25, "reward_std": 0.45998663604259493, "rewards/reward_fn/mean": -0.25, "rewards/reward_fn/std": 0.4599866420030594, "sampling/importance_sampling_ratio/max": 1.233576202392578, "sampling/importance_sampling_ratio/mean": 0.9997283637523651, "sampling/importance_sampling_ratio/min": 0.7977955639362335, "sampling/sampling_logp_difference/max": 0.2359161615371704, "sampling/sampling_logp_difference/mean": 0.02136957421898842, "step": 8630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 324.8, "completions/max_terminated_length": 324.8, "completions/mean_length": 214.4125, "completions/mean_terminated_length": 214.4125, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0040601417422295, "epoch": 0.7943366737151788, "frac_reward_zero_std": 0.3, "grad_norm": 4.951817512512207, "kl": 0.04786549654672854, "learning_rate": 4.39570229948012e-06, "loss": 0.0027, "num_tokens": 37579002.0, "reward": -0.3625, "reward_std": 0.353536981344223, "rewards/reward_fn/mean": -0.3625, "rewards/reward_fn/std": 0.353536993265152, "sampling/importance_sampling_ratio/max": 1.3053579568862914, "sampling/importance_sampling_ratio/mean": 1.0010092914104463, "sampling/importance_sampling_ratio/min": 0.7833929717540741, "sampling/sampling_logp_difference/max": 0.2886221408843994, "sampling/sampling_logp_difference/mean": 0.020288372226059437, "step": 8640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 209.6375, "completions/mean_terminated_length": 209.6375, "completions/min_length": 125.5, "completions/min_terminated_length": 125.5, "entropy": 1.0167246103286742, "epoch": 0.7952560448653121, "frac_reward_zero_std": 0.1, "grad_norm": 7.263909339904785, "kl": 0.05170157368993387, "learning_rate": 4.394049585828265e-06, "loss": 0.0026, "num_tokens": 37621261.0, "reward": -0.0125, "reward_std": 0.41830068826675415, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.41830070614814757, "sampling/importance_sampling_ratio/max": 1.3136829137802124, "sampling/importance_sampling_ratio/mean": 0.9995709776878356, "sampling/importance_sampling_ratio/min": 0.7284458369016648, "sampling/sampling_logp_difference/max": 0.37468955516815183, "sampling/sampling_logp_difference/mean": 0.02126057129353285, "step": 8650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 353.4, "completions/max_terminated_length": 353.4, "completions/mean_length": 230.4125, "completions/mean_terminated_length": 230.4125, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.03956740796566, "epoch": 0.7961754160154454, "frac_reward_zero_std": 0.2, "grad_norm": 6.940287113189697, "kl": 0.050082489562919365, "learning_rate": 4.392394926835658e-06, "loss": 0.0026, "num_tokens": 37665110.0, "reward": -0.05, "reward_std": 0.3409078598022461, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.3409078598022461, "sampling/importance_sampling_ratio/max": 1.260983943939209, "sampling/importance_sampling_ratio/mean": 0.999590802192688, "sampling/importance_sampling_ratio/min": 0.8059561014175415, "sampling/sampling_logp_difference/max": 0.2408542037010193, "sampling/sampling_logp_difference/mean": 0.020256057381629944, "step": 8660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.3, "completions/max_terminated_length": 326.3, "completions/mean_length": 226.2, "completions/mean_terminated_length": 226.2, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.09537383466959, "epoch": 0.7970947871655787, "frac_reward_zero_std": 0.3, "grad_norm": 5.319007873535156, "kl": 0.05135476163122803, "learning_rate": 4.390738324201767e-06, "loss": 0.0023, "num_tokens": 37711510.0, "reward": 0.075, "reward_std": 0.37277381420135497, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.37277381420135497, "sampling/importance_sampling_ratio/max": 1.289265477657318, "sampling/importance_sampling_ratio/mean": 1.0001185178756713, "sampling/importance_sampling_ratio/min": 0.7453424364328385, "sampling/sampling_logp_difference/max": 0.35207270085811615, "sampling/sampling_logp_difference/mean": 0.021006496623158455, "step": 8670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.5, "completions/max_terminated_length": 301.5, "completions/mean_length": 221.0, "completions/mean_terminated_length": 221.0, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.035111054778099, "epoch": 0.7980141583157121, "frac_reward_zero_std": 0.1, "grad_norm": 5.857322692871094, "kl": 0.042267977393930775, "learning_rate": 4.389079779628057e-06, "loss": 0.0022, "num_tokens": 37758558.0, "reward": -0.1, "reward_std": 0.4968202650547028, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4968202769756317, "sampling/importance_sampling_ratio/max": 1.2256409406661988, "sampling/importance_sampling_ratio/mean": 1.0000116884708405, "sampling/importance_sampling_ratio/min": 0.7691984653472901, "sampling/sampling_logp_difference/max": 0.28101028203964235, "sampling/sampling_logp_difference/mean": 0.020300885289907457, "step": 8680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 287.2, "completions/max_terminated_length": 278.2, "completions/mean_length": 203.125, "completions/mean_terminated_length": 200.90357360839843, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.0503442004323005, "epoch": 0.7989335294658454, "frac_reward_zero_std": 0.3, "grad_norm": 0.12584029138088226, "kl": 0.04647665779921226, "learning_rate": 4.3874192948179836e-06, "loss": 0.0025, "num_tokens": 37801976.0, "reward": 0.00625, "reward_std": 0.45274376273155215, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.45274377465248106, "sampling/importance_sampling_ratio/max": 1.2915782809257508, "sampling/importance_sampling_ratio/mean": 1.0004492461681367, "sampling/importance_sampling_ratio/min": 0.7633813917636871, "sampling/sampling_logp_difference/max": 0.32255802154541013, "sampling/sampling_logp_difference/mean": 0.021532826870679856, "step": 8690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.9, "completions/max_terminated_length": 273.9, "completions/mean_length": 209.0125, "completions/mean_terminated_length": 209.0125, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.0974105402827263, "epoch": 0.7998529006159787, "frac_reward_zero_std": 0.5, "grad_norm": 6.500042915344238, "kl": 0.03251999069470912, "learning_rate": 4.385756871477e-06, "loss": 0.0016, "num_tokens": 37843193.0, "reward": 0.225, "reward_std": 0.22675940096378328, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.22675940692424773, "sampling/importance_sampling_ratio/max": 1.2706981897354126, "sampling/importance_sampling_ratio/mean": 1.00041384100914, "sampling/importance_sampling_ratio/min": 0.7782668471336365, "sampling/sampling_logp_difference/max": 0.2739842295646667, "sampling/sampling_logp_difference/mean": 0.021877089887857436, "step": 8700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 251.5, "completions/max_terminated_length": 251.5, "completions/mean_length": 183.5375, "completions/mean_terminated_length": 183.5375, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.075262963399291, "epoch": 0.800772271766112, "frac_reward_zero_std": 0.3, "grad_norm": 0.26422399282455444, "kl": 0.04808458702173084, "learning_rate": 4.384092511312547e-06, "loss": 0.0026, "num_tokens": 37882668.0, "reward": -0.2125, "reward_std": 0.353536981344223, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.353536993265152, "sampling/importance_sampling_ratio/max": 1.2109286904335022, "sampling/importance_sampling_ratio/mean": 0.9996584713459015, "sampling/importance_sampling_ratio/min": 0.7896126568317413, "sampling/sampling_logp_difference/max": 0.24269652962684632, "sampling/sampling_logp_difference/mean": 0.021072769351303578, "step": 8710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 258.4, "completions/max_terminated_length": 258.4, "completions/mean_length": 188.4625, "completions/mean_terminated_length": 188.4625, "completions/min_length": 126.6, "completions/min_terminated_length": 126.6, "entropy": 0.9797229453921318, "epoch": 0.8016916429162453, "frac_reward_zero_std": 0.2, "grad_norm": 5.8295722007751465, "kl": 0.0408133186516352, "learning_rate": 4.382426216034057e-06, "loss": 0.0023, "num_tokens": 37920569.0, "reward": -0.0375, "reward_std": 0.44623149037361143, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.44623149037361143, "sampling/importance_sampling_ratio/max": 1.2410557627677918, "sampling/importance_sampling_ratio/mean": 1.0003779232501984, "sampling/importance_sampling_ratio/min": 0.7973136365413666, "sampling/sampling_logp_difference/max": 0.2386075735092163, "sampling/sampling_logp_difference/mean": 0.02035735584795475, "step": 8720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.4, "completions/max_terminated_length": 316.4, "completions/mean_length": 224.5625, "completions/mean_terminated_length": 224.5625, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.047971683740616, "epoch": 0.8026110140663786, "frac_reward_zero_std": 0.1, "grad_norm": 6.982512474060059, "kl": 0.05426092586712912, "learning_rate": 4.380757987352948e-06, "loss": 0.003, "num_tokens": 37964982.0, "reward": 0.0875, "reward_std": 0.3955159991979599, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.39551601111888884, "sampling/importance_sampling_ratio/max": 1.3352445602416991, "sampling/importance_sampling_ratio/mean": 1.0002695739269256, "sampling/importance_sampling_ratio/min": 0.7990143358707428, "sampling/sampling_logp_difference/max": 0.291439151763916, "sampling/sampling_logp_difference/mean": 0.02060706652700901, "step": 8730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 215.7875, "completions/mean_terminated_length": 215.7875, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.086515760421753, "epoch": 0.8035303852165119, "frac_reward_zero_std": 0.3, "grad_norm": 6.220156192779541, "kl": 0.040553272410761565, "learning_rate": 4.379087826982625e-06, "loss": 0.002, "num_tokens": 38010005.0, "reward": 0.19375, "reward_std": 0.3894675612449646, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.3894675672054291, "sampling/importance_sampling_ratio/max": 1.2983322143554688, "sampling/importance_sampling_ratio/mean": 0.999848073720932, "sampling/importance_sampling_ratio/min": 0.781996202468872, "sampling/sampling_logp_difference/max": 0.2927091121673584, "sampling/sampling_logp_difference/mean": 0.021740529499948025, "step": 8740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 336.3, "completions/max_terminated_length": 336.3, "completions/mean_length": 221.6125, "completions/mean_terminated_length": 221.6125, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0934100411832333, "epoch": 0.8044497563666452, "frac_reward_zero_std": 0.5, "grad_norm": 0.16055728495121002, "kl": 0.047162325971294196, "learning_rate": 4.377415736638476e-06, "loss": 0.002, "num_tokens": 38052566.0, "reward": -0.0125, "reward_std": 0.19864802658557892, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.19864802658557892, "sampling/importance_sampling_ratio/max": 1.2874309778213502, "sampling/importance_sampling_ratio/mean": 0.9998765051364898, "sampling/importance_sampling_ratio/min": 0.7950535893440247, "sampling/sampling_logp_difference/max": 0.27124361991882323, "sampling/sampling_logp_difference/mean": 0.02178285215049982, "step": 8750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.7, "completions/max_terminated_length": 278.7, "completions/mean_length": 194.75, "completions/mean_terminated_length": 194.75, "completions/min_length": 121.1, "completions/min_terminated_length": 121.1, "entropy": 1.0251334503293037, "epoch": 0.8053691275167785, "frac_reward_zero_std": 0.6, "grad_norm": 5.286441326141357, "kl": 0.04760275386506692, "learning_rate": 4.375741718037873e-06, "loss": 0.002, "num_tokens": 38091882.0, "reward": 0.0, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2806723594665528, "sampling/importance_sampling_ratio/mean": 1.0000867605209351, "sampling/importance_sampling_ratio/min": 0.7584055721759796, "sampling/sampling_logp_difference/max": 0.2932913899421692, "sampling/sampling_logp_difference/mean": 0.02150655761361122, "step": 8760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 303.3, "completions/max_terminated_length": 286.5, "completions/mean_length": 224.5125, "completions/mean_terminated_length": 216.3375, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.1176640853285789, "epoch": 0.8062884986669119, "frac_reward_zero_std": 0.3, "grad_norm": 0.1579965353012085, "kl": 0.0352932273526676, "learning_rate": 4.374065772900165e-06, "loss": 0.0016, "num_tokens": 38138163.0, "reward": 0.19375, "reward_std": 0.48904638886451723, "rewards/reward_fn/mean": 0.19375, "rewards/reward_fn/std": 0.4890464127063751, "sampling/importance_sampling_ratio/max": 1.2696520686149597, "sampling/importance_sampling_ratio/mean": 0.999466073513031, "sampling/importance_sampling_ratio/min": 0.7751040577888488, "sampling/sampling_logp_difference/max": 0.265606153011322, "sampling/sampling_logp_difference/mean": 0.02191924545913935, "step": 8770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.4, "completions/max_terminated_length": 304.4, "completions/mean_length": 190.925, "completions/mean_terminated_length": 190.925, "completions/min_length": 116.5, "completions/min_terminated_length": 116.5, "entropy": 1.0450495783239604, "epoch": 0.8072078698170452, "frac_reward_zero_std": 0.5, "grad_norm": 6.599582195281982, "kl": 0.05343142842175439, "learning_rate": 4.372387902946684e-06, "loss": 0.0025, "num_tokens": 38182573.0, "reward": 0.0375, "reward_std": 0.2667276442050934, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.2667276501655579, "sampling/importance_sampling_ratio/max": 1.2608490586280823, "sampling/importance_sampling_ratio/mean": 1.0005966007709504, "sampling/importance_sampling_ratio/min": 0.7757447600364685, "sampling/sampling_logp_difference/max": 0.2792081117630005, "sampling/sampling_logp_difference/mean": 0.02111231945455074, "step": 8780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.9, "completions/max_terminated_length": 277.9, "completions/mean_length": 200.3375, "completions/mean_terminated_length": 200.3375, "completions/min_length": 129.4, "completions/min_terminated_length": 129.4, "entropy": 1.0650831572711468, "epoch": 0.8081272409671785, "frac_reward_zero_std": 0.3, "grad_norm": 6.140339374542236, "kl": 0.0357040181173943, "learning_rate": 4.370708109900737e-06, "loss": 0.0018, "num_tokens": 38222272.0, "reward": 0.125, "reward_std": 0.4363411724567413, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.4363411784172058, "sampling/importance_sampling_ratio/max": 1.2859436869621277, "sampling/importance_sampling_ratio/mean": 0.999999725818634, "sampling/importance_sampling_ratio/min": 0.7639418482780457, "sampling/sampling_logp_difference/max": 0.288063645362854, "sampling/sampling_logp_difference/mean": 0.021699277497828007, "step": 8790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 259.0, "completions/max_terminated_length": 259.0, "completions/mean_length": 187.625, "completions/mean_terminated_length": 187.625, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 0.9379396364092827, "epoch": 0.8090466121173118, "frac_reward_zero_std": 0.4, "grad_norm": 8.193876266479492, "kl": 0.04675668946583755, "learning_rate": 4.369026395487605e-06, "loss": 0.0021, "num_tokens": 38262714.0, "reward": 0.0625, "reward_std": 0.34817772209644315, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.34817772805690766, "sampling/importance_sampling_ratio/max": 1.3576258540153503, "sampling/importance_sampling_ratio/mean": 1.0003045678138733, "sampling/importance_sampling_ratio/min": 0.707453191280365, "sampling/sampling_logp_difference/max": 0.3967010617256165, "sampling/sampling_logp_difference/mean": 0.02055827286094427, "step": 8800 }, { "epoch": 0.8090466121173118, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 287.2844677137871, "eval_completions/max_terminated_length": 286.1675392670157, "eval_completions/mean_length": 203.38852530541013, "eval_completions/mean_terminated_length": 203.06027192387074, "eval_completions/min_length": 130.64746945898779, "eval_completions/min_terminated_length": 130.64746945898779, "eval_entropy": 1.080786794684112, "eval_frac_reward_zero_std": 0.32460732984293195, "eval_kl": 0.04283483098317945, "eval_loss": 0.002224038355052471, "eval_num_tokens": 38262714.0, "eval_reward": 0.05475567190226876, "eval_reward_std": 0.35207359607203054, "eval_rewards/reward_fn/mean": 0.05475567190226876, "eval_rewards/reward_fn/std": 0.35207360595412696, "eval_runtime": 1684.7806, "eval_samples_per_second": 0.34, "eval_sampling/importance_sampling_ratio/max": 1.2736616600662418, "eval_sampling/importance_sampling_ratio/mean": 1.0000273790242993, "eval_sampling/importance_sampling_ratio/min": 0.7780227620564206, "eval_sampling/sampling_logp_difference/max": 0.2795751323666664, "eval_sampling/sampling_logp_difference/mean": 0.021095979768607317, "eval_steps_per_second": 0.043, "step": 8800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 211.525, "completions/mean_terminated_length": 211.525, "completions/min_length": 136.1, "completions/min_terminated_length": 136.1, "entropy": 1.0855183489620686, "epoch": 0.8099659832674451, "frac_reward_zero_std": 0.3, "grad_norm": 0.21091626584529877, "kl": 0.03973880144767463, "learning_rate": 4.367342761434544e-06, "loss": 0.0019, "num_tokens": 38307772.0, "reward": 0.1875, "reward_std": 0.3818357288837433, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.38183574080467225, "sampling/importance_sampling_ratio/max": 1.2601247310638428, "sampling/importance_sampling_ratio/mean": 1.0001230716705323, "sampling/importance_sampling_ratio/min": 0.7872125804424286, "sampling/sampling_logp_difference/max": 0.26520882844924926, "sampling/sampling_logp_difference/mean": 0.021789366938173772, "step": 8810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 210.975, "completions/mean_terminated_length": 210.975, "completions/min_length": 131.2, "completions/min_terminated_length": 131.2, "entropy": 1.0969344601035118, "epoch": 0.8108853544175784, "frac_reward_zero_std": 0.3, "grad_norm": 5.790411949157715, "kl": 0.04803976528928615, "learning_rate": 4.365657209470781e-06, "loss": 0.0024, "num_tokens": 38351874.0, "reward": -0.0125, "reward_std": 0.4075396120548248, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.4075396180152893, "sampling/importance_sampling_ratio/max": 1.2696874022483826, "sampling/importance_sampling_ratio/mean": 1.0002799391746522, "sampling/importance_sampling_ratio/min": 0.7690694212913514, "sampling/sampling_logp_difference/max": 0.28398908376693727, "sampling/sampling_logp_difference/mean": 0.02169826366007328, "step": 8820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 292.0, "completions/max_terminated_length": 282.1, "completions/mean_length": 206.2125, "completions/mean_terminated_length": 202.51607208251954, "completions/min_length": 136.5, "completions/min_terminated_length": 136.5, "entropy": 1.0781104639172554, "epoch": 0.8118047255677117, "frac_reward_zero_std": 0.4, "grad_norm": 0.09663721174001694, "kl": 0.028700974106322975, "learning_rate": 4.363969741327514e-06, "loss": 0.0018, "num_tokens": 38391947.0, "reward": 0.44375, "reward_std": 0.31051755249500274, "rewards/reward_fn/mean": 0.44375, "rewards/reward_fn/std": 0.31051755249500274, "sampling/importance_sampling_ratio/max": 1.2981590628623962, "sampling/importance_sampling_ratio/mean": 1.000667291879654, "sampling/importance_sampling_ratio/min": 0.7828846096992492, "sampling/sampling_logp_difference/max": 0.2753793835639954, "sampling/sampling_logp_difference/mean": 0.021466749347746374, "step": 8830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.3, "completions/max_terminated_length": 272.3, "completions/mean_length": 190.85, "completions/mean_terminated_length": 190.85, "completions/min_length": 124.4, "completions/min_terminated_length": 124.4, "entropy": 1.1052660129964351, "epoch": 0.812724096717845, "frac_reward_zero_std": 0.2, "grad_norm": 5.288593292236328, "kl": 0.03749731448478997, "learning_rate": 4.362280358737907e-06, "loss": 0.0015, "num_tokens": 38431855.0, "reward": 0.0, "reward_std": 0.43029355704784394, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4302935689687729, "sampling/importance_sampling_ratio/max": 1.254835045337677, "sampling/importance_sampling_ratio/mean": 1.0001147627830504, "sampling/importance_sampling_ratio/min": 0.7889634609222412, "sampling/sampling_logp_difference/max": 0.26200541853904724, "sampling/sampling_logp_difference/mean": 0.02180521059781313, "step": 8840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.1, "completions/max_terminated_length": 278.1, "completions/mean_length": 196.85, "completions/mean_terminated_length": 196.85, "completions/min_length": 133.1, "completions/min_terminated_length": 133.1, "entropy": 1.0610683955252171, "epoch": 0.8136434678679783, "frac_reward_zero_std": 0.5, "grad_norm": 0.09803583472967148, "kl": 0.03152940608561039, "learning_rate": 4.360589063437091e-06, "loss": 0.0016, "num_tokens": 38472763.0, "reward": 0.325, "reward_std": 0.2739637613296509, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.27396377325057986, "sampling/importance_sampling_ratio/max": 1.2634969353675842, "sampling/importance_sampling_ratio/mean": 1.000402319431305, "sampling/importance_sampling_ratio/min": 0.7855935394763947, "sampling/sampling_logp_difference/max": 0.2712444543838501, "sampling/sampling_logp_difference/mean": 0.020579307712614535, "step": 8850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.3, "completions/max_terminated_length": 280.3, "completions/mean_length": 209.775, "completions/mean_terminated_length": 209.775, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.0386340081691743, "epoch": 0.8145628390181117, "frac_reward_zero_std": 0.4, "grad_norm": 5.450444221496582, "kl": 0.038624640996567906, "learning_rate": 4.358895857162163e-06, "loss": 0.0018, "num_tokens": 38515385.0, "reward": 0.41875, "reward_std": 0.3511351436376572, "rewards/reward_fn/mean": 0.41875, "rewards/reward_fn/std": 0.3511351615190506, "sampling/importance_sampling_ratio/max": 1.2828514575958252, "sampling/importance_sampling_ratio/mean": 1.0003724694252014, "sampling/importance_sampling_ratio/min": 0.7626035332679748, "sampling/sampling_logp_difference/max": 0.28701198697090147, "sampling/sampling_logp_difference/mean": 0.021290628984570504, "step": 8860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 315.9, "completions/max_terminated_length": 304.2, "completions/mean_length": 224.3375, "completions/mean_terminated_length": 221.375, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.097715050727129, "epoch": 0.815482210168245, "frac_reward_zero_std": 0.3, "grad_norm": 0.12754617631435394, "kl": 0.041775132965995, "learning_rate": 4.35720074165218e-06, "loss": 0.0022, "num_tokens": 38557692.0, "reward": -0.14375, "reward_std": 0.42964828312397, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.42964828312397, "sampling/importance_sampling_ratio/max": 1.2867362260818482, "sampling/importance_sampling_ratio/mean": 0.9999699056148529, "sampling/importance_sampling_ratio/min": 0.786135321855545, "sampling/sampling_logp_difference/max": 0.28501527309417723, "sampling/sampling_logp_difference/mean": 0.022443701326847077, "step": 8870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.6, "completions/max_terminated_length": 273.6, "completions/mean_length": 202.35, "completions/mean_terminated_length": 202.35, "completions/min_length": 144.3, "completions/min_terminated_length": 144.3, "entropy": 0.9785729929804802, "epoch": 0.8164015813183783, "frac_reward_zero_std": 0.5, "grad_norm": 0.09973826259374619, "kl": 0.036937700293492526, "learning_rate": 4.355503718648163e-06, "loss": 0.0018, "num_tokens": 38599312.0, "reward": -0.0875, "reward_std": 0.23484184443950654, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.23484184443950654, "sampling/importance_sampling_ratio/max": 1.3120640397071839, "sampling/importance_sampling_ratio/mean": 1.0001800775527954, "sampling/importance_sampling_ratio/min": 0.7991429746150971, "sampling/sampling_logp_difference/max": 0.28285343647003175, "sampling/sampling_logp_difference/mean": 0.020083884708583355, "step": 8880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.6, "completions/max_terminated_length": 293.6, "completions/mean_length": 209.275, "completions/mean_terminated_length": 209.275, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 1.0875718615949155, "epoch": 0.8173209524685116, "frac_reward_zero_std": 0.3, "grad_norm": 5.550856113433838, "kl": 0.037851628882344814, "learning_rate": 4.35380478989309e-06, "loss": 0.002, "num_tokens": 38643430.0, "reward": 0.125, "reward_std": 0.3886036962270737, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3886037141084671, "sampling/importance_sampling_ratio/max": 1.3095089793205261, "sampling/importance_sampling_ratio/mean": 1.0009268760681151, "sampling/importance_sampling_ratio/min": 0.7880194306373596, "sampling/sampling_logp_difference/max": 0.2841110944747925, "sampling/sampling_logp_difference/mean": 0.021922591514885424, "step": 8890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 345.5, "completions/max_terminated_length": 323.3, "completions/mean_length": 234.325, "completions/mean_terminated_length": 230.87857208251953, "completions/min_length": 159.2, "completions/min_terminated_length": 159.2, "entropy": 1.0169524781405925, "epoch": 0.8182403236186448, "frac_reward_zero_std": 0.1, "grad_norm": 5.433189868927002, "kl": 0.04369154053274542, "learning_rate": 4.352103957131898e-06, "loss": 0.0024, "num_tokens": 38689392.0, "reward": -0.18125, "reward_std": 0.47442744076251986, "rewards/reward_fn/mean": -0.18125, "rewards/reward_fn/std": 0.4744274467229843, "sampling/importance_sampling_ratio/max": 1.2639684796333313, "sampling/importance_sampling_ratio/mean": 1.000538033246994, "sampling/importance_sampling_ratio/min": 0.7510909378528595, "sampling/sampling_logp_difference/max": 0.2981226921081543, "sampling/sampling_logp_difference/mean": 0.02065893579274416, "step": 8900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.8, "completions/max_terminated_length": 314.8, "completions/mean_length": 212.9, "completions/mean_terminated_length": 212.9, "completions/min_length": 140.9, "completions/min_terminated_length": 140.9, "entropy": 1.0175654284656048, "epoch": 0.8191596947687781, "frac_reward_zero_std": 0.3, "grad_norm": 0.11694075912237167, "kl": 0.04509069290943444, "learning_rate": 4.350401222111477e-06, "loss": 0.0021, "num_tokens": 38733976.0, "reward": 0.1875, "reward_std": 0.3587038516998291, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3587038516998291, "sampling/importance_sampling_ratio/max": 1.2507014632225038, "sampling/importance_sampling_ratio/mean": 1.0001662015914916, "sampling/importance_sampling_ratio/min": 0.7634660482406617, "sampling/sampling_logp_difference/max": 0.2968484401702881, "sampling/sampling_logp_difference/mean": 0.020928441546857357, "step": 8910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 365.1, "completions/max_terminated_length": 351.3, "completions/mean_length": 258.7625, "completions/mean_terminated_length": 250.1482147216797, "completions/min_length": 163.1, "completions/min_terminated_length": 163.1, "entropy": 1.1511157929897309, "epoch": 0.8200790659189114, "frac_reward_zero_std": 0.1, "grad_norm": 0.27891287207603455, "kl": 0.0469004514219705, "learning_rate": 4.348696586580673e-06, "loss": 0.0026, "num_tokens": 38783205.0, "reward": -0.05625, "reward_std": 0.4334175020456314, "rewards/reward_fn/mean": -0.05625, "rewards/reward_fn/std": 0.43341751396656036, "sampling/importance_sampling_ratio/max": 1.2654539585113525, "sampling/importance_sampling_ratio/mean": 0.9997540295124054, "sampling/importance_sampling_ratio/min": 0.7700287818908691, "sampling/sampling_logp_difference/max": 0.2753345608711243, "sampling/sampling_logp_difference/mean": 0.021777121350169182, "step": 8920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.1, "completions/max_terminated_length": 289.1, "completions/mean_length": 208.6125, "completions/mean_terminated_length": 208.6125, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.0555961526930333, "epoch": 0.8209984370690447, "frac_reward_zero_std": 0.3, "grad_norm": 4.751936912536621, "kl": 0.04429230668465607, "learning_rate": 4.346990052290283e-06, "loss": 0.0025, "num_tokens": 38824446.0, "reward": 0.1875, "reward_std": 0.269358703494072, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.269358703494072, "sampling/importance_sampling_ratio/max": 1.2969286561012268, "sampling/importance_sampling_ratio/mean": 0.9997332870960236, "sampling/importance_sampling_ratio/min": 0.7328262656927109, "sampling/sampling_logp_difference/max": 0.3552848696708679, "sampling/sampling_logp_difference/mean": 0.021560726501047612, "step": 8930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.7, "completions/max_terminated_length": 274.7, "completions/mean_length": 208.025, "completions/mean_terminated_length": 208.025, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.078001006692648, "epoch": 0.821917808219178, "frac_reward_zero_std": 0.2, "grad_norm": 7.077641010284424, "kl": 0.04658143932465464, "learning_rate": 4.345281620993056e-06, "loss": 0.0025, "num_tokens": 38867728.0, "reward": -0.1125, "reward_std": 0.4007413417100906, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.40074135959148405, "sampling/importance_sampling_ratio/max": 1.2712863445281983, "sampling/importance_sampling_ratio/mean": 0.9997042357921601, "sampling/importance_sampling_ratio/min": 0.7841499507427215, "sampling/sampling_logp_difference/max": 0.2648874998092651, "sampling/sampling_logp_difference/mean": 0.021290831640362738, "step": 8940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.9, "completions/max_terminated_length": 291.9, "completions/mean_length": 215.6875, "completions/mean_terminated_length": 215.6875, "completions/min_length": 155.6, "completions/min_terminated_length": 155.6, "entropy": 1.0468795076012611, "epoch": 0.8228371793693114, "frac_reward_zero_std": 0.4, "grad_norm": 0.12104776501655579, "kl": 0.04860833355924114, "learning_rate": 4.343571294443686e-06, "loss": 0.0019, "num_tokens": 38910031.0, "reward": 0.0375, "reward_std": 0.3472063720226288, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.3472063958644867, "sampling/importance_sampling_ratio/max": 1.3149312734603882, "sampling/importance_sampling_ratio/mean": 1.000365149974823, "sampling/importance_sampling_ratio/min": 0.7600172936916352, "sampling/sampling_logp_difference/max": 0.30601245164871216, "sampling/sampling_logp_difference/mean": 0.02072648238390684, "step": 8950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.8, "completions/max_terminated_length": 272.8, "completions/mean_length": 198.4125, "completions/mean_terminated_length": 198.4125, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.0973112799227238, "epoch": 0.8237565505194447, "frac_reward_zero_std": 0.6, "grad_norm": 0.14992687106132507, "kl": 0.05303641356294975, "learning_rate": 4.3418590743988164e-06, "loss": 0.0026, "num_tokens": 38952352.0, "reward": -0.1375, "reward_std": 0.18515617549419403, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.185156187415123, "sampling/importance_sampling_ratio/max": 1.2804394125938416, "sampling/importance_sampling_ratio/mean": 1.000007700920105, "sampling/importance_sampling_ratio/min": 0.7859462320804596, "sampling/sampling_logp_difference/max": 0.27508476972579954, "sampling/sampling_logp_difference/mean": 0.021746984496712685, "step": 8960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 330.4, "completions/max_terminated_length": 317.9, "completions/mean_length": 234.8625, "completions/mean_terminated_length": 232.4214294433594, "completions/min_length": 159.2, "completions/min_terminated_length": 159.2, "entropy": 1.162705421447754, "epoch": 0.824675921669578, "frac_reward_zero_std": 0.3, "grad_norm": 0.12086024880409241, "kl": 0.040746063087135555, "learning_rate": 4.340144962617034e-06, "loss": 0.0019, "num_tokens": 39003229.0, "reward": -0.0875, "reward_std": 0.32662735879421234, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.3266273647546768, "sampling/importance_sampling_ratio/max": 1.2446288347244263, "sampling/importance_sampling_ratio/mean": 0.9993978559970855, "sampling/importance_sampling_ratio/min": 0.788626766204834, "sampling/sampling_logp_difference/max": 0.2591118335723877, "sampling/sampling_logp_difference/mean": 0.021541816368699073, "step": 8970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.7, "completions/max_terminated_length": 276.7, "completions/mean_length": 207.0875, "completions/mean_terminated_length": 207.0875, "completions/min_length": 135.8, "completions/min_terminated_length": 135.8, "entropy": 1.1071314468979836, "epoch": 0.8255952928197113, "frac_reward_zero_std": 0.3, "grad_norm": 7.019286155700684, "kl": 0.05420791409560479, "learning_rate": 4.33842896085887e-06, "loss": 0.0023, "num_tokens": 39047012.0, "reward": -0.125, "reward_std": 0.36287996768951414, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.3628799855709076, "sampling/importance_sampling_ratio/max": 1.2749491333961487, "sampling/importance_sampling_ratio/mean": 1.000269103050232, "sampling/importance_sampling_ratio/min": 0.7728374063968658, "sampling/sampling_logp_difference/max": 0.278046977519989, "sampling/sampling_logp_difference/mean": 0.021776656992733477, "step": 8980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.0, "completions/max_terminated_length": 333.0, "completions/mean_length": 232.55, "completions/mean_terminated_length": 232.55, "completions/min_length": 147.1, "completions/min_terminated_length": 147.1, "entropy": 1.0654337860643863, "epoch": 0.8265146639698446, "frac_reward_zero_std": 0.6, "grad_norm": 6.427915096282959, "kl": 0.03480404880247079, "learning_rate": 4.336711070886795e-06, "loss": 0.0019, "num_tokens": 39095592.0, "reward": 0.1625, "reward_std": 0.18515617549419403, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.185156187415123, "sampling/importance_sampling_ratio/max": 1.2899380803108216, "sampling/importance_sampling_ratio/mean": 1.0001848757266998, "sampling/importance_sampling_ratio/min": 0.7713256061077118, "sampling/sampling_logp_difference/max": 0.27633455991744993, "sampling/sampling_logp_difference/mean": 0.021645659022033213, "step": 8990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.2, "completions/max_terminated_length": 276.2, "completions/mean_length": 209.6375, "completions/mean_terminated_length": 209.6375, "completions/min_length": 142.6, "completions/min_terminated_length": 142.6, "entropy": 1.0761123105883599, "epoch": 0.8274340351199779, "frac_reward_zero_std": 0.2, "grad_norm": 5.605615139007568, "kl": 0.06172620209399611, "learning_rate": 4.33499129446522e-06, "loss": 0.0032, "num_tokens": 39140587.0, "reward": 0.11875, "reward_std": 0.36787369549274446, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.36787369549274446, "sampling/importance_sampling_ratio/max": 1.2738475918769836, "sampling/importance_sampling_ratio/mean": 1.0001766085624695, "sampling/importance_sampling_ratio/min": 0.7630397915840149, "sampling/sampling_logp_difference/max": 0.2871168553829193, "sampling/sampling_logp_difference/mean": 0.02122314050793648, "step": 9000 }, { "epoch": 0.8274340351199779, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 286.66841186736474, "eval_completions/max_terminated_length": 285.5410122164049, "eval_completions/mean_length": 204.09707678883072, "eval_completions/mean_terminated_length": 203.72582698908568, "eval_completions/min_length": 135.1849912739965, "eval_completions/min_terminated_length": 135.1849912739965, "eval_entropy": 1.0882200900900842, "eval_frac_reward_zero_std": 0.38917975567190227, "eval_kl": 0.0510732689726103, "eval_loss": 0.0026241466403007507, "eval_num_tokens": 39140587.0, "eval_reward": 0.07340750436300175, "eval_reward_std": 0.2887389008443393, "eval_rewards/reward_fn/mean": 0.07340750436300175, "eval_rewards/reward_fn/std": 0.28873890947817094, "eval_runtime": 1681.4939, "eval_samples_per_second": 0.341, "eval_sampling/importance_sampling_ratio/max": 1.267262753391765, "eval_sampling/importance_sampling_ratio/mean": 1.0000146895803081, "eval_sampling/importance_sampling_ratio/min": 0.7786260428749038, "eval_sampling/sampling_logp_difference/max": 0.2788939066166237, "eval_sampling/sampling_logp_difference/mean": 0.020763859975218357, "eval_steps_per_second": 0.043, "step": 9000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.7, "completions/max_terminated_length": 315.7, "completions/mean_length": 211.225, "completions/mean_terminated_length": 211.225, "completions/min_length": 120.5, "completions/min_terminated_length": 120.5, "entropy": 1.1148918487131596, "epoch": 0.8283534062701112, "frac_reward_zero_std": 0.3, "grad_norm": 5.0368123054504395, "kl": 0.057120157731696965, "learning_rate": 4.333269633360493e-06, "loss": 0.0027, "num_tokens": 39184709.0, "reward": -0.1125, "reward_std": 0.31448993980884554, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.31448994576931, "sampling/importance_sampling_ratio/max": 1.3122798442840575, "sampling/importance_sampling_ratio/mean": 1.0004422128200532, "sampling/importance_sampling_ratio/min": 0.7739920496940613, "sampling/sampling_logp_difference/max": 0.30198254585266116, "sampling/sampling_logp_difference/mean": 0.021951787732541563, "step": 9010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.1, "completions/max_terminated_length": 278.1, "completions/mean_length": 204.85, "completions/mean_terminated_length": 204.85, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.0834312930703163, "epoch": 0.8292727774202445, "frac_reward_zero_std": 0.4, "grad_norm": 8.699468612670898, "kl": 0.06076372329844162, "learning_rate": 4.3315460893408975e-06, "loss": 0.0028, "num_tokens": 39228633.0, "reward": 0.1875, "reward_std": 0.31131347119808195, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.3113134950399399, "sampling/importance_sampling_ratio/max": 1.2907068371772765, "sampling/importance_sampling_ratio/mean": 1.000281608104706, "sampling/importance_sampling_ratio/min": 0.7726169645786285, "sampling/sampling_logp_difference/max": 0.27844642400741576, "sampling/sampling_logp_difference/mean": 0.021578053943812847, "step": 9020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 321.3, "completions/max_terminated_length": 321.3, "completions/mean_length": 219.55, "completions/mean_terminated_length": 219.55, "completions/min_length": 140.4, "completions/min_terminated_length": 140.4, "entropy": 1.1024599201977252, "epoch": 0.8301921485703778, "frac_reward_zero_std": 0.4, "grad_norm": 0.15127870440483093, "kl": 0.046587085886858406, "learning_rate": 4.329820664176651e-06, "loss": 0.0023, "num_tokens": 39273653.0, "reward": 0.025, "reward_std": 0.33288776576519014, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3328877717256546, "sampling/importance_sampling_ratio/max": 1.2989366888999938, "sampling/importance_sampling_ratio/mean": 0.9997116506099701, "sampling/importance_sampling_ratio/min": 0.7541512727737427, "sampling/sampling_logp_difference/max": 0.30350762605667114, "sampling/sampling_logp_difference/mean": 0.02134474255144596, "step": 9030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.2, "completions/max_terminated_length": 325.2, "completions/mean_length": 232.3125, "completions/mean_terminated_length": 232.3125, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.0963632196187973, "epoch": 0.8311115197205111, "frac_reward_zero_std": 0.5, "grad_norm": 0.10404127836227417, "kl": 0.05548367394949309, "learning_rate": 4.3280933596399066e-06, "loss": 0.0024, "num_tokens": 39324646.0, "reward": 0.225, "reward_std": 0.20958369374275207, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.20958369374275207, "sampling/importance_sampling_ratio/max": 1.2953606605529786, "sampling/importance_sampling_ratio/mean": 1.0003344476222993, "sampling/importance_sampling_ratio/min": 0.774643874168396, "sampling/sampling_logp_difference/max": 0.2935644745826721, "sampling/sampling_logp_difference/mean": 0.02066124528646469, "step": 9040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.7, "completions/max_terminated_length": 339.7, "completions/mean_length": 224.3125, "completions/mean_terminated_length": 224.3125, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.0464645341038703, "epoch": 0.8320308908706445, "frac_reward_zero_std": 0.4, "grad_norm": 0.13783994317054749, "kl": 0.0407357525837142, "learning_rate": 4.326364177504741e-06, "loss": 0.0028, "num_tokens": 39371071.0, "reward": 0.2, "reward_std": 0.33679819107055664, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3367981970310211, "sampling/importance_sampling_ratio/max": 1.254710328578949, "sampling/importance_sampling_ratio/mean": 1.0000968396663665, "sampling/importance_sampling_ratio/min": 0.7936439573764801, "sampling/sampling_logp_difference/max": 0.24865182638168334, "sampling/sampling_logp_difference/mean": 0.020618128962814807, "step": 9050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 205.1, "completions/mean_terminated_length": 205.1, "completions/min_length": 131.1, "completions/min_terminated_length": 131.1, "entropy": 1.1125460632145405, "epoch": 0.8329502620207778, "frac_reward_zero_std": 0.7, "grad_norm": 0.1732306331396103, "kl": 0.048045880493009466, "learning_rate": 4.324633119547163e-06, "loss": 0.0021, "num_tokens": 39413471.0, "reward": 0.0625, "reward_std": 0.13479767441749574, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.13479767441749574, "sampling/importance_sampling_ratio/max": 1.2365493774414062, "sampling/importance_sampling_ratio/mean": 1.0004522264003755, "sampling/importance_sampling_ratio/min": 0.7927963078022003, "sampling/sampling_logp_difference/max": 0.24077730178833007, "sampling/sampling_logp_difference/mean": 0.02182335276156664, "step": 9060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.4, "completions/max_terminated_length": 246.4, "completions/mean_length": 177.0625, "completions/mean_terminated_length": 177.0625, "completions/min_length": 112.4, "completions/min_terminated_length": 112.4, "entropy": 1.0748204439878464, "epoch": 0.8338696331709111, "frac_reward_zero_std": 0.3, "grad_norm": 6.4584197998046875, "kl": 0.038400100072612985, "learning_rate": 4.32290018754511e-06, "loss": 0.0017, "num_tokens": 39454700.0, "reward": 0.2625, "reward_std": 0.3502548158168793, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.35025482773780825, "sampling/importance_sampling_ratio/max": 1.3449970126152038, "sampling/importance_sampling_ratio/mean": 1.0013143837451934, "sampling/importance_sampling_ratio/min": 0.7819990158081055, "sampling/sampling_logp_difference/max": 0.32233076095581054, "sampling/sampling_logp_difference/mean": 0.021630468592047692, "step": 9070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 275.7, "completions/max_terminated_length": 275.7, "completions/mean_length": 186.75, "completions/mean_terminated_length": 186.75, "completions/min_length": 112.4, "completions/min_terminated_length": 112.4, "entropy": 1.0550425782799722, "epoch": 0.8347890043210444, "frac_reward_zero_std": 0.4, "grad_norm": 8.490652084350586, "kl": 0.030567353643709792, "learning_rate": 4.3211653832784415e-06, "loss": 0.0014, "num_tokens": 39491600.0, "reward": 0.225, "reward_std": 0.28198385536670684, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.28198386132717135, "sampling/importance_sampling_ratio/max": 1.2257599234580994, "sampling/importance_sampling_ratio/mean": 1.0004043102264404, "sampling/importance_sampling_ratio/min": 0.7733115494251251, "sampling/sampling_logp_difference/max": 0.2702323645353317, "sampling/sampling_logp_difference/mean": 0.02107505053281784, "step": 9080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.2, "completions/max_terminated_length": 332.2, "completions/mean_length": 223.5125, "completions/mean_terminated_length": 223.5125, "completions/min_length": 158.8, "completions/min_terminated_length": 158.8, "entropy": 1.1369658015668391, "epoch": 0.8357083754711777, "frac_reward_zero_std": 0.6, "grad_norm": 0.20092476904392242, "kl": 0.06279710702365264, "learning_rate": 4.31942870852894e-06, "loss": 0.003, "num_tokens": 39537977.0, "reward": -0.05, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.324883759021759, "sampling/importance_sampling_ratio/mean": 1.000228124856949, "sampling/importance_sampling_ratio/min": 0.7855935573577881, "sampling/sampling_logp_difference/max": 0.29696204662323, "sampling/sampling_logp_difference/mean": 0.02183693237602711, "step": 9090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.0, "completions/max_terminated_length": 307.0, "completions/mean_length": 219.525, "completions/mean_terminated_length": 219.525, "completions/min_length": 145.7, "completions/min_terminated_length": 145.7, "entropy": 1.092097295075655, "epoch": 0.836627746621311, "frac_reward_zero_std": 0.3, "grad_norm": 0.13455146551132202, "kl": 0.05791983523522504, "learning_rate": 4.31769016508031e-06, "loss": 0.0027, "num_tokens": 39583139.0, "reward": 0.04375, "reward_std": 0.328694099187851, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.32869410514831543, "sampling/importance_sampling_ratio/max": 1.2635885000228881, "sampling/importance_sampling_ratio/mean": 0.9994966864585877, "sampling/importance_sampling_ratio/min": 0.793554937839508, "sampling/sampling_logp_difference/max": 0.2487446904182434, "sampling/sampling_logp_difference/mean": 0.021288000233471393, "step": 9100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.6, "completions/max_terminated_length": 302.6, "completions/mean_length": 203.5625, "completions/mean_terminated_length": 203.5625, "completions/min_length": 117.1, "completions/min_terminated_length": 117.1, "entropy": 0.9976728424429894, "epoch": 0.8375471177714443, "frac_reward_zero_std": 0.4, "grad_norm": 5.753015995025635, "kl": 0.041356326849199834, "learning_rate": 4.315949754718174e-06, "loss": 0.0022, "num_tokens": 39625208.0, "reward": 0.0375, "reward_std": 0.2558668524026871, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.25586686432361605, "sampling/importance_sampling_ratio/max": 1.2560867667198181, "sampling/importance_sampling_ratio/mean": 0.9998450815677643, "sampling/importance_sampling_ratio/min": 0.8101845502853393, "sampling/sampling_logp_difference/max": 0.238252592086792, "sampling/sampling_logp_difference/mean": 0.020680646784603594, "step": 9110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.4, "completions/max_terminated_length": 254.4, "completions/mean_length": 190.7, "completions/mean_terminated_length": 190.7, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 0.9843381345272064, "epoch": 0.8384664889215776, "frac_reward_zero_std": 0.4, "grad_norm": 6.550142765045166, "kl": 0.054185232234885916, "learning_rate": 4.3142074792300735e-06, "loss": 0.0022, "num_tokens": 39665472.0, "reward": 0.1, "reward_std": 0.28629584312438966, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.2862958490848541, "sampling/importance_sampling_ratio/max": 1.3019490242004395, "sampling/importance_sampling_ratio/mean": 0.9998579800128937, "sampling/importance_sampling_ratio/min": 0.7970300376415252, "sampling/sampling_logp_difference/max": 0.2818925380706787, "sampling/sampling_logp_difference/mean": 0.020097876340150832, "step": 9120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 317.2, "completions/max_terminated_length": 317.2, "completions/mean_length": 223.1875, "completions/mean_terminated_length": 223.1875, "completions/min_length": 137.5, "completions/min_terminated_length": 137.5, "entropy": 1.09163998067379, "epoch": 0.8393858600717109, "frac_reward_zero_std": 0.4, "grad_norm": 0.1480615884065628, "kl": 0.04418914269772358, "learning_rate": 4.312463340405466e-06, "loss": 0.0022, "num_tokens": 39709687.0, "reward": 0.125, "reward_std": 0.32178503572940825, "rewards/reward_fn/mean": 0.125, "rewards/reward_fn/std": 0.3217850476503372, "sampling/importance_sampling_ratio/max": 1.2690747380256653, "sampling/importance_sampling_ratio/mean": 0.9998252153396606, "sampling/importance_sampling_ratio/min": 0.7846175730228424, "sampling/sampling_logp_difference/max": 0.25896793603897095, "sampling/sampling_logp_difference/mean": 0.021135289035737515, "step": 9130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.1, "completions/max_terminated_length": 285.1, "completions/mean_length": 203.45, "completions/mean_terminated_length": 203.45, "completions/min_length": 137.9, "completions/min_terminated_length": 137.9, "entropy": 1.077194471657276, "epoch": 0.8403052312218443, "frac_reward_zero_std": 0.4, "grad_norm": 6.326993465423584, "kl": 0.046213487058412286, "learning_rate": 4.310717340035721e-06, "loss": 0.0024, "num_tokens": 39751795.0, "reward": -0.175, "reward_std": 0.2411646068096161, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.2411646068096161, "sampling/importance_sampling_ratio/max": 1.2543643355369567, "sampling/importance_sampling_ratio/mean": 0.9994263887405396, "sampling/importance_sampling_ratio/min": 0.7826475501060486, "sampling/sampling_logp_difference/max": 0.25688384771347045, "sampling/sampling_logp_difference/mean": 0.02109307814389467, "step": 9140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 281.8, "completions/max_terminated_length": 281.8, "completions/mean_length": 200.55, "completions/mean_terminated_length": 200.55, "completions/min_length": 133.2, "completions/min_terminated_length": 133.2, "entropy": 1.1286091461777688, "epoch": 0.8412246023719776, "frac_reward_zero_std": 0.4, "grad_norm": 4.547336578369141, "kl": 0.05496117583825253, "learning_rate": 4.308969479914121e-06, "loss": 0.0027, "num_tokens": 39795559.0, "reward": -0.125, "reward_std": 0.2411646068096161, "rewards/reward_fn/mean": -0.125, "rewards/reward_fn/std": 0.2411646068096161, "sampling/importance_sampling_ratio/max": 1.294877290725708, "sampling/importance_sampling_ratio/mean": 0.999787712097168, "sampling/importance_sampling_ratio/min": 0.7885303974151612, "sampling/sampling_logp_difference/max": 0.2806500673294067, "sampling/sampling_logp_difference/mean": 0.022265691496431827, "step": 9150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.9, "completions/max_terminated_length": 312.9, "completions/mean_length": 217.475, "completions/mean_terminated_length": 217.475, "completions/min_length": 129.3, "completions/min_terminated_length": 129.3, "entropy": 1.0664697647094727, "epoch": 0.8421439735221109, "frac_reward_zero_std": 0.4, "grad_norm": 0.11810187995433807, "kl": 0.050366181501885876, "learning_rate": 4.307219761835859e-06, "loss": 0.0026, "num_tokens": 39843333.0, "reward": 0.325, "reward_std": 0.27019718289375305, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.27019718289375305, "sampling/importance_sampling_ratio/max": 1.3031633853912354, "sampling/importance_sampling_ratio/mean": 0.9995134472846985, "sampling/importance_sampling_ratio/min": 0.7720752239227295, "sampling/sampling_logp_difference/max": 0.2952617645263672, "sampling/sampling_logp_difference/mean": 0.021450130641460417, "step": 9160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 340.2, "completions/max_terminated_length": 340.2, "completions/mean_length": 222.3125, "completions/mean_terminated_length": 222.3125, "completions/min_length": 142.7, "completions/min_terminated_length": 142.7, "entropy": 1.0670593559741974, "epoch": 0.8430633446722442, "frac_reward_zero_std": 0.1, "grad_norm": 5.911360263824463, "kl": 0.055246830178657544, "learning_rate": 4.3054681875980345e-06, "loss": 0.0024, "num_tokens": 39888246.0, "reward": 0.2125, "reward_std": 0.41940403282642363, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.41940403282642363, "sampling/importance_sampling_ratio/max": 1.2755841493606568, "sampling/importance_sampling_ratio/mean": 0.9996138870716095, "sampling/importance_sampling_ratio/min": 0.7634301841259002, "sampling/sampling_logp_difference/max": 0.2908616304397583, "sampling/sampling_logp_difference/mean": 0.021037018671631814, "step": 9170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.2, "completions/max_terminated_length": 277.2, "completions/mean_length": 197.7, "completions/mean_terminated_length": 197.7, "completions/min_length": 132.9, "completions/min_terminated_length": 132.9, "entropy": 1.0390971057116984, "epoch": 0.8439827158223775, "frac_reward_zero_std": 0.5, "grad_norm": 0.16747625172138214, "kl": 0.038900166226085275, "learning_rate": 4.303714758999655e-06, "loss": 0.0016, "num_tokens": 39930822.0, "reward": 0.4, "reward_std": 0.25670533180236815, "rewards/reward_fn/mean": 0.4, "rewards/reward_fn/std": 0.2567053437232971, "sampling/importance_sampling_ratio/max": 1.2425483822822572, "sampling/importance_sampling_ratio/mean": 1.0001917123794555, "sampling/importance_sampling_ratio/min": 0.7860532343387604, "sampling/sampling_logp_difference/max": 0.24847247898578645, "sampling/sampling_logp_difference/mean": 0.02065215855836868, "step": 9180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.5, "completions/max_terminated_length": 261.5, "completions/mean_length": 183.425, "completions/mean_terminated_length": 183.425, "completions/min_length": 111.9, "completions/min_terminated_length": 111.9, "entropy": 1.0045614168047905, "epoch": 0.8449020869725108, "frac_reward_zero_std": 0.4, "grad_norm": 0.19063769280910492, "kl": 0.039167750254273416, "learning_rate": 4.301959477841634e-06, "loss": 0.002, "num_tokens": 39971376.0, "reward": 0.5125, "reward_std": 0.2722664296627045, "rewards/reward_fn/mean": 0.5125, "rewards/reward_fn/std": 0.2722664475440979, "sampling/importance_sampling_ratio/max": 1.2711398124694824, "sampling/importance_sampling_ratio/mean": 0.9999018847942353, "sampling/importance_sampling_ratio/min": 0.7953747391700745, "sampling/sampling_logp_difference/max": 0.25212845802307127, "sampling/sampling_logp_difference/mean": 0.02159304954111576, "step": 9190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.1, "completions/max_terminated_length": 285.1, "completions/mean_length": 196.2375, "completions/mean_terminated_length": 196.2375, "completions/min_length": 115.0, "completions/min_terminated_length": 115.0, "entropy": 1.0011788252741098, "epoch": 0.8458214581226441, "frac_reward_zero_std": 0.4, "grad_norm": 0.14685741066932678, "kl": 0.04386705849319696, "learning_rate": 4.300202345926783e-06, "loss": 0.0021, "num_tokens": 40014771.0, "reward": 0.3, "reward_std": 0.2903633385896683, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.2903633564710617, "sampling/importance_sampling_ratio/max": 1.2652434349060058, "sampling/importance_sampling_ratio/mean": 0.9998990893363953, "sampling/importance_sampling_ratio/min": 0.7806730091571807, "sampling/sampling_logp_difference/max": 0.2740874409675598, "sampling/sampling_logp_difference/mean": 0.020218341425061227, "step": 9200 }, { "epoch": 0.8458214581226441, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0017452006980802793, "eval_completions/max_length": 280.3438045375218, "eval_completions/max_terminated_length": 277.4397905759162, "eval_completions/mean_length": 199.8215532286213, "eval_completions/mean_terminated_length": 199.32784853098073, "eval_completions/min_length": 128.62303664921467, "eval_completions/min_terminated_length": 128.62303664921467, "eval_entropy": 1.0598399707264092, "eval_frac_reward_zero_std": 0.3403141361256545, "eval_kl": 0.04085794858793756, "eval_loss": 0.002103661885485053, "eval_num_tokens": 40014771.0, "eval_reward": 0.07907940663176265, "eval_reward_std": 0.33248850828065923, "eval_rewards/reward_fn/mean": 0.07907940663176265, "eval_rewards/reward_fn/std": 0.3324885173305791, "eval_runtime": 1648.4789, "eval_samples_per_second": 0.348, "eval_sampling/importance_sampling_ratio/max": 1.2619268696553627, "eval_sampling/importance_sampling_ratio/mean": 0.9999836871553258, "eval_sampling/importance_sampling_ratio/min": 0.7830266549949246, "eval_sampling/sampling_logp_difference/max": 0.2691153499572481, "eval_sampling/sampling_logp_difference/mean": 0.020553737670755304, "eval_steps_per_second": 0.044, "step": 9200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.3, "completions/max_terminated_length": 306.3, "completions/mean_length": 207.7625, "completions/mean_terminated_length": 207.7625, "completions/min_length": 134.3, "completions/min_terminated_length": 134.3, "entropy": 1.0039683520793914, "epoch": 0.8467408292727774, "frac_reward_zero_std": 0.2, "grad_norm": 5.9489359855651855, "kl": 0.04009920536191203, "learning_rate": 4.29844336505982e-06, "loss": 0.002, "num_tokens": 40058520.0, "reward": 0.05, "reward_std": 0.37287638187408445, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.37287638187408445, "sampling/importance_sampling_ratio/max": 1.2430763840675354, "sampling/importance_sampling_ratio/mean": 1.000257235765457, "sampling/importance_sampling_ratio/min": 0.7770889043807984, "sampling/sampling_logp_difference/max": 0.27759191393852234, "sampling/sampling_logp_difference/mean": 0.01987345777451992, "step": 9210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.0, "completions/max_terminated_length": 298.0, "completions/mean_length": 197.8875, "completions/mean_terminated_length": 197.8875, "completions/min_length": 125.6, "completions/min_terminated_length": 125.6, "entropy": 1.1288753256201745, "epoch": 0.8476602004229107, "frac_reward_zero_std": 0.3, "grad_norm": 6.102019309997559, "kl": 0.04638625959632918, "learning_rate": 4.296682537047357e-06, "loss": 0.0021, "num_tokens": 40102391.0, "reward": -0.1125, "reward_std": 0.31855743527412417, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.3185574531555176, "sampling/importance_sampling_ratio/max": 1.291967785358429, "sampling/importance_sampling_ratio/mean": 0.9998250126838684, "sampling/importance_sampling_ratio/min": 0.7486029088497161, "sampling/sampling_logp_difference/max": 0.3039748251438141, "sampling/sampling_logp_difference/mean": 0.022176310047507287, "step": 9220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 217.95, "completions/mean_terminated_length": 217.95, "completions/min_length": 146.8, "completions/min_terminated_length": 146.8, "entropy": 1.076299573481083, "epoch": 0.8485795715730441, "frac_reward_zero_std": 0.3, "grad_norm": 7.501277446746826, "kl": 0.041976634215097876, "learning_rate": 4.294919863697908e-06, "loss": 0.0025, "num_tokens": 40146483.0, "reward": 0.4625, "reward_std": 0.36099914014339446, "rewards/reward_fn/mean": 0.4625, "rewards/reward_fn/std": 0.3609991520643234, "sampling/importance_sampling_ratio/max": 1.2423346042633057, "sampling/importance_sampling_ratio/mean": 0.9995251178741456, "sampling/importance_sampling_ratio/min": 0.7751516163349151, "sampling/sampling_logp_difference/max": 0.2673645973205566, "sampling/sampling_logp_difference/mean": 0.02062288634479046, "step": 9230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.6, "completions/max_terminated_length": 280.6, "completions/mean_length": 204.4875, "completions/mean_terminated_length": 204.4875, "completions/min_length": 147.6, "completions/min_terminated_length": 147.6, "entropy": 1.1663407430052757, "epoch": 0.8494989427231774, "frac_reward_zero_std": 0.2, "grad_norm": 0.10546322911977768, "kl": 0.03811493905377574, "learning_rate": 4.293155346821877e-06, "loss": 0.0018, "num_tokens": 40187570.0, "reward": -0.0875, "reward_std": 0.501947420835495, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.501947432756424, "sampling/importance_sampling_ratio/max": 1.2799391627311707, "sampling/importance_sampling_ratio/mean": 0.9999857306480407, "sampling/importance_sampling_ratio/min": 0.7642562031745911, "sampling/sampling_logp_difference/max": 0.297974967956543, "sampling/sampling_logp_difference/mean": 0.022754647955298423, "step": 9240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.9, "completions/max_terminated_length": 298.9, "completions/mean_length": 216.875, "completions/mean_terminated_length": 216.875, "completions/min_length": 135.0, "completions/min_terminated_length": 135.0, "entropy": 1.0723770407959818, "epoch": 0.8504183138733107, "frac_reward_zero_std": 0.4, "grad_norm": 5.826480388641357, "kl": 0.04773159505566582, "learning_rate": 4.291388988231567e-06, "loss": 0.0021, "num_tokens": 40232080.0, "reward": 0.0875, "reward_std": 0.338620987534523, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.338620999455452, "sampling/importance_sampling_ratio/max": 1.261952781677246, "sampling/importance_sampling_ratio/mean": 0.9998500108718872, "sampling/importance_sampling_ratio/min": 0.7857359766960144, "sampling/sampling_logp_difference/max": 0.25415499210357667, "sampling/sampling_logp_difference/mean": 0.02116878693923354, "step": 9250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0375, "completions/max_length": 285.5, "completions/max_terminated_length": 280.5, "completions/mean_length": 212.0125, "completions/mean_terminated_length": 205.42749938964843, "completions/min_length": 150.5, "completions/min_terminated_length": 150.5, "entropy": 1.0163930386304856, "epoch": 0.851337685023444, "frac_reward_zero_std": 0.5, "grad_norm": 7.354737281799316, "kl": 0.03842501549515873, "learning_rate": 4.289620789741169e-06, "loss": 0.0016, "num_tokens": 40275121.0, "reward": 0.04375, "reward_std": 0.26761444509029386, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.26761446297168734, "sampling/importance_sampling_ratio/max": 1.2281743884086609, "sampling/importance_sampling_ratio/mean": 1.0004379153251648, "sampling/importance_sampling_ratio/min": 0.7780608057975769, "sampling/sampling_logp_difference/max": 0.25594727993011473, "sampling/sampling_logp_difference/mean": 0.020437494665384293, "step": 9260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.4, "completions/max_terminated_length": 286.4, "completions/mean_length": 207.85, "completions/mean_terminated_length": 207.85, "completions/min_length": 135.4, "completions/min_terminated_length": 135.4, "entropy": 1.0362411573529244, "epoch": 0.8522570561735773, "frac_reward_zero_std": 0.3, "grad_norm": 5.695846080780029, "kl": 0.03845738079980947, "learning_rate": 4.2878507531667635e-06, "loss": 0.0016, "num_tokens": 40316525.0, "reward": 0.21875, "reward_std": 0.4025459736585617, "rewards/reward_fn/mean": 0.21875, "rewards/reward_fn/std": 0.40254599750041964, "sampling/importance_sampling_ratio/max": 1.347732961177826, "sampling/importance_sampling_ratio/mean": 1.0007512390613555, "sampling/importance_sampling_ratio/min": 0.8026901960372925, "sampling/sampling_logp_difference/max": 0.2978661060333252, "sampling/sampling_logp_difference/mean": 0.020542396046221257, "step": 9270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.2, "completions/max_terminated_length": 274.2, "completions/mean_length": 203.0, "completions/mean_terminated_length": 203.0, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.0238441742956639, "epoch": 0.8531764273237106, "frac_reward_zero_std": 0.4, "grad_norm": 5.857336044311523, "kl": 0.04517930138972588, "learning_rate": 4.286078880326321e-06, "loss": 0.002, "num_tokens": 40364389.0, "reward": -0.0625, "reward_std": 0.29780173897743223, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.29780174493789674, "sampling/importance_sampling_ratio/max": 1.275265920162201, "sampling/importance_sampling_ratio/mean": 1.0000287234783172, "sampling/importance_sampling_ratio/min": 0.7684423327445984, "sampling/sampling_logp_difference/max": 0.28910396099090574, "sampling/sampling_logp_difference/mean": 0.020778720453381538, "step": 9280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.3, "completions/max_terminated_length": 269.3, "completions/mean_length": 200.55, "completions/mean_terminated_length": 200.55, "completions/min_length": 144.9, "completions/min_terminated_length": 144.9, "entropy": 1.1211679950356483, "epoch": 0.8540957984738439, "frac_reward_zero_std": 0.3, "grad_norm": 0.11894954741001129, "kl": 0.04322960283025168, "learning_rate": 4.284305173039697e-06, "loss": 0.0015, "num_tokens": 40407969.0, "reward": -0.1875, "reward_std": 0.3685124158859253, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.36851242184638977, "sampling/importance_sampling_ratio/max": 1.3969801306724547, "sampling/importance_sampling_ratio/mean": 1.0002871692180633, "sampling/importance_sampling_ratio/min": 0.7616239607334137, "sampling/sampling_logp_difference/max": 0.4036160588264465, "sampling/sampling_logp_difference/mean": 0.022723359428346156, "step": 9290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 246.9, "completions/max_terminated_length": 246.9, "completions/mean_length": 180.3, "completions/mean_terminated_length": 180.3, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0072774156928062, "epoch": 0.8550151696239772, "frac_reward_zero_std": 0.3, "grad_norm": 7.7574663162231445, "kl": 0.05088316059554927, "learning_rate": 4.28252963312863e-06, "loss": 0.0027, "num_tokens": 40448225.0, "reward": 0.2, "reward_std": 0.3564485847949982, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.3564485967159271, "sampling/importance_sampling_ratio/max": 1.2754207134246827, "sampling/importance_sampling_ratio/mean": 1.0002938032150268, "sampling/importance_sampling_ratio/min": 0.7954947233200074, "sampling/sampling_logp_difference/max": 0.255824875831604, "sampling/sampling_logp_difference/mean": 0.020475606620311736, "step": 9300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 302.6, "completions/max_terminated_length": 292.3, "completions/mean_length": 207.1625, "completions/mean_terminated_length": 204.79107360839845, "completions/min_length": 143.6, "completions/min_terminated_length": 143.6, "entropy": 1.0452521368861198, "epoch": 0.8559345407741105, "frac_reward_zero_std": 0.3, "grad_norm": 5.0919904708862305, "kl": 0.028760029276600107, "learning_rate": 4.280752262416743e-06, "loss": 0.0015, "num_tokens": 40488478.0, "reward": 0.35625, "reward_std": 0.41326538026332854, "rewards/reward_fn/mean": 0.35625, "rewards/reward_fn/std": 0.41326538026332854, "sampling/importance_sampling_ratio/max": 1.253693902492523, "sampling/importance_sampling_ratio/mean": 1.0000602841377257, "sampling/importance_sampling_ratio/min": 0.7971062898635864, "sampling/sampling_logp_difference/max": 0.2408949613571167, "sampling/sampling_logp_difference/mean": 0.02095355149358511, "step": 9310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0625, "completions/max_length": 344.4, "completions/max_terminated_length": 342.5, "completions/mean_length": 257.3125, "completions/mean_terminated_length": 255.1875, "completions/min_length": 190.1, "completions/min_terminated_length": 190.1, "entropy": 1.0408134903758763, "epoch": 0.8568539119242439, "frac_reward_zero_std": 0.0, "grad_norm": 2.6079256534576416, "kl": 0.04141123883891851, "learning_rate": 4.2789730627295375e-06, "loss": 0.0019, "num_tokens": 40535351.0, "reward": -0.21875, "reward_std": 0.49128166437149046, "rewards/reward_fn/mean": -0.21875, "rewards/reward_fn/std": 0.4912816762924194, "sampling/importance_sampling_ratio/max": 1.244085967540741, "sampling/importance_sampling_ratio/mean": 0.999877679347992, "sampling/importance_sampling_ratio/min": 0.767325896024704, "sampling/sampling_logp_difference/max": 0.27972211837768557, "sampling/sampling_logp_difference/mean": 0.019891233555972575, "step": 9320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.0, "completions/max_terminated_length": 299.0, "completions/mean_length": 209.475, "completions/mean_terminated_length": 209.475, "completions/min_length": 134.8, "completions/min_terminated_length": 134.8, "entropy": 1.053185585141182, "epoch": 0.8577732830743772, "frac_reward_zero_std": 0.5, "grad_norm": 7.106479167938232, "kl": 0.0601667522219941, "learning_rate": 4.277192035894394e-06, "loss": 0.0031, "num_tokens": 40582397.0, "reward": 0.325, "reward_std": 0.2276727557182312, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.22767276763916017, "sampling/importance_sampling_ratio/max": 1.2634025931358337, "sampling/importance_sampling_ratio/mean": 0.9998867154121399, "sampling/importance_sampling_ratio/min": 0.779437643289566, "sampling/sampling_logp_difference/max": 0.2719349145889282, "sampling/sampling_logp_difference/mean": 0.021227684058248996, "step": 9330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 212.3375, "completions/mean_terminated_length": 212.3375, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.0828462682664395, "epoch": 0.8586926542245105, "frac_reward_zero_std": 0.8, "grad_norm": 7.119572639465332, "kl": 0.05455377746839076, "learning_rate": 4.275409183740571e-06, "loss": 0.0029, "num_tokens": 40626552.0, "reward": 0.225, "reward_std": 0.07071067690849304, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.07071067690849304, "sampling/importance_sampling_ratio/max": 1.2365211606025697, "sampling/importance_sampling_ratio/mean": 0.9997987985610962, "sampling/importance_sampling_ratio/min": 0.7920197784900666, "sampling/sampling_logp_difference/max": 0.2397668719291687, "sampling/sampling_logp_difference/mean": 0.021641866490244865, "step": 9340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.1, "completions/max_terminated_length": 313.1, "completions/mean_length": 210.9125, "completions/mean_terminated_length": 210.9125, "completions/min_length": 119.9, "completions/min_terminated_length": 119.9, "entropy": 1.0368298828601836, "epoch": 0.8596120253746438, "frac_reward_zero_std": 0.4, "grad_norm": 7.060151100158691, "kl": 0.05147368920734152, "learning_rate": 4.2736245080992e-06, "loss": 0.0028, "num_tokens": 40671137.0, "reward": -0.025, "reward_std": 0.25040294229984283, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2504029482603073, "sampling/importance_sampling_ratio/max": 1.2670783519744873, "sampling/importance_sampling_ratio/mean": 1.0001860976219177, "sampling/importance_sampling_ratio/min": 0.7762269496917724, "sampling/sampling_logp_difference/max": 0.27801127433776857, "sampling/sampling_logp_difference/mean": 0.021501859277486803, "step": 9350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 210.0375, "completions/mean_terminated_length": 210.0375, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.0472055025398732, "epoch": 0.8605313965247771, "frac_reward_zero_std": 0.5, "grad_norm": 0.16727247834205627, "kl": 0.050492165464675054, "learning_rate": 4.271838010803285e-06, "loss": 0.0024, "num_tokens": 40712804.0, "reward": -0.075, "reward_std": 0.22051151394844054, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.2205115258693695, "sampling/importance_sampling_ratio/max": 1.27891263961792, "sampling/importance_sampling_ratio/mean": 1.0001486480236053, "sampling/importance_sampling_ratio/min": 0.7984284102916718, "sampling/sampling_logp_difference/max": 0.26039583683013917, "sampling/sampling_logp_difference/mean": 0.02063222713768482, "step": 9360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.7, "completions/max_terminated_length": 260.7, "completions/mean_length": 184.6, "completions/mean_terminated_length": 184.6, "completions/min_length": 123.0, "completions/min_terminated_length": 123.0, "entropy": 1.0257703594863414, "epoch": 0.8614507676749104, "frac_reward_zero_std": 0.3, "grad_norm": 4.872483253479004, "kl": 0.041812674043467266, "learning_rate": 4.270049693687703e-06, "loss": 0.0018, "num_tokens": 40752196.0, "reward": 0.1, "reward_std": 0.3038551896810532, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.30385519564151764, "sampling/importance_sampling_ratio/max": 1.2589488625526428, "sampling/importance_sampling_ratio/mean": 1.0001261413097382, "sampling/importance_sampling_ratio/min": 0.786538028717041, "sampling/sampling_logp_difference/max": 0.2517038404941559, "sampling/sampling_logp_difference/mean": 0.020944275427609682, "step": 9370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.8, "completions/max_terminated_length": 321.1, "completions/mean_length": 220.925, "completions/mean_terminated_length": 218.57142944335936, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1077851600944997, "epoch": 0.8623701388250437, "frac_reward_zero_std": 0.2, "grad_norm": 6.332583427429199, "kl": 0.035996814008103686, "learning_rate": 4.268259558589199e-06, "loss": 0.002, "num_tokens": 40794974.0, "reward": 0.00625, "reward_std": 0.4347910940647125, "rewards/reward_fn/mean": 0.00625, "rewards/reward_fn/std": 0.434791100025177, "sampling/importance_sampling_ratio/max": 1.2473416090011598, "sampling/importance_sampling_ratio/mean": 1.0004087567329407, "sampling/importance_sampling_ratio/min": 0.762635612487793, "sampling/sampling_logp_difference/max": 0.2881794452667236, "sampling/sampling_logp_difference/mean": 0.021480763517320155, "step": 9380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.9, "completions/max_terminated_length": 293.9, "completions/mean_length": 210.7625, "completions/mean_terminated_length": 210.7625, "completions/min_length": 127.9, "completions/min_terminated_length": 127.9, "entropy": 0.9693314112722874, "epoch": 0.863289509975177, "frac_reward_zero_std": 0.4, "grad_norm": 8.771162986755371, "kl": 0.04351616412168369, "learning_rate": 4.266467607346385e-06, "loss": 0.0019, "num_tokens": 40834755.0, "reward": 0.1375, "reward_std": 0.261338609457016, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.2613386154174805, "sampling/importance_sampling_ratio/max": 1.338568377494812, "sampling/importance_sampling_ratio/mean": 1.0003405570983888, "sampling/importance_sampling_ratio/min": 0.7924472033977509, "sampling/sampling_logp_difference/max": 0.29763525128364565, "sampling/sampling_logp_difference/mean": 0.019548206962645052, "step": 9390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.0, "completions/max_terminated_length": 326.0, "completions/mean_length": 225.0875, "completions/mean_terminated_length": 225.0875, "completions/min_length": 141.7, "completions/min_terminated_length": 141.7, "entropy": 1.1542256452143191, "epoch": 0.8642088811253102, "frac_reward_zero_std": 0.4, "grad_norm": 6.076078414916992, "kl": 0.03984848668333143, "learning_rate": 4.264673841799738e-06, "loss": 0.0012, "num_tokens": 40879154.0, "reward": 0.33125, "reward_std": 0.36538374722003936, "rewards/reward_fn/mean": 0.33125, "rewards/reward_fn/std": 0.36538375318050387, "sampling/importance_sampling_ratio/max": 1.29695383310318, "sampling/importance_sampling_ratio/mean": 0.9999223828315735, "sampling/importance_sampling_ratio/min": 0.7794753611087799, "sampling/sampling_logp_difference/max": 0.28198263645172117, "sampling/sampling_logp_difference/mean": 0.0225184241309762, "step": 9400 }, { "epoch": 0.8642088811253102, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 294.6282722513089, "eval_completions/max_terminated_length": 293.7434554973822, "eval_completions/mean_length": 211.76941535776615, "eval_completions/mean_terminated_length": 211.53758415501778, "eval_completions/min_length": 141.4240837696335, "eval_completions/min_terminated_length": 141.4240837696335, "eval_entropy": 1.1172681619359561, "eval_frac_reward_zero_std": 0.35951134380453753, "eval_kl": 0.04241976077240903, "eval_loss": 0.0022181645035743713, "eval_num_tokens": 40879154.0, "eval_reward": 0.07831588132635253, "eval_reward_std": 0.31453906210305177, "eval_rewards/reward_fn/mean": 0.07831588132635253, "eval_rewards/reward_fn/std": 0.31453907240123646, "eval_runtime": 1721.9439, "eval_samples_per_second": 0.333, "eval_sampling/importance_sampling_ratio/max": 1.2717437896012844, "eval_sampling/importance_sampling_ratio/mean": 1.0000231140155027, "eval_sampling/importance_sampling_ratio/min": 0.774518882468941, "eval_sampling/sampling_logp_difference/max": 0.2820114487128732, "eval_sampling/sampling_logp_difference/mean": 0.02131458327830826, "eval_steps_per_second": 0.042, "step": 9400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 278.4, "completions/max_terminated_length": 278.4, "completions/mean_length": 206.6625, "completions/mean_terminated_length": 206.6625, "completions/min_length": 139.0, "completions/min_terminated_length": 139.0, "entropy": 1.0495636306703091, "epoch": 0.8651282522754435, "frac_reward_zero_std": 0.5, "grad_norm": 6.368381977081299, "kl": 0.032928543823072684, "learning_rate": 4.2628782637916e-06, "loss": 0.0018, "num_tokens": 40921431.0, "reward": 0.15, "reward_std": 0.23691109120845794, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.23691110908985138, "sampling/importance_sampling_ratio/max": 1.303182554244995, "sampling/importance_sampling_ratio/mean": 1.0000178515911102, "sampling/importance_sampling_ratio/min": 0.7715028166770935, "sampling/sampling_logp_difference/max": 0.29765647649765015, "sampling/sampling_logp_difference/mean": 0.021498052589595317, "step": 9410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.5, "completions/max_terminated_length": 297.5, "completions/mean_length": 202.375, "completions/mean_terminated_length": 202.375, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.1371506549417973, "epoch": 0.866047623425577, "frac_reward_zero_std": 0.4, "grad_norm": 4.960000038146973, "kl": 0.058892602392006665, "learning_rate": 4.261080875166173e-06, "loss": 0.0029, "num_tokens": 40964781.0, "reward": 0.05, "reward_std": 0.3086789697408676, "rewards/reward_fn/mean": 0.05, "rewards/reward_fn/std": 0.3086789816617966, "sampling/importance_sampling_ratio/max": 1.2429131627082826, "sampling/importance_sampling_ratio/mean": 0.9996340036392212, "sampling/importance_sampling_ratio/min": 0.73636794090271, "sampling/sampling_logp_difference/max": 0.31735005378723147, "sampling/sampling_logp_difference/mean": 0.02245978843420744, "step": 9420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.8, "completions/max_terminated_length": 299.8, "completions/mean_length": 212.7875, "completions/mean_terminated_length": 212.7875, "completions/min_length": 137.0, "completions/min_terminated_length": 137.0, "entropy": 1.0053416728973388, "epoch": 0.8669669945757102, "frac_reward_zero_std": 0.2, "grad_norm": 6.631198406219482, "kl": 0.042906089848838744, "learning_rate": 4.25928167776952e-06, "loss": 0.0023, "num_tokens": 41015988.0, "reward": -0.1875, "reward_std": 0.4560851901769638, "rewards/reward_fn/mean": -0.1875, "rewards/reward_fn/std": 0.4560852020978928, "sampling/importance_sampling_ratio/max": 1.2623692631721497, "sampling/importance_sampling_ratio/mean": 0.9996741950511933, "sampling/importance_sampling_ratio/min": 0.775529658794403, "sampling/sampling_logp_difference/max": 0.26469323635101316, "sampling/sampling_logp_difference/mean": 0.02053749095648527, "step": 9430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.5, "completions/max_terminated_length": 290.5, "completions/mean_length": 200.8375, "completions/mean_terminated_length": 200.8375, "completions/min_length": 140.0, "completions/min_terminated_length": 140.0, "entropy": 1.0861917831003667, "epoch": 0.8678863657258435, "frac_reward_zero_std": 0.4, "grad_norm": 0.12043377757072449, "kl": 0.060240122966934, "learning_rate": 4.2574806734495585e-06, "loss": 0.0029, "num_tokens": 41059047.0, "reward": -0.1125, "reward_std": 0.2558668524026871, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.25586686432361605, "sampling/importance_sampling_ratio/max": 1.3006654381752014, "sampling/importance_sampling_ratio/mean": 1.000587022304535, "sampling/importance_sampling_ratio/min": 0.7896238088607788, "sampling/sampling_logp_difference/max": 0.2809868693351746, "sampling/sampling_logp_difference/mean": 0.021761747635900976, "step": 9440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 215.0625, "completions/mean_terminated_length": 215.0625, "completions/min_length": 144.7, "completions/min_terminated_length": 144.7, "entropy": 1.1245507352054118, "epoch": 0.8688057368759768, "frac_reward_zero_std": 0.3, "grad_norm": 5.208584308624268, "kl": 0.03909367670421489, "learning_rate": 4.2556778640560684e-06, "loss": 0.0021, "num_tokens": 41105612.0, "reward": 0.0375, "reward_std": 0.3389096111059189, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.33890961706638334, "sampling/importance_sampling_ratio/max": 1.3357073187828064, "sampling/importance_sampling_ratio/mean": 1.0006796002388, "sampling/importance_sampling_ratio/min": 0.7836547791957855, "sampling/sampling_logp_difference/max": 0.3044546961784363, "sampling/sampling_logp_difference/mean": 0.021762259677052496, "step": 9450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.6, "completions/max_terminated_length": 303.6, "completions/mean_length": 219.025, "completions/mean_terminated_length": 219.025, "completions/min_length": 139.7, "completions/min_terminated_length": 139.7, "entropy": 1.093241859972477, "epoch": 0.8697251080261101, "frac_reward_zero_std": 0.4, "grad_norm": 0.2245897501707077, "kl": 0.03936568092321977, "learning_rate": 4.253873251440675e-06, "loss": 0.0024, "num_tokens": 41149750.0, "reward": -0.0625, "reward_std": 0.33463136851787567, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.3346313744783401, "sampling/importance_sampling_ratio/max": 1.3437366247177125, "sampling/importance_sampling_ratio/mean": 1.0005652964115144, "sampling/importance_sampling_ratio/min": 0.7690230071544647, "sampling/sampling_logp_difference/max": 0.32501999139785764, "sampling/sampling_logp_difference/mean": 0.02135617621243, "step": 9460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 299.5, "completions/max_terminated_length": 299.5, "completions/mean_length": 218.7875, "completions/mean_terminated_length": 218.7875, "completions/min_length": 148.8, "completions/min_terminated_length": 148.8, "entropy": 1.0457527928054333, "epoch": 0.8706444791762434, "frac_reward_zero_std": 0.4, "grad_norm": 0.11355587840080261, "kl": 0.050437202595639975, "learning_rate": 4.252066837456863e-06, "loss": 0.0027, "num_tokens": 41196141.0, "reward": 0.075, "reward_std": 0.2575641840696335, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.25756419003009795, "sampling/importance_sampling_ratio/max": 1.2807958245277404, "sampling/importance_sampling_ratio/mean": 1.0000260055065155, "sampling/importance_sampling_ratio/min": 0.7765036642551422, "sampling/sampling_logp_difference/max": 0.27603033781051634, "sampling/sampling_logp_difference/mean": 0.021162834577262403, "step": 9470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.1, "completions/max_terminated_length": 282.1, "completions/mean_length": 193.7, "completions/mean_terminated_length": 193.7, "completions/min_length": 131.1, "completions/min_terminated_length": 131.1, "entropy": 1.0732170723378658, "epoch": 0.8715638503263767, "frac_reward_zero_std": 0.4, "grad_norm": 0.11755069345235825, "kl": 0.04790999605320394, "learning_rate": 4.250258623959964e-06, "loss": 0.0023, "num_tokens": 41236821.0, "reward": 0.1, "reward_std": 0.3255516141653061, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.325551638007164, "sampling/importance_sampling_ratio/max": 1.2618190407752992, "sampling/importance_sampling_ratio/mean": 0.99973384141922, "sampling/importance_sampling_ratio/min": 0.7422408819198608, "sampling/sampling_logp_difference/max": 0.3121058762073517, "sampling/sampling_logp_difference/mean": 0.0211736012250185, "step": 9480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.0, "completions/max_terminated_length": 290.0, "completions/mean_length": 208.4875, "completions/mean_terminated_length": 208.4875, "completions/min_length": 139.4, "completions/min_terminated_length": 139.4, "entropy": 1.064863720536232, "epoch": 0.87248322147651, "frac_reward_zero_std": 0.2, "grad_norm": 6.2439799308776855, "kl": 0.043975573859643194, "learning_rate": 4.248448612807158e-06, "loss": 0.0027, "num_tokens": 41280268.0, "reward": 0.1625, "reward_std": 0.3689270466566086, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.368927064538002, "sampling/importance_sampling_ratio/max": 1.262490689754486, "sampling/importance_sampling_ratio/mean": 1.0003486037254334, "sampling/importance_sampling_ratio/min": 0.7779062211513519, "sampling/sampling_logp_difference/max": 0.27479383945465086, "sampling/sampling_logp_difference/mean": 0.021718551218509675, "step": 9490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.0, "completions/max_terminated_length": 337.0, "completions/mean_length": 239.975, "completions/mean_terminated_length": 239.975, "completions/min_length": 157.2, "completions/min_terminated_length": 157.2, "entropy": 1.0880503192543984, "epoch": 0.8734025926266433, "frac_reward_zero_std": 0.5, "grad_norm": 0.11339802294969559, "kl": 0.043944471870781855, "learning_rate": 4.246636805857471e-06, "loss": 0.0019, "num_tokens": 41329050.0, "reward": -0.025, "reward_std": 0.2328435957431793, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2328436017036438, "sampling/importance_sampling_ratio/max": 1.2938705444335938, "sampling/importance_sampling_ratio/mean": 1.0003528118133544, "sampling/importance_sampling_ratio/min": 0.7880222916603088, "sampling/sampling_logp_difference/max": 0.2763191759586334, "sampling/sampling_logp_difference/mean": 0.020726339891552925, "step": 9500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 265.4, "completions/max_terminated_length": 265.4, "completions/mean_length": 200.0875, "completions/mean_terminated_length": 200.0875, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.146623032540083, "epoch": 0.8743219637767767, "frac_reward_zero_std": 0.4, "grad_norm": 5.862491607666016, "kl": 0.047342038265196604, "learning_rate": 4.244823204971773e-06, "loss": 0.0023, "num_tokens": 41371377.0, "reward": -0.1, "reward_std": 0.26680251955986023, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.2668025314807892, "sampling/importance_sampling_ratio/max": 1.2572236895561217, "sampling/importance_sampling_ratio/mean": 0.9999559581279754, "sampling/importance_sampling_ratio/min": 0.7586696684360504, "sampling/sampling_logp_difference/max": 0.29272322058677674, "sampling/sampling_logp_difference/mean": 0.022369664534926413, "step": 9510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.0, "completions/max_terminated_length": 304.0, "completions/mean_length": 221.125, "completions/mean_terminated_length": 221.125, "completions/min_length": 151.0, "completions/min_terminated_length": 151.0, "entropy": 1.0671612076461314, "epoch": 0.87524133492691, "frac_reward_zero_std": 0.1, "grad_norm": 5.720339298248291, "kl": 0.04889992101234384, "learning_rate": 4.24300781201278e-06, "loss": 0.0026, "num_tokens": 41413307.0, "reward": 0.3125, "reward_std": 0.44760257601737974, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.4476025938987732, "sampling/importance_sampling_ratio/max": 1.3142322897911072, "sampling/importance_sampling_ratio/mean": 0.9997621655464173, "sampling/importance_sampling_ratio/min": 0.7700957477092742, "sampling/sampling_logp_difference/max": 0.2946740388870239, "sampling/sampling_logp_difference/mean": 0.021997046284377576, "step": 9520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.3, "completions/max_terminated_length": 312.3, "completions/mean_length": 229.6125, "completions/mean_terminated_length": 229.6125, "completions/min_length": 153.4, "completions/min_terminated_length": 153.4, "entropy": 1.2160911850631237, "epoch": 0.8761607060770433, "frac_reward_zero_std": 0.6, "grad_norm": 0.15232910215854645, "kl": 0.03515291656367481, "learning_rate": 4.2411906288450446e-06, "loss": 0.0017, "num_tokens": 41461516.0, "reward": 0.5, "reward_std": 0.1632926881313324, "rewards/reward_fn/mean": 0.5, "rewards/reward_fn/std": 0.1632926881313324, "sampling/importance_sampling_ratio/max": 1.2607879400253297, "sampling/importance_sampling_ratio/mean": 0.9998975217342376, "sampling/importance_sampling_ratio/min": 0.7959343075752259, "sampling/sampling_logp_difference/max": 0.2527146220207214, "sampling/sampling_logp_difference/mean": 0.02243077214807272, "step": 9530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.2, "completions/max_terminated_length": 303.2, "completions/mean_length": 216.95, "completions/mean_terminated_length": 216.95, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.095352490991354, "epoch": 0.8770800772271766, "frac_reward_zero_std": 0.3, "grad_norm": 0.13643012940883636, "kl": 0.05331411722581834, "learning_rate": 4.23937165733496e-06, "loss": 0.0025, "num_tokens": 41502536.0, "reward": -0.05, "reward_std": 0.3486855089664459, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.34868552088737487, "sampling/importance_sampling_ratio/max": 1.4292441010475159, "sampling/importance_sampling_ratio/mean": 0.9999718368053436, "sampling/importance_sampling_ratio/min": 0.7763973474502563, "sampling/sampling_logp_difference/max": 0.4093906044960022, "sampling/sampling_logp_difference/mean": 0.021223792061209678, "step": 9540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.9, "completions/max_terminated_length": 322.9, "completions/mean_length": 220.5875, "completions/mean_terminated_length": 220.5875, "completions/min_length": 144.1, "completions/min_terminated_length": 144.1, "entropy": 1.1236950181424619, "epoch": 0.8779994483773099, "frac_reward_zero_std": 0.3, "grad_norm": 0.12278574705123901, "kl": 0.03723166879499331, "learning_rate": 4.237550899350755e-06, "loss": 0.0022, "num_tokens": 41547423.0, "reward": 0.0375, "reward_std": 0.3257186770439148, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.32571869492530825, "sampling/importance_sampling_ratio/max": 1.319928765296936, "sampling/importance_sampling_ratio/mean": 0.9999402165412903, "sampling/importance_sampling_ratio/min": 0.7845882117748261, "sampling/sampling_logp_difference/max": 0.2943567276000977, "sampling/sampling_logp_difference/mean": 0.022001724503934384, "step": 9550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.7, "completions/max_terminated_length": 313.7, "completions/mean_length": 222.7875, "completions/mean_terminated_length": 222.7875, "completions/min_length": 155.5, "completions/min_terminated_length": 155.5, "entropy": 1.0925446793437004, "epoch": 0.8789188195274432, "frac_reward_zero_std": 0.4, "grad_norm": 5.862927436828613, "kl": 0.046284791472135114, "learning_rate": 4.235728356762496e-06, "loss": 0.0026, "num_tokens": 41590598.0, "reward": -0.2625, "reward_std": 0.2592615157365799, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.2592615157365799, "sampling/importance_sampling_ratio/max": 1.3269363164901733, "sampling/importance_sampling_ratio/mean": 0.9994530379772186, "sampling/importance_sampling_ratio/min": 0.7182049930095673, "sampling/sampling_logp_difference/max": 0.36123629212379454, "sampling/sampling_logp_difference/mean": 0.021487678214907647, "step": 9560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 273.8, "completions/max_terminated_length": 273.8, "completions/mean_length": 198.65, "completions/mean_terminated_length": 198.65, "completions/min_length": 138.6, "completions/min_terminated_length": 138.6, "entropy": 1.0299797058105469, "epoch": 0.8798381906775765, "frac_reward_zero_std": 0.5, "grad_norm": 6.6058125495910645, "kl": 0.026624112448189408, "learning_rate": 4.2339040314420785e-06, "loss": 0.0015, "num_tokens": 41629890.0, "reward": 0.3125, "reward_std": 0.26211473941802976, "rewards/reward_fn/mean": 0.3125, "rewards/reward_fn/std": 0.2621147453784943, "sampling/importance_sampling_ratio/max": 1.2468629002571106, "sampling/importance_sampling_ratio/mean": 0.9998441338539124, "sampling/importance_sampling_ratio/min": 0.7574172198772431, "sampling/sampling_logp_difference/max": 0.29169942140579225, "sampling/sampling_logp_difference/mean": 0.021236160211265086, "step": 9570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.6, "completions/max_terminated_length": 312.6, "completions/mean_length": 222.075, "completions/mean_terminated_length": 222.075, "completions/min_length": 143.0, "completions/min_terminated_length": 143.0, "entropy": 1.1217486038804054, "epoch": 0.8807575618277098, "frac_reward_zero_std": 0.5, "grad_norm": 5.760065078735352, "kl": 0.04545778785832226, "learning_rate": 4.232077925263231e-06, "loss": 0.0023, "num_tokens": 41674960.0, "reward": -0.1375, "reward_std": 0.19864802658557892, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.19864802658557892, "sampling/importance_sampling_ratio/max": 1.2878081321716308, "sampling/importance_sampling_ratio/mean": 1.0000447690486909, "sampling/importance_sampling_ratio/min": 0.7819167137145996, "sampling/sampling_logp_difference/max": 0.27499172687530515, "sampling/sampling_logp_difference/mean": 0.021269455552101135, "step": 9580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.1, "completions/max_terminated_length": 315.1, "completions/mean_length": 223.35, "completions/mean_terminated_length": 223.35, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.139740564674139, "epoch": 0.8816769329778431, "frac_reward_zero_std": 0.4, "grad_norm": 0.1076013520359993, "kl": 0.041789746819995345, "learning_rate": 4.230250040101511e-06, "loss": 0.0018, "num_tokens": 41720276.0, "reward": 0.10625, "reward_std": 0.29083392918109896, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.29083394110202787, "sampling/importance_sampling_ratio/max": 1.2562564134597778, "sampling/importance_sampling_ratio/mean": 0.999881899356842, "sampling/importance_sampling_ratio/min": 0.7805412769317627, "sampling/sampling_logp_difference/max": 0.2723170816898346, "sampling/sampling_logp_difference/mean": 0.021944673918187618, "step": 9590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.8, "completions/max_terminated_length": 301.8, "completions/mean_length": 218.6, "completions/mean_terminated_length": 218.6, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.1017316907644272, "epoch": 0.8825963041279765, "frac_reward_zero_std": 0.4, "grad_norm": 0.1838923692703247, "kl": 0.049498744611628356, "learning_rate": 4.2284203778343035e-06, "loss": 0.0029, "num_tokens": 41765396.0, "reward": 0.2375, "reward_std": 0.2857582807540894, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.28575828671455383, "sampling/importance_sampling_ratio/max": 1.2418395280838013, "sampling/importance_sampling_ratio/mean": 1.000712263584137, "sampling/importance_sampling_ratio/min": 0.7931682825088501, "sampling/sampling_logp_difference/max": 0.2401123285293579, "sampling/sampling_logp_difference/mean": 0.021757611073553562, "step": 9600 }, { "epoch": 0.8825963041279765, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0015270506108202443, "eval_completions/max_length": 291.6125654450262, "eval_completions/max_terminated_length": 290.26003490401393, "eval_completions/mean_length": 207.48058464223385, "eval_completions/mean_terminated_length": 207.09736782426816, "eval_completions/min_length": 136.27225130890054, "eval_completions/min_terminated_length": 136.27225130890054, "eval_entropy": 1.1127032290162424, "eval_frac_reward_zero_std": 0.3699825479930192, "eval_kl": 0.04880704622988769, "eval_loss": 0.0025783744640648365, "eval_num_tokens": 41765396.0, "eval_reward": 0.07242582897033159, "eval_reward_std": 0.30528518042639285, "eval_rewards/reward_fn/mean": 0.07242582897033159, "eval_rewards/reward_fn/std": 0.30528518791598175, "eval_runtime": 1705.7337, "eval_samples_per_second": 0.336, "eval_sampling/importance_sampling_ratio/max": 1.2728185106530447, "eval_sampling/importance_sampling_ratio/mean": 1.000017749805517, "eval_sampling/importance_sampling_ratio/min": 0.7790276485691936, "eval_sampling/sampling_logp_difference/max": 0.2783717569256328, "eval_sampling/sampling_logp_difference/mean": 0.02141451346728606, "eval_steps_per_second": 0.042, "step": 9600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.5, "completions/max_terminated_length": 269.5, "completions/mean_length": 197.9625, "completions/mean_terminated_length": 197.9625, "completions/min_length": 140.8, "completions/min_terminated_length": 140.8, "entropy": 1.0829307191073894, "epoch": 0.8835156752781098, "frac_reward_zero_std": 0.3, "grad_norm": 6.826517105102539, "kl": 0.056431339180562645, "learning_rate": 4.226588940340818e-06, "loss": 0.003, "num_tokens": 41806569.0, "reward": -0.0625, "reward_std": 0.3093190997838974, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.30931911170482634, "sampling/importance_sampling_ratio/max": 1.2439457416534423, "sampling/importance_sampling_ratio/mean": 0.9995703756809234, "sampling/importance_sampling_ratio/min": 0.7803526103496552, "sampling/sampling_logp_difference/max": 0.25469855070114134, "sampling/sampling_logp_difference/mean": 0.021404120698571207, "step": 9610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.1, "completions/max_terminated_length": 269.1, "completions/mean_length": 185.1375, "completions/mean_terminated_length": 185.1375, "completions/min_length": 125.6, "completions/min_terminated_length": 125.6, "entropy": 1.0163953557610512, "epoch": 0.8844350464282431, "frac_reward_zero_std": 0.5, "grad_norm": 5.391931056976318, "kl": 0.03501465924782678, "learning_rate": 4.224755729502088e-06, "loss": 0.0018, "num_tokens": 41849476.0, "reward": 0.1625, "reward_std": 0.3021578580141068, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.3021578699350357, "sampling/importance_sampling_ratio/max": 1.3150605797767638, "sampling/importance_sampling_ratio/mean": 1.0004523694515228, "sampling/importance_sampling_ratio/min": 0.7710909605026245, "sampling/sampling_logp_difference/max": 0.3159607410430908, "sampling/sampling_logp_difference/mean": 0.020232264138758182, "step": 9620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.5, "completions/max_terminated_length": 295.5, "completions/mean_length": 207.325, "completions/mean_terminated_length": 207.325, "completions/min_length": 119.2, "completions/min_terminated_length": 119.2, "entropy": 1.0288881592452526, "epoch": 0.8853544175783764, "frac_reward_zero_std": 0.2, "grad_norm": 5.479791641235352, "kl": 0.046140060760080814, "learning_rate": 4.222920747200967e-06, "loss": 0.0025, "num_tokens": 41894230.0, "reward": 0.2625, "reward_std": 0.3627713471651077, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.3627713590860367, "sampling/importance_sampling_ratio/max": 1.2772618174552917, "sampling/importance_sampling_ratio/mean": 0.999645447731018, "sampling/importance_sampling_ratio/min": 0.7572857320308686, "sampling/sampling_logp_difference/max": 0.29203481674194337, "sampling/sampling_logp_difference/mean": 0.02117515280842781, "step": 9630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.8, "completions/max_terminated_length": 309.8, "completions/mean_length": 208.975, "completions/mean_terminated_length": 208.975, "completions/min_length": 127.4, "completions/min_terminated_length": 127.4, "entropy": 1.1508677646517753, "epoch": 0.8862737887285097, "frac_reward_zero_std": 0.3, "grad_norm": 4.340304851531982, "kl": 0.06683948591817171, "learning_rate": 4.221083995322131e-06, "loss": 0.003, "num_tokens": 41935668.0, "reward": 0.0625, "reward_std": 0.321952098608017, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.32195210456848145, "sampling/importance_sampling_ratio/max": 1.2640284299850464, "sampling/importance_sampling_ratio/mean": 1.0002631366252899, "sampling/importance_sampling_ratio/min": 0.7422779858112335, "sampling/sampling_logp_difference/max": 0.3229871392250061, "sampling/sampling_logp_difference/mean": 0.022663329914212226, "step": 9640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 294.0, "completions/max_terminated_length": 294.0, "completions/mean_length": 222.2125, "completions/mean_terminated_length": 222.2125, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.1123330168426038, "epoch": 0.887193159878643, "frac_reward_zero_std": 0.6, "grad_norm": 5.3753557205200195, "kl": 0.04909877778845839, "learning_rate": 4.219245475752069e-06, "loss": 0.0023, "num_tokens": 41981845.0, "reward": -0.05, "reward_std": 0.18685350716114044, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.18685351312160492, "sampling/importance_sampling_ratio/max": 1.2773982405662536, "sampling/importance_sampling_ratio/mean": 1.0004447519779205, "sampling/importance_sampling_ratio/min": 0.7566041707992553, "sampling/sampling_logp_difference/max": 0.29627662897109985, "sampling/sampling_logp_difference/mean": 0.021852345764636995, "step": 9650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.6, "completions/max_terminated_length": 289.6, "completions/mean_length": 217.7125, "completions/mean_terminated_length": 217.7125, "completions/min_length": 136.4, "completions/min_terminated_length": 136.4, "entropy": 1.0822629570960998, "epoch": 0.8881125310287763, "frac_reward_zero_std": 0.1, "grad_norm": 6.162141799926758, "kl": 0.05092568336986005, "learning_rate": 4.217405190379089e-06, "loss": 0.0026, "num_tokens": 42025438.0, "reward": -0.0625, "reward_std": 0.4968440443277359, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.4968440443277359, "sampling/importance_sampling_ratio/max": 1.2648176193237304, "sampling/importance_sampling_ratio/mean": 1.0001706898212432, "sampling/importance_sampling_ratio/min": 0.7700137555599212, "sampling/sampling_logp_difference/max": 0.2854828894138336, "sampling/sampling_logp_difference/mean": 0.02105266060680151, "step": 9660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 346.8, "completions/max_terminated_length": 346.8, "completions/mean_length": 233.0125, "completions/mean_terminated_length": 233.0125, "completions/min_length": 146.3, "completions/min_terminated_length": 146.3, "entropy": 1.152537815272808, "epoch": 0.8890319021789096, "frac_reward_zero_std": 0.3, "grad_norm": 8.657646179199219, "kl": 0.05522806823719293, "learning_rate": 4.215563141093312e-06, "loss": 0.003, "num_tokens": 42073471.0, "reward": -0.0875, "reward_std": 0.3093190997838974, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.30931911170482634, "sampling/importance_sampling_ratio/max": 1.2799705028533936, "sampling/importance_sampling_ratio/mean": 1.0006804287433624, "sampling/importance_sampling_ratio/min": 0.7892005026340485, "sampling/sampling_logp_difference/max": 0.2680843472480774, "sampling/sampling_logp_difference/mean": 0.0221128661185503, "step": 9670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 254.8, "completions/max_terminated_length": 254.8, "completions/mean_length": 186.4375, "completions/mean_terminated_length": 186.4375, "completions/min_length": 115.5, "completions/min_terminated_length": 115.5, "entropy": 0.9961151644587517, "epoch": 0.8899512733290429, "frac_reward_zero_std": 0.3, "grad_norm": 6.703009128570557, "kl": 0.05263791738543659, "learning_rate": 4.213719329786669e-06, "loss": 0.0027, "num_tokens": 42113378.0, "reward": 0.1, "reward_std": 0.2974700778722763, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.29747008383274076, "sampling/importance_sampling_ratio/max": 1.250830590724945, "sampling/importance_sampling_ratio/mean": 0.9997282445430755, "sampling/importance_sampling_ratio/min": 0.7526599764823914, "sampling/sampling_logp_difference/max": 0.30122084617614747, "sampling/sampling_logp_difference/mean": 0.02062029894441366, "step": 9680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 307.6, "completions/max_terminated_length": 307.6, "completions/mean_length": 233.775, "completions/mean_terminated_length": 233.775, "completions/min_length": 163.5, "completions/min_terminated_length": 163.5, "entropy": 1.1191049337387085, "epoch": 0.8908706444791762, "frac_reward_zero_std": 0.3, "grad_norm": 6.562215805053711, "kl": 0.04675295646302402, "learning_rate": 4.211873758352905e-06, "loss": 0.0022, "num_tokens": 42161408.0, "reward": 0.0, "reward_std": 0.3878450483083725, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.387845054268837, "sampling/importance_sampling_ratio/max": 1.2485342264175414, "sampling/importance_sampling_ratio/mean": 1.000311905145645, "sampling/importance_sampling_ratio/min": 0.7557773053646087, "sampling/sampling_logp_difference/max": 0.2908876299858093, "sampling/sampling_logp_difference/mean": 0.02112082876265049, "step": 9690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 297.1, "completions/max_terminated_length": 297.1, "completions/mean_length": 230.5125, "completions/mean_terminated_length": 230.5125, "completions/min_length": 168.3, "completions/min_terminated_length": 168.3, "entropy": 1.1685070395469666, "epoch": 0.8917900156293096, "frac_reward_zero_std": 0.1, "grad_norm": 6.073295593261719, "kl": 0.060053869453258815, "learning_rate": 4.2100264286875655e-06, "loss": 0.0032, "num_tokens": 42208641.0, "reward": -0.075, "reward_std": 0.4228512436151505, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.4228512614965439, "sampling/importance_sampling_ratio/max": 1.2690693855285644, "sampling/importance_sampling_ratio/mean": 0.9993915319442749, "sampling/importance_sampling_ratio/min": 0.7842618942260742, "sampling/sampling_logp_difference/max": 0.2743223309516907, "sampling/sampling_logp_difference/mean": 0.022326854430139063, "step": 9700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.5, "completions/max_terminated_length": 296.5, "completions/mean_length": 224.625, "completions/mean_terminated_length": 224.625, "completions/min_length": 168.8, "completions/min_terminated_length": 168.8, "entropy": 1.0453823491930962, "epoch": 0.8927093867794429, "frac_reward_zero_std": 0.3, "grad_norm": 5.0743865966796875, "kl": 0.041981905058491975, "learning_rate": 4.208177342688009e-06, "loss": 0.0016, "num_tokens": 42255699.0, "reward": -0.025, "reward_std": 0.3716338902711868, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.3716339021921158, "sampling/importance_sampling_ratio/max": 1.2806673288345336, "sampling/importance_sampling_ratio/mean": 1.0004289627075196, "sampling/importance_sampling_ratio/min": 0.7232143551111221, "sampling/sampling_logp_difference/max": 0.3720430850982666, "sampling/sampling_logp_difference/mean": 0.020231438241899012, "step": 9710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.1, "completions/max_terminated_length": 269.1, "completions/mean_length": 192.2625, "completions/mean_terminated_length": 192.2625, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.163520159572363, "epoch": 0.8936287579295762, "frac_reward_zero_std": 0.6, "grad_norm": 8.238802909851074, "kl": 0.050715148559538646, "learning_rate": 4.206326502253395e-06, "loss": 0.0027, "num_tokens": 42299448.0, "reward": 0.0875, "reward_std": 0.1977891743183136, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.19778918027877807, "sampling/importance_sampling_ratio/max": 1.2369333624839782, "sampling/importance_sampling_ratio/mean": 0.9993122518062592, "sampling/importance_sampling_ratio/min": 0.800760930776596, "sampling/sampling_logp_difference/max": 0.23701193332672119, "sampling/sampling_logp_difference/mean": 0.022435952723026276, "step": 9720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.5, "completions/max_terminated_length": 276.5, "completions/mean_length": 192.0125, "completions/mean_terminated_length": 192.0125, "completions/min_length": 131.0, "completions/min_terminated_length": 131.0, "entropy": 1.1158318415284156, "epoch": 0.8945481290797095, "frac_reward_zero_std": 0.2, "grad_norm": 6.269508361816406, "kl": 0.048604181536938997, "learning_rate": 4.204473909284683e-06, "loss": 0.0025, "num_tokens": 42339105.0, "reward": 0.025, "reward_std": 0.41999801993370056, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.4199980318546295, "sampling/importance_sampling_ratio/max": 1.2698224186897278, "sampling/importance_sampling_ratio/mean": 0.9997045993804932, "sampling/importance_sampling_ratio/min": 0.7689110994338989, "sampling/sampling_logp_difference/max": 0.2866933584213257, "sampling/sampling_logp_difference/mean": 0.02233223579823971, "step": 9730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.8, "completions/max_terminated_length": 286.8, "completions/mean_length": 208.2875, "completions/mean_terminated_length": 208.2875, "completions/min_length": 138.5, "completions/min_terminated_length": 138.5, "entropy": 1.1071766138076782, "epoch": 0.8954675002298428, "frac_reward_zero_std": 0.3, "grad_norm": 0.12204127013683319, "kl": 0.05414634409826249, "learning_rate": 4.202619565684637e-06, "loss": 0.0026, "num_tokens": 42384864.0, "reward": 0.1375, "reward_std": 0.32480532228946685, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.3248053342103958, "sampling/importance_sampling_ratio/max": 1.278862726688385, "sampling/importance_sampling_ratio/mean": 1.0000852704048158, "sampling/importance_sampling_ratio/min": 0.7806693851947785, "sampling/sampling_logp_difference/max": 0.2705940842628479, "sampling/sampling_logp_difference/mean": 0.021467828005552293, "step": 9740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 243.7, "completions/max_terminated_length": 243.7, "completions/mean_length": 177.5375, "completions/mean_terminated_length": 177.5375, "completions/min_length": 107.5, "completions/min_terminated_length": 107.5, "entropy": 1.036444703489542, "epoch": 0.8963868713799761, "frac_reward_zero_std": 0.3, "grad_norm": 0.280074805021286, "kl": 0.0429881299380213, "learning_rate": 4.200763473357815e-06, "loss": 0.0021, "num_tokens": 42425307.0, "reward": 0.0875, "reward_std": 0.29291952252388, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.2929195284843445, "sampling/importance_sampling_ratio/max": 1.2604297876358033, "sampling/importance_sampling_ratio/mean": 0.99969522356987, "sampling/importance_sampling_ratio/min": 0.7613808035850524, "sampling/sampling_logp_difference/max": 0.2779817461967468, "sampling/sampling_logp_difference/mean": 0.021500294841825963, "step": 9750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 278.9, "completions/max_terminated_length": 269.6, "completions/mean_length": 197.5875, "completions/mean_terminated_length": 194.57857360839844, "completions/min_length": 119.8, "completions/min_terminated_length": 119.8, "entropy": 1.100955756008625, "epoch": 0.8973062425301094, "frac_reward_zero_std": 0.4, "grad_norm": 0.21760457754135132, "kl": 0.051878090959507975, "learning_rate": 4.198905634210574e-06, "loss": 0.0022, "num_tokens": 42468834.0, "reward": -0.11875, "reward_std": 0.32717646956443786, "rewards/reward_fn/mean": -0.11875, "rewards/reward_fn/std": 0.3271764814853668, "sampling/importance_sampling_ratio/max": 1.2780219674110413, "sampling/importance_sampling_ratio/mean": 0.9996322333812714, "sampling/importance_sampling_ratio/min": 0.7522034108638763, "sampling/sampling_logp_difference/max": 0.3005282640457153, "sampling/sampling_logp_difference/mean": 0.021998685225844382, "step": 9760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.5, "completions/max_terminated_length": 249.5, "completions/mean_length": 185.1375, "completions/mean_terminated_length": 185.1375, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.058626901358366, "epoch": 0.8982256136802427, "frac_reward_zero_std": 0.3, "grad_norm": 0.14348600804805756, "kl": 0.04166007454041391, "learning_rate": 4.197046050151061e-06, "loss": 0.0021, "num_tokens": 42507109.0, "reward": -0.0625, "reward_std": 0.3475900113582611, "rewards/reward_fn/mean": -0.0625, "rewards/reward_fn/std": 0.34759002923965454, "sampling/importance_sampling_ratio/max": 1.2821545600891113, "sampling/importance_sampling_ratio/mean": 1.0005722522735596, "sampling/importance_sampling_ratio/min": 0.7877614319324493, "sampling/sampling_logp_difference/max": 0.2655537486076355, "sampling/sampling_logp_difference/mean": 0.021121001057326793, "step": 9770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.8, "completions/max_terminated_length": 280.8, "completions/mean_length": 196.075, "completions/mean_terminated_length": 196.075, "completions/min_length": 126.5, "completions/min_terminated_length": 126.5, "entropy": 1.1112434238195419, "epoch": 0.899144984830376, "frac_reward_zero_std": 0.5, "grad_norm": 6.007650375366211, "kl": 0.042061880818801, "learning_rate": 4.19518472308922e-06, "loss": 0.0019, "num_tokens": 42547011.0, "reward": 0.4875, "reward_std": 0.2095758467912674, "rewards/reward_fn/mean": 0.4875, "rewards/reward_fn/std": 0.20957585871219636, "sampling/importance_sampling_ratio/max": 1.2591874837875365, "sampling/importance_sampling_ratio/mean": 0.999758917093277, "sampling/importance_sampling_ratio/min": 0.7797321140766144, "sampling/sampling_logp_difference/max": 0.27152804136276243, "sampling/sampling_logp_difference/mean": 0.02207108736038208, "step": 9780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.0, "completions/max_terminated_length": 280.0, "completions/mean_length": 203.2125, "completions/mean_terminated_length": 203.2125, "completions/min_length": 124.9, "completions/min_terminated_length": 124.9, "entropy": 1.1558429151773453, "epoch": 0.9000643559805094, "frac_reward_zero_std": 0.6, "grad_norm": 9.507164001464844, "kl": 0.05067404091823846, "learning_rate": 4.193321654936784e-06, "loss": 0.0029, "num_tokens": 42586884.0, "reward": -0.025, "reward_std": 0.20295216739177704, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.20295217931270598, "sampling/importance_sampling_ratio/max": 1.2489335656166076, "sampling/importance_sampling_ratio/mean": 1.000390750169754, "sampling/importance_sampling_ratio/min": 0.7842198371887207, "sampling/sampling_logp_difference/max": 0.2511386752128601, "sampling/sampling_logp_difference/mean": 0.022299526445567607, "step": 9790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 284.0, "completions/max_terminated_length": 284.0, "completions/mean_length": 201.85, "completions/mean_terminated_length": 201.85, "completions/min_length": 131.3, "completions/min_terminated_length": 131.3, "entropy": 1.1762912645936012, "epoch": 0.9009837271306427, "frac_reward_zero_std": 0.4, "grad_norm": 0.12105719745159149, "kl": 0.05483786647673696, "learning_rate": 4.191456847607271e-06, "loss": 0.0023, "num_tokens": 42628088.0, "reward": -0.0125, "reward_std": 0.26681036651134493, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.26681036651134493, "sampling/importance_sampling_ratio/max": 1.2537169218063355, "sampling/importance_sampling_ratio/mean": 1.0003124713897704, "sampling/importance_sampling_ratio/min": 0.7906040370464325, "sampling/sampling_logp_difference/max": 0.25084388256073, "sampling/sampling_logp_difference/mean": 0.022775582410395144, "step": 9800 }, { "epoch": 0.9009837271306427, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0008726003490401396, "eval_completions/max_length": 280.74869109947645, "eval_completions/max_terminated_length": 279.5602094240838, "eval_completions/mean_length": 197.31239092495636, "eval_completions/mean_terminated_length": 197.07725634083906, "eval_completions/min_length": 127.43979057591623, "eval_completions/min_terminated_length": 127.43979057591623, "eval_entropy": 1.1062782172237182, "eval_frac_reward_zero_std": 0.34904013961605584, "eval_kl": 0.047489816098625126, "eval_loss": 0.0024971631355583668, "eval_num_tokens": 42628088.0, "eval_reward": 0.07569808027923211, "eval_reward_std": 0.31951899615881957, "eval_rewards/reward_fn/mean": 0.07569808027923211, "eval_rewards/reward_fn/std": 0.31951900489667323, "eval_runtime": 1652.3955, "eval_samples_per_second": 0.347, "eval_sampling/importance_sampling_ratio/max": 1.2657346534063263, "eval_sampling/importance_sampling_ratio/mean": 0.9999833720724828, "eval_sampling/importance_sampling_ratio/min": 0.7812690228795922, "eval_sampling/sampling_logp_difference/max": 0.2738112529534944, "eval_sampling/sampling_logp_difference/mean": 0.02136060033143087, "eval_steps_per_second": 0.044, "step": 9800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.2, "completions/max_terminated_length": 279.2, "completions/mean_length": 207.3625, "completions/mean_terminated_length": 207.3625, "completions/min_length": 148.7, "completions/min_terminated_length": 148.7, "entropy": 1.0698274560272694, "epoch": 0.901903098280776, "frac_reward_zero_std": 0.1, "grad_norm": 5.936789512634277, "kl": 0.058363816619385035, "learning_rate": 4.189590303015989e-06, "loss": 0.0029, "num_tokens": 42673981.0, "reward": 0.0, "reward_std": 0.4608172684907913, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4608172863721848, "sampling/importance_sampling_ratio/max": 1.3146578550338746, "sampling/importance_sampling_ratio/mean": 1.0000967919826507, "sampling/importance_sampling_ratio/min": 0.7536275386810303, "sampling/sampling_logp_difference/max": 0.3268724262714386, "sampling/sampling_logp_difference/mean": 0.02157493242993951, "step": 9810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 196.1625, "completions/mean_terminated_length": 196.1625, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 0.9949974793940782, "epoch": 0.9028224694309093, "frac_reward_zero_std": 0.5, "grad_norm": 0.13990524411201477, "kl": 0.0469113961677067, "learning_rate": 4.187722023080028e-06, "loss": 0.0023, "num_tokens": 42714114.0, "reward": 0.275, "reward_std": 0.2112731784582138, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.2112731844186783, "sampling/importance_sampling_ratio/max": 1.3319814682006836, "sampling/importance_sampling_ratio/mean": 1.0006243526935577, "sampling/importance_sampling_ratio/min": 0.7772722601890564, "sampling/sampling_logp_difference/max": 0.3133630394935608, "sampling/sampling_logp_difference/mean": 0.020639171823859215, "step": 9820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.7, "completions/max_terminated_length": 295.7, "completions/mean_length": 212.175, "completions/mean_terminated_length": 212.175, "completions/min_length": 143.3, "completions/min_terminated_length": 143.3, "entropy": 1.155884674936533, "epoch": 0.9037418405810426, "frac_reward_zero_std": 0.3, "grad_norm": 5.2534589767456055, "kl": 0.07070289278635755, "learning_rate": 4.1858520097182635e-06, "loss": 0.003, "num_tokens": 42758664.0, "reward": 0.0125, "reward_std": 0.29291952252388, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.2929195284843445, "sampling/importance_sampling_ratio/max": 1.2690081477165223, "sampling/importance_sampling_ratio/mean": 1.0004143595695496, "sampling/importance_sampling_ratio/min": 0.7489427924156189, "sampling/sampling_logp_difference/max": 0.3149022817611694, "sampling/sampling_logp_difference/mean": 0.02249901872128248, "step": 9830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.4, "completions/max_terminated_length": 266.4, "completions/mean_length": 187.575, "completions/mean_terminated_length": 187.575, "completions/min_length": 115.6, "completions/min_terminated_length": 115.6, "entropy": 1.0035645239055158, "epoch": 0.9046612117311759, "frac_reward_zero_std": 0.5, "grad_norm": 7.56233549118042, "kl": 0.05291398569825105, "learning_rate": 4.183980264851345e-06, "loss": 0.0027, "num_tokens": 42800046.0, "reward": -0.05, "reward_std": 0.2276727557182312, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.22767276763916017, "sampling/importance_sampling_ratio/max": 1.2687315464019775, "sampling/importance_sampling_ratio/mean": 0.9998767256736756, "sampling/importance_sampling_ratio/min": 0.7907434582710267, "sampling/sampling_logp_difference/max": 0.26653594970703126, "sampling/sampling_logp_difference/mean": 0.020193965174257755, "step": 9840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 282.0, "completions/max_terminated_length": 282.0, "completions/mean_length": 205.5125, "completions/mean_terminated_length": 205.5125, "completions/min_length": 146.5, "completions/min_terminated_length": 146.5, "entropy": 1.1803817585110665, "epoch": 0.9055805828813092, "frac_reward_zero_std": 0.4, "grad_norm": 4.895437240600586, "kl": 0.06135212822118774, "learning_rate": 4.182106790401708e-06, "loss": 0.0028, "num_tokens": 42844935.0, "reward": -0.0125, "reward_std": 0.2808319330215454, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.2808319330215454, "sampling/importance_sampling_ratio/max": 1.2608348369598388, "sampling/importance_sampling_ratio/mean": 1.000287228822708, "sampling/importance_sampling_ratio/min": 0.8008495986461639, "sampling/sampling_logp_difference/max": 0.2604467451572418, "sampling/sampling_logp_difference/mean": 0.022341980040073393, "step": 9850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 249.0, "completions/max_terminated_length": 249.0, "completions/mean_length": 176.9125, "completions/mean_terminated_length": 176.9125, "completions/min_length": 113.1, "completions/min_terminated_length": 113.1, "entropy": 1.1014343447983266, "epoch": 0.9064999540314425, "frac_reward_zero_std": 0.5, "grad_norm": 0.12081068754196167, "kl": 0.06852991638006642, "learning_rate": 4.1802315882935605e-06, "loss": 0.0037, "num_tokens": 42886400.0, "reward": -0.1125, "reward_std": 0.19864802658557892, "rewards/reward_fn/mean": -0.1125, "rewards/reward_fn/std": 0.19864802658557892, "sampling/importance_sampling_ratio/max": 1.275366449356079, "sampling/importance_sampling_ratio/mean": 0.9991794168949127, "sampling/importance_sampling_ratio/min": 0.7355645060539245, "sampling/sampling_logp_difference/max": 0.37657909393310546, "sampling/sampling_logp_difference/mean": 0.021964114159345627, "step": 9860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.1, "completions/max_terminated_length": 304.1, "completions/mean_length": 215.0375, "completions/mean_terminated_length": 215.0375, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.0922230079770088, "epoch": 0.9074193251815758, "frac_reward_zero_std": 0.5, "grad_norm": 5.570449352264404, "kl": 0.05736344971228391, "learning_rate": 4.178354660452885e-06, "loss": 0.0029, "num_tokens": 42934603.0, "reward": 0.1625, "reward_std": 0.22220884561538695, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.22220885157585143, "sampling/importance_sampling_ratio/max": 1.2846588969230652, "sampling/importance_sampling_ratio/mean": 1.00008225440979, "sampling/importance_sampling_ratio/min": 0.7467437446117401, "sampling/sampling_logp_difference/max": 0.2975830793380737, "sampling/sampling_logp_difference/mean": 0.021936732903122903, "step": 9870 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 266.0, "completions/max_terminated_length": 266.0, "completions/mean_length": 196.95, "completions/mean_terminated_length": 196.95, "completions/min_length": 130.1, "completions/min_terminated_length": 130.1, "entropy": 1.071858535706997, "epoch": 0.9083386963317092, "frac_reward_zero_std": 0.1, "grad_norm": 0.13019002974033356, "kl": 0.057886114652501416, "learning_rate": 4.176476008807435e-06, "loss": 0.0029, "num_tokens": 42975351.0, "reward": 0.13125, "reward_std": 0.40080119073390963, "rewards/reward_fn/mean": 0.13125, "rewards/reward_fn/std": 0.40080119073390963, "sampling/importance_sampling_ratio/max": 1.2981379866600036, "sampling/importance_sampling_ratio/mean": 1.0002736091613769, "sampling/importance_sampling_ratio/min": 0.7947067618370056, "sampling/sampling_logp_difference/max": 0.2724072337150574, "sampling/sampling_logp_difference/mean": 0.021525019966065883, "step": 9880 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 203.8625, "completions/mean_terminated_length": 203.8625, "completions/min_length": 132.2, "completions/min_terminated_length": 132.2, "entropy": 1.1793571673333645, "epoch": 0.9092580674818425, "frac_reward_zero_std": 0.6, "grad_norm": 0.12472919374704361, "kl": 0.053053760924376546, "learning_rate": 4.174595635286739e-06, "loss": 0.0028, "num_tokens": 43021732.0, "reward": 0.25, "reward_std": 0.20958369374275207, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.20958369374275207, "sampling/importance_sampling_ratio/max": 1.2728296995162964, "sampling/importance_sampling_ratio/mean": 0.9999460637569427, "sampling/importance_sampling_ratio/min": 0.7885064840316772, "sampling/sampling_logp_difference/max": 0.2676731765270233, "sampling/sampling_logp_difference/mean": 0.022822683677077293, "step": 9890 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.0, "completions/max_terminated_length": 283.0, "completions/mean_length": 207.425, "completions/mean_terminated_length": 207.425, "completions/min_length": 137.2, "completions/min_terminated_length": 137.2, "entropy": 1.11990627348423, "epoch": 0.9101774386319758, "frac_reward_zero_std": 0.4, "grad_norm": 5.911685943603516, "kl": 0.06903767628828064, "learning_rate": 4.17271354182209e-06, "loss": 0.0039, "num_tokens": 43066350.0, "reward": -0.025, "reward_std": 0.28659676015377045, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2865967661142349, "sampling/importance_sampling_ratio/max": 1.2657966613769531, "sampling/importance_sampling_ratio/mean": 1.0005126535892486, "sampling/importance_sampling_ratio/min": 0.797953587770462, "sampling/sampling_logp_difference/max": 0.2551021337509155, "sampling/sampling_logp_difference/mean": 0.021488823182880878, "step": 9900 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 255.6, "completions/max_terminated_length": 255.6, "completions/mean_length": 188.6625, "completions/mean_terminated_length": 188.6625, "completions/min_length": 130.6, "completions/min_terminated_length": 130.6, "entropy": 1.0584544934332372, "epoch": 0.911096809782109, "frac_reward_zero_std": 0.4, "grad_norm": 7.512333869934082, "kl": 0.06185520152794197, "learning_rate": 4.170829730346549e-06, "loss": 0.0033, "num_tokens": 43105163.0, "reward": 0.0875, "reward_std": 0.2394672751426697, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.23946728110313414, "sampling/importance_sampling_ratio/max": 1.2746208310127258, "sampling/importance_sampling_ratio/mean": 0.9999439895153046, "sampling/importance_sampling_ratio/min": 0.769288969039917, "sampling/sampling_logp_difference/max": 0.290193772315979, "sampling/sampling_logp_difference/mean": 0.021467641554772855, "step": 9910 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.7, "completions/max_terminated_length": 270.7, "completions/mean_length": 187.6375, "completions/mean_terminated_length": 187.6375, "completions/min_length": 126.7, "completions/min_terminated_length": 126.7, "entropy": 1.0328537292778492, "epoch": 0.9120161809322423, "frac_reward_zero_std": 0.5, "grad_norm": 0.11428701877593994, "kl": 0.05851017193053849, "learning_rate": 4.168944202794942e-06, "loss": 0.003, "num_tokens": 43146038.0, "reward": 0.4125, "reward_std": 0.22220884561538695, "rewards/reward_fn/mean": 0.4125, "rewards/reward_fn/std": 0.22220885157585143, "sampling/importance_sampling_ratio/max": 1.2857640504837036, "sampling/importance_sampling_ratio/mean": 0.9997570037841796, "sampling/importance_sampling_ratio/min": 0.770370090007782, "sampling/sampling_logp_difference/max": 0.28543384075164796, "sampling/sampling_logp_difference/mean": 0.0211546678096056, "step": 9920 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 270.3, "completions/max_terminated_length": 270.3, "completions/mean_length": 184.525, "completions/mean_terminated_length": 184.525, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 1.0869535781443118, "epoch": 0.9129355520823756, "frac_reward_zero_std": 0.1, "grad_norm": 7.079781532287598, "kl": 0.06194143147440627, "learning_rate": 4.167056961103857e-06, "loss": 0.0031, "num_tokens": 43187976.0, "reward": -0.15, "reward_std": 0.41369563043117524, "rewards/reward_fn/mean": -0.15, "rewards/reward_fn/std": 0.4136956363916397, "sampling/importance_sampling_ratio/max": 1.2453197360038757, "sampling/importance_sampling_ratio/mean": 0.9996575653553009, "sampling/importance_sampling_ratio/min": 0.7913166224956513, "sampling/sampling_logp_difference/max": 0.24218196868896485, "sampling/sampling_logp_difference/mean": 0.02168115172535181, "step": 9930 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.0, "completions/max_terminated_length": 274.0, "completions/mean_length": 192.6375, "completions/mean_terminated_length": 192.6375, "completions/min_length": 120.1, "completions/min_terminated_length": 120.1, "entropy": 1.0639159835875034, "epoch": 0.9138549232325089, "frac_reward_zero_std": 0.4, "grad_norm": 0.12257706373929977, "kl": 0.05870253099128604, "learning_rate": 4.165168007211641e-06, "loss": 0.0028, "num_tokens": 43229947.0, "reward": 0.3375, "reward_std": 0.27851431667804716, "rewards/reward_fn/mean": 0.3375, "rewards/reward_fn/std": 0.2785143285989761, "sampling/importance_sampling_ratio/max": 1.2802238941192627, "sampling/importance_sampling_ratio/mean": 1.000627338886261, "sampling/importance_sampling_ratio/min": 0.8046356737613678, "sampling/sampling_logp_difference/max": 0.2600499868392944, "sampling/sampling_logp_difference/mean": 0.02083715479820967, "step": 9940 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 198.7625, "completions/mean_terminated_length": 198.7625, "completions/min_length": 126.1, "completions/min_terminated_length": 126.1, "entropy": 1.078206367045641, "epoch": 0.9147742943826422, "frac_reward_zero_std": 0.6, "grad_norm": 6.023697853088379, "kl": 0.041168544196989384, "learning_rate": 4.163277343058402e-06, "loss": 0.0024, "num_tokens": 43272808.0, "reward": 0.2375, "reward_std": 0.18138959705829621, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.18138959705829621, "sampling/importance_sampling_ratio/max": 1.2977841377258301, "sampling/importance_sampling_ratio/mean": 0.9998321771621704, "sampling/importance_sampling_ratio/min": 0.7729258179664612, "sampling/sampling_logp_difference/max": 0.29594180583953855, "sampling/sampling_logp_difference/mean": 0.021799837611615656, "step": 9950 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 339.7, "completions/max_terminated_length": 339.7, "completions/mean_length": 242.475, "completions/mean_terminated_length": 242.475, "completions/min_length": 160.3, "completions/min_terminated_length": 160.3, "entropy": 1.1220299161970615, "epoch": 0.9156936655327755, "frac_reward_zero_std": 0.4, "grad_norm": 5.079929828643799, "kl": 0.04531771424226463, "learning_rate": 4.1613849705860045e-06, "loss": 0.0021, "num_tokens": 43317446.0, "reward": 0.0625, "reward_std": 0.307040074467659, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.30704008638858793, "sampling/importance_sampling_ratio/max": 1.328687596321106, "sampling/importance_sampling_ratio/mean": 1.000167453289032, "sampling/importance_sampling_ratio/min": 0.7499541193246841, "sampling/sampling_logp_difference/max": 0.3055580735206604, "sampling/sampling_logp_difference/mean": 0.0210553215816617, "step": 9960 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.8, "completions/max_terminated_length": 298.8, "completions/mean_length": 207.5625, "completions/mean_terminated_length": 207.5625, "completions/min_length": 138.3, "completions/min_terminated_length": 138.3, "entropy": 1.0736882220953703, "epoch": 0.9166130366829089, "frac_reward_zero_std": 0.3, "grad_norm": 5.911686897277832, "kl": 0.04477341107558459, "learning_rate": 4.159490891738065e-06, "loss": 0.0022, "num_tokens": 43359507.0, "reward": -0.0875, "reward_std": 0.3228109508752823, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.3228109508752823, "sampling/importance_sampling_ratio/max": 1.2542143225669862, "sampling/importance_sampling_ratio/mean": 1.0001445591449738, "sampling/importance_sampling_ratio/min": 0.7846239686012269, "sampling/sampling_logp_difference/max": 0.2642161905765533, "sampling/sampling_logp_difference/mean": 0.02142866849899292, "step": 9970 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.0, "completions/max_terminated_length": 286.0, "completions/mean_length": 203.6625, "completions/mean_terminated_length": 203.6625, "completions/min_length": 128.2, "completions/min_terminated_length": 128.2, "entropy": 1.0757963739335537, "epoch": 0.9175324078330422, "frac_reward_zero_std": 0.2, "grad_norm": 0.130559504032135, "kl": 0.048978907556738706, "learning_rate": 4.157595108459956e-06, "loss": 0.0024, "num_tokens": 43402576.0, "reward": -0.2125, "reward_std": 0.3728685349225998, "rewards/reward_fn/mean": -0.2125, "rewards/reward_fn/std": 0.37286854684352877, "sampling/importance_sampling_ratio/max": 1.3121480703353883, "sampling/importance_sampling_ratio/mean": 1.0003035902976989, "sampling/importance_sampling_ratio/min": 0.8058617532253265, "sampling/sampling_logp_difference/max": 0.2850976288318634, "sampling/sampling_logp_difference/mean": 0.02138405852019787, "step": 9980 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 333.7, "completions/max_terminated_length": 333.7, "completions/mean_length": 226.175, "completions/mean_terminated_length": 226.175, "completions/min_length": 149.9, "completions/min_terminated_length": 149.9, "entropy": 1.109637549519539, "epoch": 0.9184517789831755, "frac_reward_zero_std": 0.3, "grad_norm": 5.069444179534912, "kl": 0.03635451651061885, "learning_rate": 4.155697622698798e-06, "loss": 0.0021, "num_tokens": 43446054.0, "reward": 0.1875, "reward_std": 0.3595789670944214, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.35957897305488584, "sampling/importance_sampling_ratio/max": 1.2771041750907899, "sampling/importance_sampling_ratio/mean": 0.9994903087615967, "sampling/importance_sampling_ratio/min": 0.7605353116989135, "sampling/sampling_logp_difference/max": 0.29628159999847414, "sampling/sampling_logp_difference/mean": 0.02149726580828428, "step": 9990 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.3, "completions/max_terminated_length": 269.3, "completions/mean_length": 201.1125, "completions/mean_terminated_length": 201.1125, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.1079287990927695, "epoch": 0.9193711501333088, "frac_reward_zero_std": 0.3, "grad_norm": 10.081376075744629, "kl": 0.047466059145517644, "learning_rate": 4.153798436403461e-06, "loss": 0.0022, "num_tokens": 43487727.0, "reward": -0.0375, "reward_std": 0.39962208569049834, "rewards/reward_fn/mean": -0.0375, "rewards/reward_fn/std": 0.3996220976114273, "sampling/importance_sampling_ratio/max": 1.2673521995544434, "sampling/importance_sampling_ratio/mean": 1.0002014219760895, "sampling/importance_sampling_ratio/min": 0.7535767048597336, "sampling/sampling_logp_difference/max": 0.3077139317989349, "sampling/sampling_logp_difference/mean": 0.022001378051936627, "step": 10000 }, { "epoch": 0.9193711501333088, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 293.19895287958116, "eval_completions/max_terminated_length": 291.32460732984293, "eval_completions/mean_length": 206.18848167539267, "eval_completions/mean_terminated_length": 205.8286900012697, "eval_completions/min_length": 132.62303664921467, "eval_completions/min_terminated_length": 132.62303664921467, "eval_entropy": 1.106627268056803, "eval_frac_reward_zero_std": 0.33507853403141363, "eval_kl": 0.04242372071292466, "eval_loss": 0.002180905546993017, "eval_num_tokens": 43487727.0, "eval_reward": 0.07907940663176265, "eval_reward_std": 0.3361895477064409, "eval_rewards/reward_fn/mean": 0.07907940663176265, "eval_rewards/reward_fn/std": 0.336189557068427, "eval_runtime": 1714.9811, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2720633002475918, "eval_sampling/importance_sampling_ratio/mean": 1.0000231427255935, "eval_sampling/importance_sampling_ratio/min": 0.7819440018314133, "eval_sampling/sampling_logp_difference/max": 0.27638481627584127, "eval_sampling/sampling_logp_difference/mean": 0.021283950173925044, "eval_steps_per_second": 0.042, "step": 10000 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.6, "completions/max_terminated_length": 261.6, "completions/mean_length": 193.0875, "completions/mean_terminated_length": 193.0875, "completions/min_length": 135.9, "completions/min_terminated_length": 135.9, "entropy": 1.1175101712346076, "epoch": 0.9202905212834421, "frac_reward_zero_std": 0.4, "grad_norm": 4.9315080642700195, "kl": 0.04182431444060057, "learning_rate": 4.151897551524562e-06, "loss": 0.0022, "num_tokens": 43526478.0, "reward": -0.025, "reward_std": 0.25040294229984283, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2504029482603073, "sampling/importance_sampling_ratio/max": 1.279539656639099, "sampling/importance_sampling_ratio/mean": 1.0005105555057525, "sampling/importance_sampling_ratio/min": 0.7738273978233338, "sampling/sampling_logp_difference/max": 0.300219988822937, "sampling/sampling_logp_difference/mean": 0.022065847553312777, "step": 10010 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.7, "completions/max_terminated_length": 303.7, "completions/mean_length": 218.3625, "completions/mean_terminated_length": 218.3625, "completions/min_length": 147.8, "completions/min_terminated_length": 147.8, "entropy": 1.078858845680952, "epoch": 0.9212098924335754, "frac_reward_zero_std": 0.1, "grad_norm": 7.221519470214844, "kl": 0.034373096813214946, "learning_rate": 4.149994970014461e-06, "loss": 0.0018, "num_tokens": 43570179.0, "reward": -0.075, "reward_std": 0.49618045389652254, "rewards/reward_fn/mean": -0.075, "rewards/reward_fn/std": 0.496180459856987, "sampling/importance_sampling_ratio/max": 1.4886763453483582, "sampling/importance_sampling_ratio/mean": 1.0004341721534729, "sampling/importance_sampling_ratio/min": 0.7414919972419739, "sampling/sampling_logp_difference/max": 0.447513222694397, "sampling/sampling_logp_difference/mean": 0.021396753191947938, "step": 10020 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 271.0, "completions/max_terminated_length": 271.0, "completions/mean_length": 199.425, "completions/mean_terminated_length": 199.425, "completions/min_length": 129.1, "completions/min_terminated_length": 129.1, "entropy": 1.1138561457395553, "epoch": 0.9221292635837087, "frac_reward_zero_std": 0.3, "grad_norm": 7.212597846984863, "kl": 0.05524435707484372, "learning_rate": 4.1480906938272625e-06, "loss": 0.0027, "num_tokens": 43611853.0, "reward": 0.375, "reward_std": 0.3167575001716614, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.3167575001716614, "sampling/importance_sampling_ratio/max": 1.2674006819725037, "sampling/importance_sampling_ratio/mean": 0.9998987436294555, "sampling/importance_sampling_ratio/min": 0.7750279426574707, "sampling/sampling_logp_difference/max": 0.2715704143047333, "sampling/sampling_logp_difference/mean": 0.022041104920208453, "step": 10030 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.3, "completions/max_terminated_length": 316.3, "completions/mean_length": 218.375, "completions/mean_terminated_length": 218.375, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.1344326540827752, "epoch": 0.923048634733842, "frac_reward_zero_std": 0.7, "grad_norm": 0.12102018296718597, "kl": 0.04625592551892623, "learning_rate": 4.14618472491881e-06, "loss": 0.0024, "num_tokens": 43656387.0, "reward": 0.1375, "reward_std": 0.13509859144687653, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.13509859144687653, "sampling/importance_sampling_ratio/max": 1.2803356528282166, "sampling/importance_sampling_ratio/mean": 0.9999973893165588, "sampling/importance_sampling_ratio/min": 0.7905018031597137, "sampling/sampling_logp_difference/max": 0.2589735984802246, "sampling/sampling_logp_difference/mean": 0.021617263741791248, "step": 10040 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.4, "completions/max_terminated_length": 276.4, "completions/mean_length": 202.1375, "completions/mean_terminated_length": 202.1375, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.023564712703228, "epoch": 0.9239680058839753, "frac_reward_zero_std": 0.2, "grad_norm": 5.862313747406006, "kl": 0.043661712884204465, "learning_rate": 4.1442770652466894e-06, "loss": 0.002, "num_tokens": 43697830.0, "reward": -0.1375, "reward_std": 0.34637176990509033, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.3463717758655548, "sampling/importance_sampling_ratio/max": 1.2744048953056335, "sampling/importance_sampling_ratio/mean": 0.9996317446231842, "sampling/importance_sampling_ratio/min": 0.7942969143390656, "sampling/sampling_logp_difference/max": 0.24789116382598878, "sampling/sampling_logp_difference/mean": 0.020310519449412822, "step": 10050 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 321.9, "completions/max_terminated_length": 304.3, "completions/mean_length": 214.0, "completions/mean_terminated_length": 210.37857208251953, "completions/min_length": 134.7, "completions/min_terminated_length": 134.7, "entropy": 1.159387330710888, "epoch": 0.9248873770341086, "frac_reward_zero_std": 0.2, "grad_norm": 0.11104026436805725, "kl": 0.04339903194922954, "learning_rate": 4.142367716770216e-06, "loss": 0.0017, "num_tokens": 43741574.0, "reward": 0.11875, "reward_std": 0.37845076620578766, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.3784507840871811, "sampling/importance_sampling_ratio/max": 1.271242046356201, "sampling/importance_sampling_ratio/mean": 1.000460785627365, "sampling/importance_sampling_ratio/min": 0.750554895401001, "sampling/sampling_logp_difference/max": 0.31674994230270387, "sampling/sampling_logp_difference/mean": 0.023078887723386288, "step": 10060 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.9, "completions/max_terminated_length": 326.9, "completions/mean_length": 219.0375, "completions/mean_terminated_length": 219.0375, "completions/min_length": 133.6, "completions/min_terminated_length": 133.6, "entropy": 1.1225076988339424, "epoch": 0.925806748184242, "frac_reward_zero_std": 0.3, "grad_norm": 6.00573205947876, "kl": 0.03885363736189902, "learning_rate": 4.1404566814504465e-06, "loss": 0.0021, "num_tokens": 43785905.0, "reward": 0.1875, "reward_std": 0.34120489954948424, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.34120491743087766, "sampling/importance_sampling_ratio/max": 1.3655216336250304, "sampling/importance_sampling_ratio/mean": 1.0005740821361542, "sampling/importance_sampling_ratio/min": 0.7938507914543151, "sampling/sampling_logp_difference/max": 0.319132924079895, "sampling/sampling_logp_difference/mean": 0.021599754877388477, "step": 10070 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 311.9, "completions/max_terminated_length": 311.9, "completions/mean_length": 214.0, "completions/mean_terminated_length": 214.0, "completions/min_length": 136.2, "completions/min_terminated_length": 136.2, "entropy": 1.0852583475410937, "epoch": 0.9267261193343753, "frac_reward_zero_std": 0.3, "grad_norm": 6.248187065124512, "kl": 0.043491474015172574, "learning_rate": 4.138543961250167e-06, "loss": 0.0022, "num_tokens": 43830665.0, "reward": -0.2, "reward_std": 0.34928734302520753, "rewards/reward_fn/mean": -0.2, "rewards/reward_fn/std": 0.3492873549461365, "sampling/importance_sampling_ratio/max": 1.261192524433136, "sampling/importance_sampling_ratio/mean": 1.0002699196338654, "sampling/importance_sampling_ratio/min": 0.783576500415802, "sampling/sampling_logp_difference/max": 0.2687194228172302, "sampling/sampling_logp_difference/mean": 0.021574455499649047, "step": 10080 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.6, "completions/max_terminated_length": 290.6, "completions/mean_length": 198.5125, "completions/mean_terminated_length": 198.5125, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0963963769376277, "epoch": 0.9276454904845086, "frac_reward_zero_std": 0.4, "grad_norm": 0.16739299893379211, "kl": 0.04293804967892356, "learning_rate": 4.136629558133892e-06, "loss": 0.0024, "num_tokens": 43875738.0, "reward": -0.0125, "reward_std": 0.30754689276218417, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.3075469046831131, "sampling/importance_sampling_ratio/max": 1.2990479350090027, "sampling/importance_sampling_ratio/mean": 0.9999501526355743, "sampling/importance_sampling_ratio/min": 0.7951829969882965, "sampling/sampling_logp_difference/max": 0.2698585748672485, "sampling/sampling_logp_difference/mean": 0.02218684498220682, "step": 10090 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 269.6, "completions/max_terminated_length": 269.6, "completions/mean_length": 198.025, "completions/mean_terminated_length": 198.025, "completions/min_length": 127.6, "completions/min_terminated_length": 127.6, "entropy": 1.0207598965615035, "epoch": 0.9285648616346419, "frac_reward_zero_std": 0.2, "grad_norm": 6.476590633392334, "kl": 0.04273106640903279, "learning_rate": 4.134713474067871e-06, "loss": 0.002, "num_tokens": 43919132.0, "reward": 0.1875, "reward_std": 0.3834284096956253, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.38342841565608976, "sampling/importance_sampling_ratio/max": 1.241877281665802, "sampling/importance_sampling_ratio/mean": 1.0000837743282318, "sampling/importance_sampling_ratio/min": 0.7797056853771209, "sampling/sampling_logp_difference/max": 0.26245195865631105, "sampling/sampling_logp_difference/mean": 0.020670629851520062, "step": 10100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.4, "completions/max_terminated_length": 322.4, "completions/mean_length": 227.55, "completions/mean_terminated_length": 227.55, "completions/min_length": 152.8, "completions/min_terminated_length": 152.8, "entropy": 1.0423229604959487, "epoch": 0.9294842327847752, "frac_reward_zero_std": 0.3, "grad_norm": 0.10241851210594177, "kl": 0.033168822061270475, "learning_rate": 4.132795711020072e-06, "loss": 0.0014, "num_tokens": 43964264.0, "reward": -0.1625, "reward_std": 0.3882720351219177, "rewards/reward_fn/mean": -0.1625, "rewards/reward_fn/std": 0.38827205300331114, "sampling/importance_sampling_ratio/max": 1.2814345121383668, "sampling/importance_sampling_ratio/mean": 1.000113022327423, "sampling/importance_sampling_ratio/min": 0.7712997794151306, "sampling/sampling_logp_difference/max": 0.2917073488235474, "sampling/sampling_logp_difference/mean": 0.020850983448326588, "step": 10110 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.8, "completions/max_terminated_length": 272.8, "completions/mean_length": 193.2625, "completions/mean_terminated_length": 193.2625, "completions/min_length": 123.2, "completions/min_terminated_length": 123.2, "entropy": 1.0889507330954076, "epoch": 0.9304036039349085, "frac_reward_zero_std": 0.7, "grad_norm": 0.15005534887313843, "kl": 0.03510474353097379, "learning_rate": 4.130876270960194e-06, "loss": 0.0016, "num_tokens": 44006533.0, "reward": 0.375, "reward_std": 0.11700168251991272, "rewards/reward_fn/mean": 0.375, "rewards/reward_fn/std": 0.11700168251991272, "sampling/importance_sampling_ratio/max": 1.242002785205841, "sampling/importance_sampling_ratio/mean": 0.9996794700622559, "sampling/importance_sampling_ratio/min": 0.7888605296611786, "sampling/sampling_logp_difference/max": 0.24492315649986268, "sampling/sampling_logp_difference/mean": 0.02183618824928999, "step": 10120 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.0, "completions/max_terminated_length": 257.0, "completions/mean_length": 186.45, "completions/mean_terminated_length": 186.45, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.0243648651987314, "epoch": 0.9313229750850418, "frac_reward_zero_std": 0.5, "grad_norm": 5.017781734466553, "kl": 0.04347193767898716, "learning_rate": 4.128955155859655e-06, "loss": 0.002, "num_tokens": 44048841.0, "reward": 0.2375, "reward_std": 0.23830750584602356, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.23830751776695253, "sampling/importance_sampling_ratio/max": 1.2669746160507203, "sampling/importance_sampling_ratio/mean": 0.9998945713043212, "sampling/importance_sampling_ratio/min": 0.800959974527359, "sampling/sampling_logp_difference/max": 0.255924391746521, "sampling/sampling_logp_difference/mean": 0.020703633781522512, "step": 10130 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.6, "completions/max_terminated_length": 298.6, "completions/mean_length": 209.8375, "completions/mean_terminated_length": 209.8375, "completions/min_length": 130.9, "completions/min_terminated_length": 130.9, "entropy": 1.0491491258144379, "epoch": 0.9322423462351751, "frac_reward_zero_std": 0.4, "grad_norm": 7.495232582092285, "kl": 0.03814993262058124, "learning_rate": 4.127032367691595e-06, "loss": 0.0021, "num_tokens": 44094612.0, "reward": 0.225, "reward_std": 0.3202547669410706, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3202547788619995, "sampling/importance_sampling_ratio/max": 1.2811823964118958, "sampling/importance_sampling_ratio/mean": 1.0000887930393219, "sampling/importance_sampling_ratio/min": 0.7703660070896149, "sampling/sampling_logp_difference/max": 0.2962468147277832, "sampling/sampling_logp_difference/mean": 0.02153892759233713, "step": 10140 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.3, "completions/max_terminated_length": 280.3, "completions/mean_length": 197.25, "completions/mean_terminated_length": 197.25, "completions/min_length": 131.7, "completions/min_terminated_length": 131.7, "entropy": 1.121644602715969, "epoch": 0.9331617173853084, "frac_reward_zero_std": 0.5, "grad_norm": 5.980490684509277, "kl": 0.0493826134246774, "learning_rate": 4.1251079084308696e-06, "loss": 0.0026, "num_tokens": 44134688.0, "reward": 0.2375, "reward_std": 0.2722664296627045, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.2722664475440979, "sampling/importance_sampling_ratio/max": 1.2625227212905883, "sampling/importance_sampling_ratio/mean": 1.0004570841789246, "sampling/importance_sampling_ratio/min": 0.8016109704971314, "sampling/sampling_logp_difference/max": 0.2479767084121704, "sampling/sampling_logp_difference/mean": 0.022446214407682418, "step": 10150 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.0, "completions/max_terminated_length": 310.0, "completions/mean_length": 219.35, "completions/mean_terminated_length": 219.35, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.1479847468435764, "epoch": 0.9340810885354418, "frac_reward_zero_std": 0.3, "grad_norm": 5.437140464782715, "kl": 0.038918071117950606, "learning_rate": 4.123181780054054e-06, "loss": 0.0024, "num_tokens": 44178548.0, "reward": -0.2625, "reward_std": 0.3743988037109375, "rewards/reward_fn/mean": -0.2625, "rewards/reward_fn/std": 0.37439881563186644, "sampling/importance_sampling_ratio/max": 1.2357716679573059, "sampling/importance_sampling_ratio/mean": 1.0000833213329314, "sampling/importance_sampling_ratio/min": 0.7927090287208557, "sampling/sampling_logp_difference/max": 0.24702129364013672, "sampling/sampling_logp_difference/mean": 0.02197853084653616, "step": 10160 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 310.7, "completions/max_terminated_length": 310.7, "completions/mean_length": 204.675, "completions/mean_terminated_length": 204.675, "completions/min_length": 122.5, "completions/min_terminated_length": 122.5, "entropy": 1.1130825310945511, "epoch": 0.9350004596855751, "frac_reward_zero_std": 0.5, "grad_norm": 0.14885349571704865, "kl": 0.03417475215392187, "learning_rate": 4.121253984539437e-06, "loss": 0.0015, "num_tokens": 44222338.0, "reward": 0.225, "reward_std": 0.336654344201088, "rewards/reward_fn/mean": 0.225, "rewards/reward_fn/std": 0.3366543620824814, "sampling/importance_sampling_ratio/max": 1.2490541458129882, "sampling/importance_sampling_ratio/mean": 0.999740082025528, "sampling/importance_sampling_ratio/min": 0.7749578654766083, "sampling/sampling_logp_difference/max": 0.26064772605895997, "sampling/sampling_logp_difference/mean": 0.021991936862468718, "step": 10170 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 279.7, "completions/max_terminated_length": 279.7, "completions/mean_length": 194.7125, "completions/mean_terminated_length": 194.7125, "completions/min_length": 130.0, "completions/min_terminated_length": 130.0, "entropy": 1.1041462056338787, "epoch": 0.9359198308357084, "frac_reward_zero_std": 0.5, "grad_norm": 6.3849334716796875, "kl": 0.03447888652444817, "learning_rate": 4.1193245238670185e-06, "loss": 0.0019, "num_tokens": 44263099.0, "reward": 0.0375, "reward_std": 0.2581017464399338, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.25810175240039823, "sampling/importance_sampling_ratio/max": 1.293313467502594, "sampling/importance_sampling_ratio/mean": 0.9996035099029541, "sampling/importance_sampling_ratio/min": 0.7614112794399261, "sampling/sampling_logp_difference/max": 0.288004732131958, "sampling/sampling_logp_difference/mean": 0.02223816458135843, "step": 10180 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 309.1, "completions/max_terminated_length": 309.1, "completions/mean_length": 227.825, "completions/mean_terminated_length": 227.825, "completions/min_length": 161.6, "completions/min_terminated_length": 161.6, "entropy": 1.059245329350233, "epoch": 0.9368392019858417, "frac_reward_zero_std": 0.1, "grad_norm": 6.509138107299805, "kl": 0.042996865278109905, "learning_rate": 4.117393400018508e-06, "loss": 0.0022, "num_tokens": 44307525.0, "reward": 0.275, "reward_std": 0.5376395136117935, "rewards/reward_fn/mean": 0.275, "rewards/reward_fn/std": 0.537639531493187, "sampling/importance_sampling_ratio/max": 1.243626093864441, "sampling/importance_sampling_ratio/mean": 0.9997692883014679, "sampling/importance_sampling_ratio/min": 0.771395879983902, "sampling/sampling_logp_difference/max": 0.2736286222934723, "sampling/sampling_logp_difference/mean": 0.021303676813840867, "step": 10190 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 261.6, "completions/max_terminated_length": 261.6, "completions/mean_length": 192.8125, "completions/mean_terminated_length": 192.8125, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.1562938824295999, "epoch": 0.937758573135975, "frac_reward_zero_std": 0.4, "grad_norm": 0.12893037497997284, "kl": 0.03760570762678981, "learning_rate": 4.115460614977327e-06, "loss": 0.002, "num_tokens": 44351230.0, "reward": 0.1875, "reward_std": 0.32165118157863615, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.32165118753910066, "sampling/importance_sampling_ratio/max": 1.2510242104530334, "sampling/importance_sampling_ratio/mean": 0.9993673980236053, "sampling/importance_sampling_ratio/min": 0.8053878366947174, "sampling/sampling_logp_difference/max": 0.24389951229095458, "sampling/sampling_logp_difference/mean": 0.02238678764551878, "step": 10200 }, { "epoch": 0.937758573135975, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 300.8848167539267, "eval_completions/max_terminated_length": 299.67539267015707, "eval_completions/mean_length": 212.23276614310646, "eval_completions/mean_terminated_length": 211.8950490635311, "eval_completions/min_length": 138.26352530541013, "eval_completions/min_terminated_length": 138.26352530541013, "eval_entropy": 1.1049813820637542, "eval_frac_reward_zero_std": 0.3298429319371728, "eval_kl": 0.03762181090226764, "eval_loss": 0.0018853785004466772, "eval_num_tokens": 44351230.0, "eval_reward": 0.06479057591623037, "eval_reward_std": 0.3453926005600635, "eval_rewards/reward_fn/mean": 0.06479057591623037, "eval_rewards/reward_fn/std": 0.3453926114823806, "eval_runtime": 1753.6436, "eval_samples_per_second": 0.327, "eval_sampling/importance_sampling_ratio/max": 1.2748777233909354, "eval_sampling/importance_sampling_ratio/mean": 1.0000669922920213, "eval_sampling/importance_sampling_ratio/min": 0.7792097128290573, "eval_sampling/sampling_logp_difference/max": 0.2789977702586855, "eval_sampling/sampling_logp_difference/mean": 0.021610155227036583, "eval_steps_per_second": 0.041, "step": 10200 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 345.5, "completions/max_terminated_length": 321.4, "completions/mean_length": 237.5625, "completions/mean_terminated_length": 233.4232147216797, "completions/min_length": 149.8, "completions/min_terminated_length": 149.8, "entropy": 1.1053246222436428, "epoch": 0.9386779442861083, "frac_reward_zero_std": 0.3, "grad_norm": 5.814712047576904, "kl": 0.0349264758056961, "learning_rate": 4.113526170728599e-06, "loss": 0.0019, "num_tokens": 44393211.0, "reward": 0.10625, "reward_std": 0.41637334525585173, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.41637335121631625, "sampling/importance_sampling_ratio/max": 1.3118617177009582, "sampling/importance_sampling_ratio/mean": 0.9999534308910369, "sampling/importance_sampling_ratio/min": 0.7669457614421844, "sampling/sampling_logp_difference/max": 0.30400129556655886, "sampling/sampling_logp_difference/mean": 0.022214807011187075, "step": 10210 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.0, "completions/max_terminated_length": 293.0, "completions/mean_length": 208.4375, "completions/mean_terminated_length": 208.4375, "completions/min_length": 148.5, "completions/min_terminated_length": 148.5, "entropy": 1.1892416171729565, "epoch": 0.9395973154362416, "frac_reward_zero_std": 0.1, "grad_norm": 5.578960418701172, "kl": 0.05054844588739797, "learning_rate": 4.111590069259154e-06, "loss": 0.0024, "num_tokens": 44440254.0, "reward": -0.3375, "reward_std": 0.3981345325708389, "rewards/reward_fn/mean": -0.3375, "rewards/reward_fn/std": 0.3981345325708389, "sampling/importance_sampling_ratio/max": 1.265317153930664, "sampling/importance_sampling_ratio/mean": 0.9993602216243744, "sampling/importance_sampling_ratio/min": 0.8003243505954742, "sampling/sampling_logp_difference/max": 0.257047438621521, "sampling/sampling_logp_difference/mean": 0.02259406503289938, "step": 10220 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.3, "completions/max_terminated_length": 293.3, "completions/mean_length": 218.675, "completions/mean_terminated_length": 218.675, "completions/min_length": 146.6, "completions/min_terminated_length": 146.6, "entropy": 1.0946271598339081, "epoch": 0.9405166865863749, "frac_reward_zero_std": 0.3, "grad_norm": 5.726904392242432, "kl": 0.03711949006537907, "learning_rate": 4.109652312557523e-06, "loss": 0.0018, "num_tokens": 44484620.0, "reward": 0.425, "reward_std": 0.3202547669410706, "rewards/reward_fn/mean": 0.425, "rewards/reward_fn/std": 0.3202547788619995, "sampling/importance_sampling_ratio/max": 1.2811763882637024, "sampling/importance_sampling_ratio/mean": 0.9998382806777955, "sampling/importance_sampling_ratio/min": 0.800939393043518, "sampling/sampling_logp_difference/max": 0.25599147081375123, "sampling/sampling_logp_difference/mean": 0.021071030013263225, "step": 10230 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.0, "completions/max_terminated_length": 291.0, "completions/mean_length": 209.725, "completions/mean_terminated_length": 209.725, "completions/min_length": 136.7, "completions/min_terminated_length": 136.7, "entropy": 1.0350177951157093, "epoch": 0.9414360577365082, "frac_reward_zero_std": 0.4, "grad_norm": 6.088034629821777, "kl": 0.024944693577708676, "learning_rate": 4.107712902613936e-06, "loss": 0.0012, "num_tokens": 44527414.0, "reward": 0.5625, "reward_std": 0.3484488636255264, "rewards/reward_fn/mean": 0.5625, "rewards/reward_fn/std": 0.34844887554645537, "sampling/importance_sampling_ratio/max": 1.3179100394248962, "sampling/importance_sampling_ratio/mean": 0.9999079942703247, "sampling/importance_sampling_ratio/min": 0.731504213809967, "sampling/sampling_logp_difference/max": 0.35937021374702455, "sampling/sampling_logp_difference/mean": 0.02133547030389309, "step": 10240 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 301.2, "completions/max_terminated_length": 301.2, "completions/mean_length": 212.9, "completions/mean_terminated_length": 212.9, "completions/min_length": 139.6, "completions/min_terminated_length": 139.6, "entropy": 1.0743322379887104, "epoch": 0.9423554288866416, "frac_reward_zero_std": 0.3, "grad_norm": 0.15123139321804047, "kl": 0.04109329528873786, "learning_rate": 4.105771841420326e-06, "loss": 0.0018, "num_tokens": 44571198.0, "reward": -0.0125, "reward_std": 0.34728909432888033, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.34728911221027375, "sampling/importance_sampling_ratio/max": 1.3009016036987304, "sampling/importance_sampling_ratio/mean": 0.9996633112430573, "sampling/importance_sampling_ratio/min": 0.7681370854377747, "sampling/sampling_logp_difference/max": 0.2990386962890625, "sampling/sampling_logp_difference/mean": 0.020904244296252726, "step": 10250 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.1, "completions/max_terminated_length": 277.1, "completions/mean_length": 198.0375, "completions/mean_terminated_length": 198.0375, "completions/min_length": 127.8, "completions/min_terminated_length": 127.8, "entropy": 1.1008296683430672, "epoch": 0.9432748000367749, "frac_reward_zero_std": 0.4, "grad_norm": 7.316898822784424, "kl": 0.052229932916816324, "learning_rate": 4.103829130970316e-06, "loss": 0.0025, "num_tokens": 44613385.0, "reward": 0.0375, "reward_std": 0.30008076429367064, "rewards/reward_fn/mean": 0.0375, "rewards/reward_fn/std": 0.30008077025413515, "sampling/importance_sampling_ratio/max": 1.3893833875656127, "sampling/importance_sampling_ratio/mean": 1.000041651725769, "sampling/importance_sampling_ratio/min": 0.7663739085197449, "sampling/sampling_logp_difference/max": 0.38043294548988343, "sampling/sampling_logp_difference/mean": 0.021824513375759125, "step": 10260 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 298.8, "completions/max_terminated_length": 298.8, "completions/mean_length": 201.95, "completions/mean_terminated_length": 201.95, "completions/min_length": 128.7, "completions/min_terminated_length": 128.7, "entropy": 1.087373359501362, "epoch": 0.9441941711869082, "frac_reward_zero_std": 0.2, "grad_norm": 5.971785068511963, "kl": 0.03778417706489563, "learning_rate": 4.101884773259226e-06, "loss": 0.0011, "num_tokens": 44655949.0, "reward": 0.075, "reward_std": 0.36277919411659243, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.36277919411659243, "sampling/importance_sampling_ratio/max": 1.2491565942764282, "sampling/importance_sampling_ratio/mean": 1.0001887202262878, "sampling/importance_sampling_ratio/min": 0.7989834904670715, "sampling/sampling_logp_difference/max": 0.24308784008026124, "sampling/sampling_logp_difference/mean": 0.021357456222176552, "step": 10270 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.7, "completions/max_terminated_length": 308.7, "completions/mean_length": 219.4375, "completions/mean_terminated_length": 219.4375, "completions/min_length": 148.0, "completions/min_terminated_length": 148.0, "entropy": 1.1221811167895794, "epoch": 0.9451135423370415, "frac_reward_zero_std": 0.2, "grad_norm": 5.410228729248047, "kl": 0.0418703637260478, "learning_rate": 4.099938770284066e-06, "loss": 0.002, "num_tokens": 44702456.0, "reward": 0.3, "reward_std": 0.42824027836322787, "rewards/reward_fn/mean": 0.3, "rewards/reward_fn/std": 0.4282402962446213, "sampling/importance_sampling_ratio/max": 1.3488599181175231, "sampling/importance_sampling_ratio/mean": 1.000151389837265, "sampling/importance_sampling_ratio/min": 0.7894112348556519, "sampling/sampling_logp_difference/max": 0.29748125076293946, "sampling/sampling_logp_difference/mean": 0.021881341747939587, "step": 10280 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 288.4, "completions/max_terminated_length": 288.4, "completions/mean_length": 210.875, "completions/mean_terminated_length": 210.875, "completions/min_length": 141.5, "completions/min_terminated_length": 141.5, "entropy": 1.1194141037762164, "epoch": 0.9460329134871748, "frac_reward_zero_std": 0.5, "grad_norm": 5.95265007019043, "kl": 0.03852440757327713, "learning_rate": 4.09799112404354e-06, "loss": 0.0018, "num_tokens": 44746390.0, "reward": 0.1625, "reward_std": 0.22768060266971588, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.22768060266971588, "sampling/importance_sampling_ratio/max": 1.2550797700881957, "sampling/importance_sampling_ratio/mean": 1.0006340980529784, "sampling/importance_sampling_ratio/min": 0.7692474305629731, "sampling/sampling_logp_difference/max": 0.2870879292488098, "sampling/sampling_logp_difference/mean": 0.02200090270489454, "step": 10290 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 312.4, "completions/max_terminated_length": 312.4, "completions/mean_length": 221.6875, "completions/mean_terminated_length": 221.6875, "completions/min_length": 152.8, "completions/min_terminated_length": 152.8, "entropy": 1.1098796352744102, "epoch": 0.9469522846373081, "frac_reward_zero_std": 0.5, "grad_norm": 0.12116257101297379, "kl": 0.04647766302805394, "learning_rate": 4.096041836538034e-06, "loss": 0.0023, "num_tokens": 44795693.0, "reward": 0.175, "reward_std": 0.22051151394844054, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.2205115258693695, "sampling/importance_sampling_ratio/max": 1.2870426893234252, "sampling/importance_sampling_ratio/mean": 0.9997973620891571, "sampling/importance_sampling_ratio/min": 0.7948153734207153, "sampling/sampling_logp_difference/max": 0.26373398303985596, "sampling/sampling_logp_difference/mean": 0.021479869447648525, "step": 10300 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 344.1, "completions/max_terminated_length": 344.1, "completions/mean_length": 231.1, "completions/mean_terminated_length": 231.1, "completions/min_length": 140.3, "completions/min_terminated_length": 140.3, "entropy": 1.074521803855896, "epoch": 0.9478716557874414, "frac_reward_zero_std": 0.5, "grad_norm": 4.28732967376709, "kl": 0.031806351919658485, "learning_rate": 4.094090909769624e-06, "loss": 0.0017, "num_tokens": 44841069.0, "reward": 0.11875, "reward_std": 0.2901452392339706, "rewards/reward_fn/mean": 0.11875, "rewards/reward_fn/std": 0.2901452511548996, "sampling/importance_sampling_ratio/max": 1.2504416108131409, "sampling/importance_sampling_ratio/mean": 0.9999383211135864, "sampling/importance_sampling_ratio/min": 0.8030679225921631, "sampling/sampling_logp_difference/max": 0.23733532428741455, "sampling/sampling_logp_difference/mean": 0.021142423897981644, "step": 10310 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 291.2, "completions/max_terminated_length": 291.2, "completions/mean_length": 212.4875, "completions/mean_terminated_length": 212.4875, "completions/min_length": 137.7, "completions/min_terminated_length": 137.7, "entropy": 1.1351409830152988, "epoch": 0.9487910269375747, "frac_reward_zero_std": 0.2, "grad_norm": 6.122869491577148, "kl": 0.04445941095473245, "learning_rate": 4.092138345742066e-06, "loss": 0.0024, "num_tokens": 44884084.0, "reward": 0.0, "reward_std": 0.43609668016433717, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.4360966980457306, "sampling/importance_sampling_ratio/max": 1.2811456561088561, "sampling/importance_sampling_ratio/mean": 0.9999609231948853, "sampling/importance_sampling_ratio/min": 0.8035965740680695, "sampling/sampling_logp_difference/max": 0.26156005859375, "sampling/sampling_logp_difference/mean": 0.022171403281390667, "step": 10320 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.8, "completions/max_terminated_length": 289.8, "completions/mean_length": 200.4625, "completions/mean_terminated_length": 200.4625, "completions/min_length": 123.5, "completions/min_terminated_length": 123.5, "entropy": 1.09742796048522, "epoch": 0.949710398087708, "frac_reward_zero_std": 0.2, "grad_norm": 7.2112860679626465, "kl": 0.04795564757660031, "learning_rate": 4.090184146460803e-06, "loss": 0.0019, "num_tokens": 44924873.0, "reward": 0.1375, "reward_std": 0.43528797626495364, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.43528798818588255, "sampling/importance_sampling_ratio/max": 1.2881731629371642, "sampling/importance_sampling_ratio/mean": 0.9997185647487641, "sampling/importance_sampling_ratio/min": 0.768883228302002, "sampling/sampling_logp_difference/max": 0.30654571056365965, "sampling/sampling_logp_difference/mean": 0.021639818139374257, "step": 10330 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.1, "completions/max_terminated_length": 286.1, "completions/mean_length": 209.4375, "completions/mean_terminated_length": 209.4375, "completions/min_length": 147.5, "completions/min_terminated_length": 147.5, "entropy": 1.1303051091730594, "epoch": 0.9506297692378414, "frac_reward_zero_std": 0.0, "grad_norm": 8.516763687133789, "kl": 0.04249354059575126, "learning_rate": 4.088228313932952e-06, "loss": 0.0023, "num_tokens": 44973852.0, "reward": -0.175, "reward_std": 0.48268860280513765, "rewards/reward_fn/mean": -0.175, "rewards/reward_fn/std": 0.48268862068653107, "sampling/importance_sampling_ratio/max": 1.2844378232955933, "sampling/importance_sampling_ratio/mean": 0.9999155104160309, "sampling/importance_sampling_ratio/min": 0.7481306195259094, "sampling/sampling_logp_difference/max": 0.3106665849685669, "sampling/sampling_logp_difference/mean": 0.0215362723916769, "step": 10340 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 324.5, "completions/max_terminated_length": 314.3, "completions/mean_length": 223.8875, "completions/mean_terminated_length": 220.54107208251952, "completions/min_length": 133.9, "completions/min_terminated_length": 133.9, "entropy": 1.0112129610031844, "epoch": 0.9515491403879747, "frac_reward_zero_std": 0.2, "grad_norm": 5.894485950469971, "kl": 0.03037277526455, "learning_rate": 4.08627085016731e-06, "loss": 0.0014, "num_tokens": 45019083.0, "reward": 0.09375, "reward_std": 0.479353791475296, "rewards/reward_fn/mean": 0.09375, "rewards/reward_fn/std": 0.4793538093566895, "sampling/importance_sampling_ratio/max": 1.2665707349777222, "sampling/importance_sampling_ratio/mean": 0.9999996721744537, "sampling/importance_sampling_ratio/min": 0.7669129192829132, "sampling/sampling_logp_difference/max": 0.27528184056282046, "sampling/sampling_logp_difference/mean": 0.020327751338481904, "step": 10350 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 224.5, "completions/mean_terminated_length": 224.5, "completions/min_length": 147.3, "completions/min_terminated_length": 147.3, "entropy": 1.100806525349617, "epoch": 0.952468511538108, "frac_reward_zero_std": 0.3, "grad_norm": 6.407718181610107, "kl": 0.03967062687734142, "learning_rate": 4.084311757174351e-06, "loss": 0.0015, "num_tokens": 45064171.0, "reward": 0.2625, "reward_std": 0.348606663942337, "rewards/reward_fn/mean": 0.2625, "rewards/reward_fn/std": 0.348606663942337, "sampling/importance_sampling_ratio/max": 1.2466299176216125, "sampling/importance_sampling_ratio/mean": 1.0002277851104737, "sampling/importance_sampling_ratio/min": 0.79264235496521, "sampling/sampling_logp_difference/max": 0.2533001065254211, "sampling/sampling_logp_difference/mean": 0.02188583854585886, "step": 10360 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 336.1, "completions/max_terminated_length": 322.4, "completions/mean_length": 232.3125, "completions/mean_terminated_length": 229.16607360839845, "completions/min_length": 141.0, "completions/min_terminated_length": 141.0, "entropy": 1.0800414830446243, "epoch": 0.9533878826882413, "frac_reward_zero_std": 0.3, "grad_norm": 4.4677653312683105, "kl": 0.03788230170030147, "learning_rate": 4.082351036966219e-06, "loss": 0.0021, "num_tokens": 45113820.0, "reward": -0.03125, "reward_std": 0.3796415150165558, "rewards/reward_fn/mean": -0.03125, "rewards/reward_fn/std": 0.37964152693748476, "sampling/importance_sampling_ratio/max": 1.3185320973396302, "sampling/importance_sampling_ratio/mean": 1.0000298917293549, "sampling/importance_sampling_ratio/min": 0.7852094233036041, "sampling/sampling_logp_difference/max": 0.29634692668914797, "sampling/sampling_logp_difference/mean": 0.02104109898209572, "step": 10370 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.4, "completions/max_terminated_length": 295.4, "completions/mean_length": 210.1625, "completions/mean_terminated_length": 210.1625, "completions/min_length": 132.5, "completions/min_terminated_length": 132.5, "entropy": 1.134573294967413, "epoch": 0.9543072538383746, "frac_reward_zero_std": 0.6, "grad_norm": 5.526520252227783, "kl": 0.04285432985052466, "learning_rate": 4.080388691556731e-06, "loss": 0.0021, "num_tokens": 45158217.0, "reward": -0.05, "reward_std": 0.18685350716114044, "rewards/reward_fn/mean": -0.05, "rewards/reward_fn/std": 0.18685351312160492, "sampling/importance_sampling_ratio/max": 1.3135084986686707, "sampling/importance_sampling_ratio/mean": 1.0004297852516175, "sampling/importance_sampling_ratio/min": 0.7943588852882385, "sampling/sampling_logp_difference/max": 0.2781773805618286, "sampling/sampling_logp_difference/mean": 0.021459191851317883, "step": 10380 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 292.1, "completions/max_terminated_length": 292.1, "completions/mean_length": 223.15, "completions/mean_terminated_length": 223.15, "completions/min_length": 146.0, "completions/min_terminated_length": 146.0, "entropy": 0.9993706054985523, "epoch": 0.9552266249885079, "frac_reward_zero_std": 0.0, "grad_norm": 5.806354522705078, "kl": 0.05351685233181343, "learning_rate": 4.0784247229613725e-06, "loss": 0.0022, "num_tokens": 45203213.0, "reward": 0.0125, "reward_std": 0.5401845872402191, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.5401846051216126, "sampling/importance_sampling_ratio/max": 1.3444734692573548, "sampling/importance_sampling_ratio/mean": 0.9998006582260132, "sampling/importance_sampling_ratio/min": 0.7332471027970314, "sampling/sampling_logp_difference/max": 0.42358884811401365, "sampling/sampling_logp_difference/mean": 0.02031092420220375, "step": 10390 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 337.3, "completions/max_terminated_length": 337.3, "completions/mean_length": 228.9875, "completions/mean_terminated_length": 228.9875, "completions/min_length": 141.3, "completions/min_terminated_length": 141.3, "entropy": 1.092549277842045, "epoch": 0.9561459961386412, "frac_reward_zero_std": 0.5, "grad_norm": 0.153485506772995, "kl": 0.04453683134051971, "learning_rate": 4.076459133197297e-06, "loss": 0.0026, "num_tokens": 45250036.0, "reward": -0.1375, "reward_std": 0.22768060266971588, "rewards/reward_fn/mean": -0.1375, "rewards/reward_fn/std": 0.22768060266971588, "sampling/importance_sampling_ratio/max": 1.2927642822265626, "sampling/importance_sampling_ratio/mean": 0.9999365508556366, "sampling/importance_sampling_ratio/min": 0.7745014488697052, "sampling/sampling_logp_difference/max": 0.2783425450325012, "sampling/sampling_logp_difference/mean": 0.020684493146836756, "step": 10400 }, { "epoch": 0.9561459961386412, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 299.35602094240835, "eval_completions/max_terminated_length": 297.6038394415358, "eval_completions/mean_length": 211.33987783595114, "eval_completions/mean_terminated_length": 210.99887819939258, "eval_completions/min_length": 136.3089005235602, "eval_completions/min_terminated_length": 136.3089005235602, "eval_entropy": 1.1057024242886282, "eval_frac_reward_zero_std": 0.35602094240837695, "eval_kl": 0.04526400217302078, "eval_loss": 0.002282296773046255, "eval_num_tokens": 45250036.0, "eval_reward": 0.06697207678883071, "eval_reward_std": 0.311346808939288, "eval_rewards/reward_fn/mean": 0.06697207678883071, "eval_rewards/reward_fn/std": 0.31134681632485484, "eval_runtime": 1746.1043, "eval_samples_per_second": 0.328, "eval_sampling/importance_sampling_ratio/max": 1.2714436260193431, "eval_sampling/importance_sampling_ratio/mean": 0.9999922506680248, "eval_sampling/importance_sampling_ratio/min": 0.783341272651213, "eval_sampling/sampling_logp_difference/max": 0.2741212345802763, "eval_sampling/sampling_logp_difference/mean": 0.021184583444856953, "eval_steps_per_second": 0.041, "step": 10400 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 285.2, "completions/max_terminated_length": 285.2, "completions/mean_length": 200.425, "completions/mean_terminated_length": 200.425, "completions/min_length": 132.1, "completions/min_terminated_length": 132.1, "entropy": 1.0518465787172318, "epoch": 0.9570653672887744, "frac_reward_zero_std": 0.3, "grad_norm": 0.1126018688082695, "kl": 0.04162971572950482, "learning_rate": 4.0744919242833245e-06, "loss": 0.002, "num_tokens": 45289734.0, "reward": 0.49375, "reward_std": 0.42412149012088773, "rewards/reward_fn/mean": 0.49375, "rewards/reward_fn/std": 0.4241215199232101, "sampling/importance_sampling_ratio/max": 1.2854412913322448, "sampling/importance_sampling_ratio/mean": 1.0004142999649048, "sampling/importance_sampling_ratio/min": 0.7297350227832794, "sampling/sampling_logp_difference/max": 0.33484777212142947, "sampling/sampling_logp_difference/mean": 0.022272344678640366, "step": 10410 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 341.9, "completions/max_terminated_length": 341.9, "completions/mean_length": 235.8625, "completions/mean_terminated_length": 235.8625, "completions/min_length": 153.6, "completions/min_terminated_length": 153.6, "entropy": 1.0714528247714044, "epoch": 0.9579847384389077, "frac_reward_zero_std": 0.7, "grad_norm": 4.253077507019043, "kl": 0.03938657914986834, "learning_rate": 4.072523098239934e-06, "loss": 0.0018, "num_tokens": 45338683.0, "reward": -0.0125, "reward_std": 0.10606601536273956, "rewards/reward_fn/mean": -0.0125, "rewards/reward_fn/std": 0.10606601536273956, "sampling/importance_sampling_ratio/max": 1.2506747603416444, "sampling/importance_sampling_ratio/mean": 1.0004184603691102, "sampling/importance_sampling_ratio/min": 0.7712302386760712, "sampling/sampling_logp_difference/max": 0.27767720222473147, "sampling/sampling_logp_difference/mean": 0.020465991646051406, "step": 10420 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 257.1, "completions/max_terminated_length": 257.1, "completions/mean_length": 186.15, "completions/mean_terminated_length": 186.15, "completions/min_length": 127.2, "completions/min_terminated_length": 127.2, "entropy": 1.0581976093351841, "epoch": 0.958904109589041, "frac_reward_zero_std": 0.6, "grad_norm": 0.12409593164920807, "kl": 0.045708527555689216, "learning_rate": 4.070552657089268e-06, "loss": 0.0024, "num_tokens": 45381039.0, "reward": 0.175, "reward_std": 0.18655259013175965, "rewards/reward_fn/mean": 0.175, "rewards/reward_fn/std": 0.18655259609222413, "sampling/importance_sampling_ratio/max": 1.2400532841682435, "sampling/importance_sampling_ratio/mean": 1.0003699898719787, "sampling/importance_sampling_ratio/min": 0.8047185063362121, "sampling/sampling_logp_difference/max": 0.23777810335159302, "sampling/sampling_logp_difference/mean": 0.020782478712499143, "step": 10430 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 290.4, "completions/max_terminated_length": 290.4, "completions/mean_length": 213.3875, "completions/mean_terminated_length": 213.3875, "completions/min_length": 147.1, "completions/min_terminated_length": 147.1, "entropy": 1.0435740955173969, "epoch": 0.9598234807391745, "frac_reward_zero_std": 0.5, "grad_norm": 0.14533483982086182, "kl": 0.04180521623347886, "learning_rate": 4.068580602855128e-06, "loss": 0.0016, "num_tokens": 45420550.0, "reward": 0.2375, "reward_std": 0.24434949159622193, "rewards/reward_fn/mean": 0.2375, "rewards/reward_fn/std": 0.2443494975566864, "sampling/importance_sampling_ratio/max": 1.3305577397346497, "sampling/importance_sampling_ratio/mean": 0.999878317117691, "sampling/importance_sampling_ratio/min": 0.7734332740306854, "sampling/sampling_logp_difference/max": 0.3185199499130249, "sampling/sampling_logp_difference/mean": 0.020883983559906484, "step": 10440 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 280.1, "completions/max_terminated_length": 280.1, "completions/mean_length": 201.8125, "completions/mean_terminated_length": 201.8125, "completions/min_length": 124.1, "completions/min_terminated_length": 124.1, "entropy": 1.0251258060336113, "epoch": 0.9607428518893077, "frac_reward_zero_std": 0.3, "grad_norm": 5.106839656829834, "kl": 0.05958384588593617, "learning_rate": 4.066606937562972e-06, "loss": 0.0027, "num_tokens": 45461055.0, "reward": 0.1375, "reward_std": 0.32838348150253294, "rewards/reward_fn/mean": 0.1375, "rewards/reward_fn/std": 0.3283834934234619, "sampling/importance_sampling_ratio/max": 1.235090935230255, "sampling/importance_sampling_ratio/mean": 1.0003528654575349, "sampling/importance_sampling_ratio/min": 0.7531449735164643, "sampling/sampling_logp_difference/max": 0.286450457572937, "sampling/sampling_logp_difference/mean": 0.02066156528890133, "step": 10450 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 308.6, "completions/max_terminated_length": 308.6, "completions/mean_length": 206.6125, "completions/mean_terminated_length": 206.6125, "completions/min_length": 129.2, "completions/min_terminated_length": 129.2, "entropy": 1.064510666579008, "epoch": 0.961662223039441, "frac_reward_zero_std": 0.4, "grad_norm": 0.22392216324806213, "kl": 0.04095704212086275, "learning_rate": 4.064631663239913e-06, "loss": 0.0018, "num_tokens": 45501960.0, "reward": 0.1, "reward_std": 0.28112500309944155, "rewards/reward_fn/mean": 0.1, "rewards/reward_fn/std": 0.28112501502037046, "sampling/importance_sampling_ratio/max": 1.3254770159721374, "sampling/importance_sampling_ratio/mean": 0.9999226272106171, "sampling/importance_sampling_ratio/min": 0.7863228738307952, "sampling/sampling_logp_difference/max": 0.2991049289703369, "sampling/sampling_logp_difference/mean": 0.02175606582313776, "step": 10460 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 328.4, "completions/max_terminated_length": 315.4, "completions/mean_length": 223.7875, "completions/mean_terminated_length": 220.75, "completions/min_length": 142.1, "completions/min_terminated_length": 142.1, "entropy": 1.109752208739519, "epoch": 0.9625815941895743, "frac_reward_zero_std": 0.3, "grad_norm": 0.12965983152389526, "kl": 0.04293912897119299, "learning_rate": 4.0626547819147145e-06, "loss": 0.002, "num_tokens": 45550591.0, "reward": 0.13125, "reward_std": 0.3534108102321625, "rewards/reward_fn/mean": 0.13125, "rewards/reward_fn/std": 0.3534108340740204, "sampling/importance_sampling_ratio/max": 1.2344344854354858, "sampling/importance_sampling_ratio/mean": 1.000164932012558, "sampling/importance_sampling_ratio/min": 0.755420196056366, "sampling/sampling_logp_difference/max": 0.2939692497253418, "sampling/sampling_logp_difference/mean": 0.02146379016339779, "step": 10470 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 332.6, "completions/max_terminated_length": 332.6, "completions/mean_length": 234.6875, "completions/mean_terminated_length": 234.6875, "completions/min_length": 142.3, "completions/min_terminated_length": 142.3, "entropy": 1.1467086419463157, "epoch": 0.9635009653397076, "frac_reward_zero_std": 0.4, "grad_norm": 0.1450110524892807, "kl": 0.04648222279502079, "learning_rate": 4.060676295617793e-06, "loss": 0.0025, "num_tokens": 45597862.0, "reward": 0.04375, "reward_std": 0.380746054649353, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.3807460844516754, "sampling/importance_sampling_ratio/max": 1.2943558812141418, "sampling/importance_sampling_ratio/mean": 0.9997293531894684, "sampling/importance_sampling_ratio/min": 0.7739385962486267, "sampling/sampling_logp_difference/max": 0.28885374069213865, "sampling/sampling_logp_difference/mean": 0.02238720078021288, "step": 10480 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 292.5, "completions/max_terminated_length": 276.3, "completions/mean_length": 200.425, "completions/mean_terminated_length": 197.0642868041992, "completions/min_length": 119.4, "completions/min_terminated_length": 119.4, "entropy": 1.0634149111807347, "epoch": 0.9644203364898409, "frac_reward_zero_std": 0.4, "grad_norm": 4.664721488952637, "kl": 0.044651976285967974, "learning_rate": 4.058696206381212e-06, "loss": 0.0021, "num_tokens": 45640568.0, "reward": -0.00625, "reward_std": 0.3202360600233078, "rewards/reward_fn/mean": -0.00625, "rewards/reward_fn/std": 0.3202360659837723, "sampling/importance_sampling_ratio/max": 1.307364273071289, "sampling/importance_sampling_ratio/mean": 1.0001919090747833, "sampling/importance_sampling_ratio/min": 0.7898947954177856, "sampling/sampling_logp_difference/max": 0.30578471422195436, "sampling/sampling_logp_difference/mean": 0.02079794593155384, "step": 10490 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 326.0, "completions/max_terminated_length": 326.0, "completions/mean_length": 212.4875, "completions/mean_terminated_length": 212.4875, "completions/min_length": 136.0, "completions/min_terminated_length": 136.0, "entropy": 1.1407897278666497, "epoch": 0.9653397076399742, "frac_reward_zero_std": 0.3, "grad_norm": 5.448066711425781, "kl": 0.03198553828406148, "learning_rate": 4.056714516238683e-06, "loss": 0.0016, "num_tokens": 45686087.0, "reward": -0.025, "reward_std": 0.36426674723625185, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.36426675915718076, "sampling/importance_sampling_ratio/max": 1.2527343869209289, "sampling/importance_sampling_ratio/mean": 1.0003202319145204, "sampling/importance_sampling_ratio/min": 0.7954131484031677, "sampling/sampling_logp_difference/max": 0.24433987140655516, "sampling/sampling_logp_difference/mean": 0.022898581624031068, "step": 10500 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.3, "completions/max_terminated_length": 316.3, "completions/mean_length": 225.7875, "completions/mean_terminated_length": 225.7875, "completions/min_length": 148.9, "completions/min_terminated_length": 148.9, "entropy": 1.0881177850067616, "epoch": 0.9662590787901075, "frac_reward_zero_std": 0.5, "grad_norm": 4.399587631225586, "kl": 0.048546767968218774, "learning_rate": 4.054731227225559e-06, "loss": 0.0027, "num_tokens": 45729318.0, "reward": 0.2, "reward_std": 0.2112731784582138, "rewards/reward_fn/mean": 0.2, "rewards/reward_fn/std": 0.2112731844186783, "sampling/importance_sampling_ratio/max": 1.3067634344100951, "sampling/importance_sampling_ratio/mean": 1.000279676914215, "sampling/importance_sampling_ratio/min": 0.8044389307498931, "sampling/sampling_logp_difference/max": 0.271681535243988, "sampling/sampling_logp_difference/mean": 0.021586478874087333, "step": 10510 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 329.8, "completions/max_terminated_length": 329.8, "completions/mean_length": 219.2, "completions/mean_terminated_length": 219.2, "completions/min_length": 125.2, "completions/min_terminated_length": 125.2, "entropy": 1.143178606033325, "epoch": 0.9671784499402408, "frac_reward_zero_std": 0.3, "grad_norm": 0.1652517020702362, "kl": 0.041311096376739444, "learning_rate": 4.0527463413788385e-06, "loss": 0.0022, "num_tokens": 45773622.0, "reward": 0.1625, "reward_std": 0.3155669867992401, "rewards/reward_fn/mean": 0.1625, "rewards/reward_fn/std": 0.31556699275970457, "sampling/importance_sampling_ratio/max": 1.2878233075141907, "sampling/importance_sampling_ratio/mean": 1.0004111528396606, "sampling/importance_sampling_ratio/min": 0.7516210079193115, "sampling/sampling_logp_difference/max": 0.3114285707473755, "sampling/sampling_logp_difference/mean": 0.021550540812313557, "step": 10520 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.7, "completions/max_terminated_length": 302.7, "completions/mean_length": 216.7875, "completions/mean_terminated_length": 216.7875, "completions/min_length": 139.5, "completions/min_terminated_length": 139.5, "entropy": 1.134613348543644, "epoch": 0.9680978210903742, "frac_reward_zero_std": 0.3, "grad_norm": 6.383565902709961, "kl": 0.04815573546802625, "learning_rate": 4.050759860737158e-06, "loss": 0.0024, "num_tokens": 45817549.0, "reward": 0.025, "reward_std": 0.2828427076339722, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.2828427076339722, "sampling/importance_sampling_ratio/max": 1.2433046221733093, "sampling/importance_sampling_ratio/mean": 0.9996812045574188, "sampling/importance_sampling_ratio/min": 0.7778947591781616, "sampling/sampling_logp_difference/max": 0.26329495906829836, "sampling/sampling_logp_difference/mean": 0.02150979358702898, "step": 10530 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.6, "completions/max_terminated_length": 277.6, "completions/mean_length": 198.5875, "completions/mean_terminated_length": 198.5875, "completions/min_length": 131.4, "completions/min_terminated_length": 131.4, "entropy": 1.0837464720010757, "epoch": 0.9690171922405075, "frac_reward_zero_std": 0.4, "grad_norm": 0.17261937260627747, "kl": 0.03875091255758889, "learning_rate": 4.0487717873407905e-06, "loss": 0.0019, "num_tokens": 45861508.0, "reward": 0.15, "reward_std": 0.2739637613296509, "rewards/reward_fn/mean": 0.15, "rewards/reward_fn/std": 0.27396377325057986, "sampling/importance_sampling_ratio/max": 1.2494304895401, "sampling/importance_sampling_ratio/mean": 1.000165432691574, "sampling/importance_sampling_ratio/min": 0.8084233701229095, "sampling/sampling_logp_difference/max": 0.23188527822494506, "sampling/sampling_logp_difference/mean": 0.020969351194798945, "step": 10540 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 328.2, "completions/max_terminated_length": 328.2, "completions/mean_length": 225.925, "completions/mean_terminated_length": 225.925, "completions/min_length": 132.3, "completions/min_terminated_length": 132.3, "entropy": 1.1044180184602737, "epoch": 0.9699365633906408, "frac_reward_zero_std": 0.3, "grad_norm": 4.855181694030762, "kl": 0.03996765591436997, "learning_rate": 4.046782123231649e-06, "loss": 0.0023, "num_tokens": 45904718.0, "reward": 0.0875, "reward_std": 0.332049286365509, "rewards/reward_fn/mean": 0.0875, "rewards/reward_fn/std": 0.3320492923259735, "sampling/importance_sampling_ratio/max": 1.2515955567359924, "sampling/importance_sampling_ratio/mean": 1.0004275441169739, "sampling/importance_sampling_ratio/min": 0.7667010009288788, "sampling/sampling_logp_difference/max": 0.2769248247146606, "sampling/sampling_logp_difference/mean": 0.02228622790426016, "step": 10550 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 293.5, "completions/max_terminated_length": 293.5, "completions/mean_length": 209.025, "completions/mean_terminated_length": 209.025, "completions/min_length": 145.1, "completions/min_terminated_length": 145.1, "entropy": 1.0964238591492177, "epoch": 0.9708559345407741, "frac_reward_zero_std": 0.5, "grad_norm": 0.0910649523139, "kl": 0.04920785235590301, "learning_rate": 4.044790870453277e-06, "loss": 0.0022, "num_tokens": 45948888.0, "reward": 0.25, "reward_std": 0.2927524596452713, "rewards/reward_fn/mean": 0.25, "rewards/reward_fn/std": 0.29275247156620027, "sampling/importance_sampling_ratio/max": 1.229606318473816, "sampling/importance_sampling_ratio/mean": 1.0000321090221405, "sampling/importance_sampling_ratio/min": 0.7531989216804504, "sampling/sampling_logp_difference/max": 0.2971944689750671, "sampling/sampling_logp_difference/mean": 0.02184291649609804, "step": 10560 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.6, "completions/max_terminated_length": 286.6, "completions/mean_length": 192.7, "completions/mean_terminated_length": 192.7, "completions/min_length": 125.0, "completions/min_terminated_length": 125.0, "entropy": 1.0779813207685947, "epoch": 0.9717753056909074, "frac_reward_zero_std": 0.3, "grad_norm": 5.180692672729492, "kl": 0.03135224454454146, "learning_rate": 4.042798031050851e-06, "loss": 0.0015, "num_tokens": 45990120.0, "reward": 0.3625, "reward_std": 0.2857582807540894, "rewards/reward_fn/mean": 0.3625, "rewards/reward_fn/std": 0.28575828671455383, "sampling/importance_sampling_ratio/max": 1.285517978668213, "sampling/importance_sampling_ratio/mean": 1.0002331852912902, "sampling/importance_sampling_ratio/min": 0.7724617063999176, "sampling/sampling_logp_difference/max": 0.31488857269287107, "sampling/sampling_logp_difference/mean": 0.021045699529349805, "step": 10570 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.6, "completions/max_terminated_length": 315.6, "completions/mean_length": 222.5125, "completions/mean_terminated_length": 222.5125, "completions/min_length": 145.6, "completions/min_terminated_length": 145.6, "entropy": 1.1628859020769595, "epoch": 0.9726946768410407, "frac_reward_zero_std": 0.5, "grad_norm": 6.529232978820801, "kl": 0.05697717809816823, "learning_rate": 4.040803607071177e-06, "loss": 0.0027, "num_tokens": 46036321.0, "reward": 0.04375, "reward_std": 0.27085447013378144, "rewards/reward_fn/mean": 0.04375, "rewards/reward_fn/std": 0.2708544820547104, "sampling/importance_sampling_ratio/max": 1.2764482140541076, "sampling/importance_sampling_ratio/mean": 0.9996897161006928, "sampling/importance_sampling_ratio/min": 0.7757793724536896, "sampling/sampling_logp_difference/max": 0.28104729056358335, "sampling/sampling_logp_difference/mean": 0.02207276560366154, "step": 10580 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 325.6, "completions/max_terminated_length": 325.6, "completions/mean_length": 212.125, "completions/mean_terminated_length": 212.125, "completions/min_length": 132.8, "completions/min_terminated_length": 132.8, "entropy": 1.060155439376831, "epoch": 0.973614047991174, "frac_reward_zero_std": 0.5, "grad_norm": 0.14923733472824097, "kl": 0.03344500265084207, "learning_rate": 4.038807600562686e-06, "loss": 0.0018, "num_tokens": 46083755.0, "reward": 0.2875, "reward_std": 0.26074906885623933, "rewards/reward_fn/mean": 0.2875, "rewards/reward_fn/std": 0.2607490807771683, "sampling/importance_sampling_ratio/max": 1.2602178454399109, "sampling/importance_sampling_ratio/mean": 1.0003794193267823, "sampling/importance_sampling_ratio/min": 0.7772373974323272, "sampling/sampling_logp_difference/max": 0.2759662985801697, "sampling/sampling_logp_difference/mean": 0.021500770561397077, "step": 10590 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 306.0, "completions/max_terminated_length": 306.0, "completions/mean_length": 214.35, "completions/mean_terminated_length": 214.35, "completions/min_length": 138.0, "completions/min_terminated_length": 138.0, "entropy": 1.1174485310912132, "epoch": 0.9745334191413073, "frac_reward_zero_std": 0.2, "grad_norm": 0.09379025548696518, "kl": 0.05338641300913878, "learning_rate": 4.036810013575441e-06, "loss": 0.0023, "num_tokens": 46126295.0, "reward": 0.0625, "reward_std": 0.4027158319950104, "rewards/reward_fn/mean": 0.0625, "rewards/reward_fn/std": 0.4027158319950104, "sampling/importance_sampling_ratio/max": 1.2558752417564392, "sampling/importance_sampling_ratio/mean": 0.9998934984207153, "sampling/importance_sampling_ratio/min": 0.7756098926067352, "sampling/sampling_logp_difference/max": 0.27413382530212405, "sampling/sampling_logp_difference/mean": 0.021913588047027588, "step": 10600 }, { "epoch": 0.9745334191413073, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.002617801047120419, "eval_completions/max_length": 305.5776614310646, "eval_completions/max_terminated_length": 302.38568935427577, "eval_completions/mean_length": 210.04363001745202, "eval_completions/mean_terminated_length": 209.3403247879854, "eval_completions/min_length": 133.1937172774869, "eval_completions/min_terminated_length": 133.1937172774869, "eval_entropy": 1.1039680640422862, "eval_frac_reward_zero_std": 0.36300174520069806, "eval_kl": 0.0462461728773309, "eval_loss": 0.0024066148325800896, "eval_num_tokens": 46126295.0, "eval_reward": 0.0650087260034904, "eval_reward_std": 0.31234918594568395, "eval_rewards/reward_fn/mean": 0.0650087260034904, "eval_rewards/reward_fn/std": 0.31234919437147146, "eval_runtime": 1776.5787, "eval_samples_per_second": 0.323, "eval_sampling/importance_sampling_ratio/max": 1.2697018655390848, "eval_sampling/importance_sampling_ratio/mean": 1.0000249891172945, "eval_sampling/importance_sampling_ratio/min": 0.7778603796039368, "eval_sampling/sampling_logp_difference/max": 0.27872646494685666, "eval_sampling/sampling_logp_difference/mean": 0.02105851740256134, "eval_steps_per_second": 0.041, "step": 10600 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 272.0, "completions/max_terminated_length": 272.0, "completions/mean_length": 183.425, "completions/mean_terminated_length": 183.425, "completions/min_length": 110.0, "completions/min_terminated_length": 110.0, "entropy": 1.0763877734541893, "epoch": 0.9754527902914406, "frac_reward_zero_std": 0.5, "grad_norm": 8.570212364196777, "kl": 0.05944876414141618, "learning_rate": 4.034810848161121e-06, "loss": 0.0031, "num_tokens": 46167705.0, "reward": -0.025, "reward_std": 0.2945584118366241, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.2945584237575531, "sampling/importance_sampling_ratio/max": 1.2308587670326232, "sampling/importance_sampling_ratio/mean": 1.0003653705120086, "sampling/importance_sampling_ratio/min": 0.7976280927658081, "sampling/sampling_logp_difference/max": 0.25031859874725343, "sampling/sampling_logp_difference/mean": 0.020743202045559885, "step": 10610 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 276.8, "completions/max_terminated_length": 276.8, "completions/mean_length": 201.225, "completions/mean_terminated_length": 201.225, "completions/min_length": 132.4, "completions/min_terminated_length": 132.4, "entropy": 1.0714590944349767, "epoch": 0.976372161441574, "frac_reward_zero_std": 0.4, "grad_norm": 0.08955306559801102, "kl": 0.03883964841370471, "learning_rate": 4.03281010637303e-06, "loss": 0.002, "num_tokens": 46210043.0, "reward": -0.4125, "reward_std": 0.30646984577178954, "rewards/reward_fn/mean": -0.4125, "rewards/reward_fn/std": 0.3064698576927185, "sampling/importance_sampling_ratio/max": 1.2699307084083558, "sampling/importance_sampling_ratio/mean": 0.9997348785400391, "sampling/importance_sampling_ratio/min": 0.793196564912796, "sampling/sampling_logp_difference/max": 0.26364885568618773, "sampling/sampling_logp_difference/mean": 0.021123329736292364, "step": 10620 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.2, "completions/max_terminated_length": 286.6, "completions/mean_length": 210.3, "completions/mean_terminated_length": 206.93392944335938, "completions/min_length": 145.9, "completions/min_terminated_length": 145.9, "entropy": 1.1149389427155256, "epoch": 0.9772915325917073, "frac_reward_zero_std": 0.2, "grad_norm": 0.12780416011810303, "kl": 0.05383593825972639, "learning_rate": 4.030807790266089e-06, "loss": 0.0032, "num_tokens": 46254755.0, "reward": 0.14375, "reward_std": 0.4107950896024704, "rewards/reward_fn/mean": 0.14375, "rewards/reward_fn/std": 0.41079509556293486, "sampling/importance_sampling_ratio/max": 1.2807302713394164, "sampling/importance_sampling_ratio/mean": 1.0001554787158966, "sampling/importance_sampling_ratio/min": 0.7978473007678986, "sampling/sampling_logp_difference/max": 0.2565577030181885, "sampling/sampling_logp_difference/mean": 0.021979923360049725, "step": 10630 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 308.4, "completions/max_terminated_length": 291.7, "completions/mean_length": 224.325, "completions/mean_terminated_length": 221.30892944335938, "completions/min_length": 151.9, "completions/min_terminated_length": 151.9, "entropy": 1.1214619420468808, "epoch": 0.9782109037418406, "frac_reward_zero_std": 0.2, "grad_norm": 5.0095648765563965, "kl": 0.045149215031415224, "learning_rate": 4.028803901896839e-06, "loss": 0.0018, "num_tokens": 46299261.0, "reward": 0.10625, "reward_std": 0.3626292645931244, "rewards/reward_fn/mean": 0.10625, "rewards/reward_fn/std": 0.3626292645931244, "sampling/importance_sampling_ratio/max": 1.230244553089142, "sampling/importance_sampling_ratio/mean": 0.9998919427394867, "sampling/importance_sampling_ratio/min": 0.7904980719089508, "sampling/sampling_logp_difference/max": 0.23734110593795776, "sampling/sampling_logp_difference/mean": 0.021747793443500994, "step": 10640 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 253.1, "completions/max_terminated_length": 253.1, "completions/mean_length": 183.7875, "completions/mean_terminated_length": 183.7875, "completions/min_length": 116.1, "completions/min_terminated_length": 116.1, "entropy": 0.9508635349571705, "epoch": 0.9791302748919739, "frac_reward_zero_std": 0.7, "grad_norm": 0.13384227454662323, "kl": 0.037569528320454994, "learning_rate": 4.026798443323433e-06, "loss": 0.0018, "num_tokens": 46337140.0, "reward": 0.4625, "reward_std": 0.10606601536273956, "rewards/reward_fn/mean": 0.4625, "rewards/reward_fn/std": 0.10606601536273956, "sampling/importance_sampling_ratio/max": 1.2932259559631347, "sampling/importance_sampling_ratio/mean": 1.0002963900566102, "sampling/importance_sampling_ratio/min": 0.7995527446269989, "sampling/sampling_logp_difference/max": 0.26656681299209595, "sampling/sampling_logp_difference/mean": 0.019971209997311234, "step": 10650 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.025, "completions/max_length": 348.7, "completions/max_terminated_length": 321.9, "completions/mean_length": 230.3, "completions/mean_terminated_length": 224.51071472167968, "completions/min_length": 139.1, "completions/min_terminated_length": 139.1, "entropy": 1.124989303201437, "epoch": 0.9800496460421072, "frac_reward_zero_std": 0.3, "grad_norm": 5.871088027954102, "kl": 0.04560805626679212, "learning_rate": 4.024791416605638e-06, "loss": 0.002, "num_tokens": 46381876.0, "reward": 0.0, "reward_std": 0.3611987292766571, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.3611987292766571, "sampling/importance_sampling_ratio/max": 1.2567938923835755, "sampling/importance_sampling_ratio/mean": 0.9997237861156464, "sampling/importance_sampling_ratio/min": 0.7835067808628082, "sampling/sampling_logp_difference/max": 0.2627923250198364, "sampling/sampling_logp_difference/mean": 0.021575084142386913, "step": 10660 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 314.1, "completions/max_terminated_length": 314.1, "completions/mean_length": 231.225, "completions/mean_terminated_length": 231.225, "completions/min_length": 146.9, "completions/min_terminated_length": 146.9, "entropy": 1.0659235760569572, "epoch": 0.9809690171922405, "frac_reward_zero_std": 0.3, "grad_norm": 4.913619041442871, "kl": 0.04187723956420086, "learning_rate": 4.022782823804832e-06, "loss": 0.0018, "num_tokens": 46425430.0, "reward": -0.1, "reward_std": 0.3446744382381439, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.3446744501590729, "sampling/importance_sampling_ratio/max": 1.290179181098938, "sampling/importance_sampling_ratio/mean": 0.9999665856361389, "sampling/importance_sampling_ratio/min": 0.769069892168045, "sampling/sampling_logp_difference/max": 0.2794591188430786, "sampling/sampling_logp_difference/mean": 0.021413156390190126, "step": 10670 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 260.6, "completions/max_terminated_length": 260.6, "completions/mean_length": 192.0875, "completions/mean_terminated_length": 192.0875, "completions/min_length": 132.6, "completions/min_terminated_length": 132.6, "entropy": 1.0125107638537885, "epoch": 0.9818883883423738, "frac_reward_zero_std": 0.3, "grad_norm": 5.907232761383057, "kl": 0.03358312457276043, "learning_rate": 4.0207726669839984e-06, "loss": 0.0016, "num_tokens": 46472781.0, "reward": 0.075, "reward_std": 0.3282748609781265, "rewards/reward_fn/mean": 0.075, "rewards/reward_fn/std": 0.328274866938591, "sampling/importance_sampling_ratio/max": 1.2436286211013794, "sampling/importance_sampling_ratio/mean": 1.000118726491928, "sampling/importance_sampling_ratio/min": 0.7986629605293274, "sampling/sampling_logp_difference/max": 0.25396203994750977, "sampling/sampling_logp_difference/mean": 0.02032058909535408, "step": 10680 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 319.4, "completions/max_terminated_length": 319.4, "completions/mean_length": 224.35, "completions/mean_terminated_length": 224.35, "completions/min_length": 152.6, "completions/min_terminated_length": 152.6, "entropy": 1.0958833031356334, "epoch": 0.9828077594925071, "frac_reward_zero_std": 0.3, "grad_norm": 5.126735687255859, "kl": 0.048667197313625364, "learning_rate": 4.018760948207731e-06, "loss": 0.0023, "num_tokens": 46516737.0, "reward": 0.025, "reward_std": 0.3093269467353821, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.3093269467353821, "sampling/importance_sampling_ratio/max": 1.3144049048423767, "sampling/importance_sampling_ratio/mean": 1.0004535794258118, "sampling/importance_sampling_ratio/min": 0.7688483357429504, "sampling/sampling_logp_difference/max": 0.3154332756996155, "sampling/sampling_logp_difference/mean": 0.021407747454941273, "step": 10690 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 334.8, "completions/max_terminated_length": 334.8, "completions/mean_length": 227.925, "completions/mean_terminated_length": 227.925, "completions/min_length": 152.1, "completions/min_terminated_length": 152.1, "entropy": 1.0676656879484654, "epoch": 0.9837271306426404, "frac_reward_zero_std": 0.2, "grad_norm": 0.12427646666765213, "kl": 0.04723160498542711, "learning_rate": 4.016747669542227e-06, "loss": 0.0022, "num_tokens": 46562275.0, "reward": 0.3625, "reward_std": 0.38434176445007323, "rewards/reward_fn/mean": 0.3625, "rewards/reward_fn/std": 0.3843417763710022, "sampling/importance_sampling_ratio/max": 1.2663423061370849, "sampling/importance_sampling_ratio/mean": 1.000032937526703, "sampling/importance_sampling_ratio/min": 0.7741970658302307, "sampling/sampling_logp_difference/max": 0.28989644050598146, "sampling/sampling_logp_difference/mean": 0.020921942219138144, "step": 10700 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0125, "completions/max_length": 329.0, "completions/max_terminated_length": 318.2, "completions/mean_length": 229.5875, "completions/mean_terminated_length": 226.76607360839844, "completions/min_length": 144.0, "completions/min_terminated_length": 144.0, "entropy": 1.0542023926973343, "epoch": 0.9846465017927737, "frac_reward_zero_std": 0.3, "grad_norm": 6.42956018447876, "kl": 0.050429430173244326, "learning_rate": 4.014732833055284e-06, "loss": 0.0028, "num_tokens": 46606546.0, "reward": -0.14375, "reward_std": 0.3627240300178528, "rewards/reward_fn/mean": -0.14375, "rewards/reward_fn/std": 0.3627240478992462, "sampling/importance_sampling_ratio/max": 1.2930376052856445, "sampling/importance_sampling_ratio/mean": 0.9997085988521576, "sampling/importance_sampling_ratio/min": 0.770572566986084, "sampling/sampling_logp_difference/max": 0.29186494946479796, "sampling/sampling_logp_difference/mean": 0.020957361534237863, "step": 10710 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 252.4, "completions/max_terminated_length": 252.4, "completions/mean_length": 187.5375, "completions/mean_terminated_length": 187.5375, "completions/min_length": 124.0, "completions/min_terminated_length": 124.0, "entropy": 1.0119339033961297, "epoch": 0.9855658729429071, "frac_reward_zero_std": 0.5, "grad_norm": 0.12501323223114014, "kl": 0.03492223062203266, "learning_rate": 4.0127164408163e-06, "loss": 0.0017, "num_tokens": 46645541.0, "reward": 0.2125, "reward_std": 0.2230676978826523, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.2230676978826523, "sampling/importance_sampling_ratio/max": 1.3230002403259278, "sampling/importance_sampling_ratio/mean": 1.0002218723297118, "sampling/importance_sampling_ratio/min": 0.756602555513382, "sampling/sampling_logp_difference/max": 0.328651225566864, "sampling/sampling_logp_difference/mean": 0.02056221794337034, "step": 10720 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 315.5, "completions/max_terminated_length": 315.5, "completions/mean_length": 217.125, "completions/mean_terminated_length": 217.125, "completions/min_length": 146.7, "completions/min_terminated_length": 146.7, "entropy": 1.0627942934632302, "epoch": 0.9864852440930404, "frac_reward_zero_std": 0.4, "grad_norm": 7.318994522094727, "kl": 0.06559816369554028, "learning_rate": 4.010698494896274e-06, "loss": 0.0031, "num_tokens": 46686871.0, "reward": 0.0125, "reward_std": 0.28489942848682404, "rewards/reward_fn/mean": 0.0125, "rewards/reward_fn/std": 0.284899440407753, "sampling/importance_sampling_ratio/max": 1.245354425907135, "sampling/importance_sampling_ratio/mean": 1.0001755833625794, "sampling/importance_sampling_ratio/min": 0.7547447919845581, "sampling/sampling_logp_difference/max": 0.290567022562027, "sampling/sampling_logp_difference/mean": 0.021006381325423717, "step": 10730 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 322.9, "completions/max_terminated_length": 322.9, "completions/mean_length": 226.175, "completions/mean_terminated_length": 226.175, "completions/min_length": 143.2, "completions/min_terminated_length": 143.2, "entropy": 1.1587119542062283, "epoch": 0.9874046152431737, "frac_reward_zero_std": 0.4, "grad_norm": 4.134674549102783, "kl": 0.03835391944739967, "learning_rate": 4.008678997367795e-06, "loss": 0.002, "num_tokens": 46733165.0, "reward": 0.1875, "reward_std": 0.31933356523513795, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.31933358311653137, "sampling/importance_sampling_ratio/max": 1.2341816663742065, "sampling/importance_sampling_ratio/mean": 0.9999795913696289, "sampling/importance_sampling_ratio/min": 0.7795907139778138, "sampling/sampling_logp_difference/max": 0.2498707354068756, "sampling/sampling_logp_difference/mean": 0.021864561550319194, "step": 10740 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 277.7, "completions/max_terminated_length": 277.7, "completions/mean_length": 194.8875, "completions/mean_terminated_length": 194.8875, "completions/min_length": 133.3, "completions/min_terminated_length": 133.3, "entropy": 1.0575469985604287, "epoch": 0.988323986393307, "frac_reward_zero_std": 0.4, "grad_norm": 6.4292192459106445, "kl": 0.04080540960421786, "learning_rate": 4.006657950305051e-06, "loss": 0.0021, "num_tokens": 46775636.0, "reward": 0.1125, "reward_std": 0.26819893419742585, "rewards/reward_fn/mean": 0.1125, "rewards/reward_fn/std": 0.2681989401578903, "sampling/importance_sampling_ratio/max": 1.3108070492744446, "sampling/importance_sampling_ratio/mean": 0.9998845219612121, "sampling/importance_sampling_ratio/min": 0.8049184024333954, "sampling/sampling_logp_difference/max": 0.2715925097465515, "sampling/sampling_logp_difference/mean": 0.02117980606853962, "step": 10750 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 289.4, "completions/max_terminated_length": 289.4, "completions/mean_length": 206.5, "completions/mean_terminated_length": 206.5, "completions/min_length": 131.6, "completions/min_terminated_length": 131.6, "entropy": 1.0935260951519012, "epoch": 0.9892433575434403, "frac_reward_zero_std": 0.4, "grad_norm": 5.3015055656433105, "kl": 0.05091552591184154, "learning_rate": 4.0046353557838165e-06, "loss": 0.0026, "num_tokens": 46821516.0, "reward": 0.325, "reward_std": 0.2958350956439972, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.29583510756492615, "sampling/importance_sampling_ratio/max": 1.2403937816619872, "sampling/importance_sampling_ratio/mean": 0.9999417185783386, "sampling/importance_sampling_ratio/min": 0.8000710666179657, "sampling/sampling_logp_difference/max": 0.23776893615722655, "sampling/sampling_logp_difference/mean": 0.020979249477386476, "step": 10760 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 313.6, "completions/max_terminated_length": 313.6, "completions/mean_length": 220.6625, "completions/mean_terminated_length": 220.6625, "completions/min_length": 143.5, "completions/min_terminated_length": 143.5, "entropy": 1.0781245514750482, "epoch": 0.9901627286935736, "frac_reward_zero_std": 0.3, "grad_norm": 5.216675281524658, "kl": 0.04917086389614269, "learning_rate": 4.0026112158814624e-06, "loss": 0.0029, "num_tokens": 46864977.0, "reward": 0.2125, "reward_std": 0.35166861712932584, "rewards/reward_fn/mean": 0.2125, "rewards/reward_fn/std": 0.3516686350107193, "sampling/importance_sampling_ratio/max": 1.2704369664192199, "sampling/importance_sampling_ratio/mean": 1.0001218438148498, "sampling/importance_sampling_ratio/min": 0.8047501444816589, "sampling/sampling_logp_difference/max": 0.25602006912231445, "sampling/sampling_logp_difference/mean": 0.021384000964462756, "step": 10770 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 303.4, "completions/max_terminated_length": 303.4, "completions/mean_length": 212.075, "completions/mean_terminated_length": 212.075, "completions/min_length": 154.8, "completions/min_terminated_length": 154.8, "entropy": 1.1556730784475804, "epoch": 0.9910820998437069, "frac_reward_zero_std": 0.1, "grad_norm": 6.765624523162842, "kl": 0.04916229965747334, "learning_rate": 4.00058553267694e-06, "loss": 0.0022, "num_tokens": 46912055.0, "reward": 0.1875, "reward_std": 0.4377940118312836, "rewards/reward_fn/mean": 0.1875, "rewards/reward_fn/std": 0.43779402375221255, "sampling/importance_sampling_ratio/max": 1.2556164026260377, "sampling/importance_sampling_ratio/mean": 1.000164794921875, "sampling/importance_sampling_ratio/min": 0.7557412564754487, "sampling/sampling_logp_difference/max": 0.3009467363357544, "sampling/sampling_logp_difference/mean": 0.022604018449783325, "step": 10780 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 274.1, "completions/max_terminated_length": 274.1, "completions/mean_length": 192.0875, "completions/mean_terminated_length": 192.0875, "completions/min_length": 122.6, "completions/min_terminated_length": 122.6, "entropy": 1.0514204286038875, "epoch": 0.9920014709938402, "frac_reward_zero_std": 0.4, "grad_norm": 0.11940952390432358, "kl": 0.045541573059745134, "learning_rate": 3.99855830825079e-06, "loss": 0.0023, "num_tokens": 46954750.0, "reward": 0.025, "reward_std": 0.2575641840696335, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.25756419003009795, "sampling/importance_sampling_ratio/max": 1.2530922293663025, "sampling/importance_sampling_ratio/mean": 0.9998751640319824, "sampling/importance_sampling_ratio/min": 0.790794450044632, "sampling/sampling_logp_difference/max": 0.25152078866958616, "sampling/sampling_logp_difference/mean": 0.02078450843691826, "step": 10790 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 304.5, "completions/max_terminated_length": 304.5, "completions/mean_length": 204.9, "completions/mean_terminated_length": 204.9, "completions/min_length": 137.3, "completions/min_terminated_length": 137.3, "entropy": 1.0131013922393322, "epoch": 0.9929208421439735, "frac_reward_zero_std": 0.2, "grad_norm": 6.889503479003906, "kl": 0.04277862436138093, "learning_rate": 3.996529544685132e-06, "loss": 0.002, "num_tokens": 46994526.0, "reward": 0.025, "reward_std": 0.4935811996459961, "rewards/reward_fn/mean": 0.025, "rewards/reward_fn/std": 0.493581211566925, "sampling/importance_sampling_ratio/max": 1.267315721511841, "sampling/importance_sampling_ratio/mean": 0.9999509453773499, "sampling/importance_sampling_ratio/min": 0.7814526081085205, "sampling/sampling_logp_difference/max": 0.2762034058570862, "sampling/sampling_logp_difference/mean": 0.020524901524186134, "step": 10800 }, { "epoch": 0.9929208421439735, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.0013089005235602095, "eval_completions/max_length": 293.4694589877836, "eval_completions/max_terminated_length": 291.9528795811518, "eval_completions/mean_length": 203.68760907504364, "eval_completions/mean_terminated_length": 203.33972213239153, "eval_completions/min_length": 129.20942408376965, "eval_completions/min_terminated_length": 129.20942408376965, "eval_entropy": 1.1081976287235973, "eval_frac_reward_zero_std": 0.3368237347294939, "eval_kl": 0.03997780676007947, "eval_loss": 0.002060601022094488, "eval_num_tokens": 46994526.0, "eval_reward": 0.06479057591623037, "eval_reward_std": 0.33846851762052604, "eval_rewards/reward_fn/mean": 0.06479057591623037, "eval_rewards/reward_fn/std": 0.338468528334799, "eval_runtime": 1716.124, "eval_samples_per_second": 0.334, "eval_sampling/importance_sampling_ratio/max": 1.2655667663660766, "eval_sampling/importance_sampling_ratio/mean": 1.0000561010775142, "eval_sampling/importance_sampling_ratio/min": 0.7830570527916387, "eval_sampling/sampling_logp_difference/max": 0.2704578622733527, "eval_sampling/sampling_logp_difference/mean": 0.02117616586861758, "eval_steps_per_second": 0.042, "step": 10800 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 283.9, "completions/max_terminated_length": 283.9, "completions/mean_length": 203.9375, "completions/mean_terminated_length": 203.9375, "completions/min_length": 136.6, "completions/min_terminated_length": 136.6, "entropy": 0.9913013383746148, "epoch": 0.9938402132941069, "frac_reward_zero_std": 0.5, "grad_norm": 5.941771984100342, "kl": 0.03260976006858982, "learning_rate": 3.9944992440636705e-06, "loss": 0.0015, "num_tokens": 47035569.0, "reward": 0.4875, "reward_std": 0.2394672751426697, "rewards/reward_fn/mean": 0.4875, "rewards/reward_fn/std": 0.23946728110313414, "sampling/importance_sampling_ratio/max": 1.2419529557228088, "sampling/importance_sampling_ratio/mean": 0.9998937308788299, "sampling/importance_sampling_ratio/min": 0.7698903560638428, "sampling/sampling_logp_difference/max": 0.2705738663673401, "sampling/sampling_logp_difference/mean": 0.020100896432995796, "step": 10810 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 267.0, "completions/max_terminated_length": 267.0, "completions/mean_length": 188.2625, "completions/mean_terminated_length": 188.2625, "completions/min_length": 125.3, "completions/min_terminated_length": 125.3, "entropy": 1.075051011890173, "epoch": 0.9947595844442402, "frac_reward_zero_std": 0.6, "grad_norm": 7.5150017738342285, "kl": 0.033782070776214826, "learning_rate": 3.992467408471686e-06, "loss": 0.0016, "num_tokens": 47073838.0, "reward": 0.0, "reward_std": 0.22598327100276946, "rewards/reward_fn/mean": 0.0, "rewards/reward_fn/std": 0.22598327696323395, "sampling/importance_sampling_ratio/max": 1.2625146150588988, "sampling/importance_sampling_ratio/mean": 0.9999453842639923, "sampling/importance_sampling_ratio/min": 0.7921305418014526, "sampling/sampling_logp_difference/max": 0.258840012550354, "sampling/sampling_logp_difference/mean": 0.0216575151309371, "step": 10820 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 296.6, "completions/max_terminated_length": 296.6, "completions/mean_length": 197.6, "completions/mean_terminated_length": 197.6, "completions/min_length": 123.5, "completions/min_terminated_length": 123.5, "entropy": 1.1157053284347058, "epoch": 0.9956789555943735, "frac_reward_zero_std": 0.1, "grad_norm": 5.532589912414551, "kl": 0.0465478798490949, "learning_rate": 3.990434039996038e-06, "loss": 0.0017, "num_tokens": 47115718.0, "reward": -0.1, "reward_std": 0.4555900037288666, "rewards/reward_fn/mean": -0.1, "rewards/reward_fn/std": 0.4555900156497955, "sampling/importance_sampling_ratio/max": 1.266064465045929, "sampling/importance_sampling_ratio/mean": 0.9998319029808045, "sampling/importance_sampling_ratio/min": 0.7836755692958832, "sampling/sampling_logp_difference/max": 0.27566248178482056, "sampling/sampling_logp_difference/mean": 0.021598699688911437, "step": 10830 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 286.5, "completions/max_terminated_length": 286.5, "completions/mean_length": 203.6625, "completions/mean_terminated_length": 203.6625, "completions/min_length": 125.9, "completions/min_terminated_length": 125.9, "entropy": 1.0822812251746654, "epoch": 0.9965983267445068, "frac_reward_zero_std": 0.2, "grad_norm": 6.528209209442139, "kl": 0.04015933805494569, "learning_rate": 3.988399140725157e-06, "loss": 0.0025, "num_tokens": 47156683.0, "reward": -0.025, "reward_std": 0.5010453939437867, "rewards/reward_fn/mean": -0.025, "rewards/reward_fn/std": 0.50104541182518, "sampling/importance_sampling_ratio/max": 1.3065037608146668, "sampling/importance_sampling_ratio/mean": 0.999961006641388, "sampling/importance_sampling_ratio/min": 0.7814095377922058, "sampling/sampling_logp_difference/max": 0.29769723415374755, "sampling/sampling_logp_difference/mean": 0.02156674899160862, "step": 10840 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 295.0, "completions/max_terminated_length": 295.0, "completions/mean_length": 201.8375, "completions/mean_terminated_length": 201.8375, "completions/min_length": 133.7, "completions/min_terminated_length": 133.7, "entropy": 1.1300729371607303, "epoch": 0.9975176978946401, "frac_reward_zero_std": 0.3, "grad_norm": 7.41755485534668, "kl": 0.03942642385372892, "learning_rate": 3.98636271274905e-06, "loss": 0.0021, "num_tokens": 47202126.0, "reward": 0.325, "reward_std": 0.3095962584018707, "rewards/reward_fn/mean": 0.325, "rewards/reward_fn/std": 0.3095962584018707, "sampling/importance_sampling_ratio/max": 1.2735634922981263, "sampling/importance_sampling_ratio/mean": 1.000087285041809, "sampling/importance_sampling_ratio/min": 0.8021250605583191, "sampling/sampling_logp_difference/max": 0.25763792991638185, "sampling/sampling_logp_difference/mean": 0.02226375211030245, "step": 10850 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 302.2, "completions/max_terminated_length": 302.2, "completions/mean_length": 217.0875, "completions/mean_terminated_length": 217.0875, "completions/min_length": 138.9, "completions/min_terminated_length": 138.9, "entropy": 1.0599846214056015, "epoch": 0.9984370690447734, "frac_reward_zero_std": 0.5, "grad_norm": 0.12183108180761337, "kl": 0.03809760370641015, "learning_rate": 3.984324758159289e-06, "loss": 0.0015, "num_tokens": 47246061.0, "reward": -0.0875, "reward_std": 0.26106746792793273, "rewards/reward_fn/mean": -0.0875, "rewards/reward_fn/std": 0.26106746792793273, "sampling/importance_sampling_ratio/max": 1.3003975749015808, "sampling/importance_sampling_ratio/mean": 0.9999008655548096, "sampling/importance_sampling_ratio/min": 0.7912079811096191, "sampling/sampling_logp_difference/max": 0.2872904360294342, "sampling/sampling_logp_difference/mean": 0.02050208393484354, "step": 10860 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 316.1, "completions/max_terminated_length": 316.1, "completions/mean_length": 231.7875, "completions/mean_terminated_length": 231.7875, "completions/min_length": 152.4, "completions/min_terminated_length": 152.4, "entropy": 1.1013551726937294, "epoch": 0.9993564401949067, "frac_reward_zero_std": 0.3, "grad_norm": 5.377572059631348, "kl": 0.030392793647479266, "learning_rate": 3.98228527904902e-06, "loss": 0.0014, "num_tokens": 47293596.0, "reward": 0.05625, "reward_std": 0.43449294567108154, "rewards/reward_fn/mean": 0.05625, "rewards/reward_fn/std": 0.4344929575920105, "sampling/importance_sampling_ratio/max": 1.2969264030456542, "sampling/importance_sampling_ratio/mean": 0.9999637007713318, "sampling/importance_sampling_ratio/min": 0.7700069010257721, "sampling/sampling_logp_difference/max": 0.29123281836509707, "sampling/sampling_logp_difference/mean": 0.0211251026019454, "step": 10870 } ], "logging_steps": 10, "max_steps": 32631, "num_input_tokens_seen": 47321224, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }