{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 6.666666666666667, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8625, "completions/max_length": 379.3, "completions/max_terminated_length": 57.8, "completions/mean_length": 362.7625, "completions/mean_terminated_length": 46.4125, "completions/min_length": 318.1, "completions/min_terminated_length": 38.1, "entropy": 0.5937434114515782, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.8, "grad_norm": 0.0, "learning_rate": 1.9280000000000002e-05, "loss": -0.017474760115146638, "num_tokens": 61847.0, "reward": -0.0987912505865097, "reward_std": 0.9303328216075897, "rewards/reward_fn/mean": -0.0987912505865097, "rewards/reward_fn/std": 0.9303328156471252, "step": 10, "step_time": 31.724731107300066 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 400.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 400.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 400.0, "completions/min_terminated_length": 0.0, "entropy": 0.6593769532628357, "epoch": 0.6666666666666666, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1.8480000000000003e-05, "loss": 0.0, "num_tokens": 126843.0, "reward": 0.048249998688697816, "reward_std": 0.8191121600568294, "rewards/reward_fn/mean": 0.048249998688697816, "rewards/reward_fn/std": 0.8191121838986873, "step": 20, "step_time": 33.554111711299946 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.975, "completions/max_length": 400.0, "completions/max_terminated_length": 49.7, "completions/mean_length": 396.2125, "completions/mean_terminated_length": 49.7, "completions/min_length": 369.7, "completions/min_terminated_length": 49.7, "entropy": 0.5578590292367153, "epoch": 1.0, "frac_reward_zero_std": 0.9, "grad_norm": 1.8046875, "learning_rate": 1.768e-05, "loss": -0.005067326501011849, "num_tokens": 191538.0, "reward": 0.11049998998641967, "reward_std": 0.711869715154171, "rewards/reward_fn/mean": 0.11049998998641967, "rewards/reward_fn/std": 0.7118697345256806, "step": 30, "step_time": 33.49379607419994 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 400.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 400.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 400.0, "completions/min_terminated_length": 0.0, "entropy": 0.004339599603554234, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.95, "grad_norm": 0.0, "learning_rate": 1.688e-05, "loss": 0.0, "num_tokens": 256536.0, "reward": 0.07887499779462814, "reward_std": 0.8938172444701195, "rewards/reward_fn/mean": 0.07887499779462814, "rewards/reward_fn/std": 0.8938172534108162, "step": 40, "step_time": 33.4393088197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9875, "completions/max_length": 400.0, "completions/max_terminated_length": 0.9, "completions/mean_length": 395.1125, "completions/mean_terminated_length": 0.9, "completions/min_length": 360.9, "completions/min_terminated_length": 0.9, "entropy": 0.004430754791246727, "epoch": 1.6666666666666665, "frac_reward_zero_std": 0.975, "grad_norm": 0.384765625, "learning_rate": 1.6080000000000002e-05, "loss": 0.0, "num_tokens": 321071.0, "reward": -0.16437500417232515, "reward_std": 1.063568216562271, "rewards/reward_fn/mean": -0.16437500417232515, "rewards/reward_fn/std": 1.0635682106018067, "step": 50, "step_time": 33.5646041849001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9625, "completions/max_length": 400.0, "completions/max_terminated_length": 3.7, "completions/mean_length": 385.4625, "completions/mean_terminated_length": 3.7, "completions/min_length": 283.7, "completions/min_terminated_length": 3.7, "entropy": 0.09348616125644185, "epoch": 2.0, "frac_reward_zero_std": 0.925, "grad_norm": 0.0, "learning_rate": 1.5280000000000003e-05, "loss": 0.009755914658308029, "num_tokens": 384804.0, "reward": -0.16562500596046448, "reward_std": 0.9360396310687065, "rewards/reward_fn/mean": -0.16562500596046448, "rewards/reward_fn/std": 0.9360396608710289, "step": 60, "step_time": 33.48749011520003 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.8625, "completions/max_length": 400.0, "completions/max_terminated_length": 44.3, "completions/mean_length": 351.7625, "completions/mean_terminated_length": 42.26666679382324, "completions/min_length": 159.8, "completions/min_terminated_length": 39.8, "entropy": 0.6013902719132602, "epoch": 2.3333333333333335, "frac_reward_zero_std": 0.925, "grad_norm": 2.40625, "learning_rate": 1.448e-05, "loss": -0.011668374389410019, "num_tokens": 445825.0, "reward": 0.08912500143051147, "reward_std": 0.9901719689369202, "rewards/reward_fn/mean": 0.08912500143051147, "rewards/reward_fn/std": 0.9901719927787781, "step": 70, "step_time": 33.48351585430014 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9, "completions/max_length": 400.0, "completions/max_terminated_length": 36.8, "completions/mean_length": 365.4875, "completions/mean_terminated_length": 34.75, "completions/min_length": 192.7, "completions/min_terminated_length": 32.7, "entropy": 0.7647990029305219, "epoch": 2.6666666666666665, "frac_reward_zero_std": 0.975, "grad_norm": 0.0, "learning_rate": 1.3680000000000003e-05, "loss": 0.0, "num_tokens": 507944.0, "reward": -0.14375001043081284, "reward_std": 0.6796593680977822, "rewards/reward_fn/mean": -0.14375001043081284, "rewards/reward_fn/std": 0.6796593815088272, "step": 80, "step_time": 33.443731949500034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.95, "completions/max_length": 400.0, "completions/max_terminated_length": 38.7, "completions/mean_length": 386.45, "completions/mean_terminated_length": 35.4, "completions/min_length": 312.1, "completions/min_terminated_length": 32.1, "entropy": 0.817728553712368, "epoch": 3.0, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1.2880000000000002e-05, "loss": 0.0, "num_tokens": 571920.0, "reward": -0.09100000560283661, "reward_std": 0.7491292104125022, "rewards/reward_fn/mean": -0.09100000560283661, "rewards/reward_fn/std": 0.7491292074322701, "step": 90, "step_time": 33.4239154713001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.95, "completions/max_length": 400.0, "completions/max_terminated_length": 52.1, "completions/mean_length": 386.5125, "completions/mean_terminated_length": 52.1, "completions/min_length": 292.1, "completions/min_terminated_length": 52.1, "entropy": 0.7871452756226063, "epoch": 3.3333333333333335, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1.2080000000000001e-05, "loss": 0.0, "num_tokens": 635831.0, "reward": -0.11849999725818634, "reward_std": 0.9780218183994294, "rewards/reward_fn/mean": -0.11849999725818634, "rewards/reward_fn/std": 0.9780218482017518, "step": 100, "step_time": 33.44708331829988 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.95, "completions/max_length": 400.0, "completions/max_terminated_length": 75.3, "completions/mean_length": 389.775, "completions/mean_terminated_length": 65.95, "completions/min_length": 336.6, "completions/min_terminated_length": 56.6, "entropy": 0.8245016686618328, "epoch": 3.6666666666666665, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1.128e-05, "loss": 0.0, "num_tokens": 699917.0, "reward": 0.06824999898672104, "reward_std": 0.8597677208483219, "rewards/reward_fn/mean": 0.06824999898672104, "rewards/reward_fn/std": 0.8597677327692509, "step": 110, "step_time": 33.43369797569976 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9875, "completions/max_length": 400.0, "completions/max_terminated_length": 29.0, "completions/mean_length": 398.625, "completions/mean_terminated_length": 29.0, "completions/min_length": 389.0, "completions/min_terminated_length": 29.0, "entropy": 0.8096485503017903, "epoch": 4.0, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 1.0480000000000001e-05, "loss": 0.0, "num_tokens": 764733.0, "reward": -0.05875001698732376, "reward_std": 0.7073742881417274, "rewards/reward_fn/mean": -0.05875001698732376, "rewards/reward_fn/std": 0.7073742985725403, "step": 120, "step_time": 33.47654410239984 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9625, "completions/max_length": 400.0, "completions/max_terminated_length": 51.8, "completions/mean_length": 391.475, "completions/mean_terminated_length": 51.8, "completions/min_length": 331.8, "completions/min_terminated_length": 51.8, "entropy": 0.8178432904183864, "epoch": 4.333333333333333, "frac_reward_zero_std": 0.975, "grad_norm": 0.0, "learning_rate": 9.68e-06, "loss": 0.0, "num_tokens": 828855.0, "reward": -0.017000006139278413, "reward_std": 0.7560826383531094, "rewards/reward_fn/mean": -0.017000006139278413, "rewards/reward_fn/std": 0.7560826435685157, "step": 130, "step_time": 33.514995820599964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 400.0, "completions/max_terminated_length": 46.2, "completions/mean_length": 380.775, "completions/mean_terminated_length": 46.2, "completions/min_length": 246.2, "completions/min_terminated_length": 46.2, "entropy": 0.8854692216962576, "epoch": 4.666666666666667, "frac_reward_zero_std": 0.95, "grad_norm": 0.0, "learning_rate": 8.880000000000001e-06, "loss": -0.012354153394699096, "num_tokens": 892373.0, "reward": -0.13475000411272048, "reward_std": 0.9627669990062714, "rewards/reward_fn/mean": -0.13475000411272048, "rewards/reward_fn/std": 0.9627670347690582, "step": 140, "step_time": 33.50577180429955 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.975, "completions/max_length": 400.0, "completions/max_terminated_length": 32.3, "completions/mean_length": 394.275, "completions/mean_terminated_length": 17.1, "completions/min_length": 361.9, "completions/min_terminated_length": 1.9, "entropy": 0.8237141810357571, "epoch": 5.0, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 8.08e-06, "loss": 0.0, "num_tokens": 956875.0, "reward": -0.030000001192092896, "reward_std": 0.7297740295529366, "rewards/reward_fn/mean": -0.030000001192092896, "rewards/reward_fn/std": 0.7297740370035172, "step": 150, "step_time": 33.46883516300004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.975, "completions/max_length": 400.0, "completions/max_terminated_length": 38.5, "completions/mean_length": 394.8125, "completions/mean_terminated_length": 38.5, "completions/min_length": 358.5, "completions/min_terminated_length": 38.5, "entropy": 0.9815139673650265, "epoch": 5.333333333333333, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 7.280000000000001e-06, "loss": 0.0, "num_tokens": 1021402.0, "reward": -0.06250000596046448, "reward_std": 0.9604951858520507, "rewards/reward_fn/mean": -0.06250000596046448, "rewards/reward_fn/std": 0.9604952037334442, "step": 160, "step_time": 33.41488298210034 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.975, "completions/max_length": 400.0, "completions/max_terminated_length": 21.2, "completions/mean_length": 392.65, "completions/mean_terminated_length": 21.2, "completions/min_length": 341.2, "completions/min_terminated_length": 21.2, "entropy": 0.9070465706288815, "epoch": 5.666666666666667, "frac_reward_zero_std": 0.975, "grad_norm": 0.0, "learning_rate": 6.480000000000001e-06, "loss": 0.0, "num_tokens": 1085632.0, "reward": -0.1513125091791153, "reward_std": 0.855255764722824, "rewards/reward_fn/mean": -0.1513125091791153, "rewards/reward_fn/std": 0.8552557826042175, "step": 170, "step_time": 33.4777201513999 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 400.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 400.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 400.0, "completions/min_terminated_length": 0.0, "entropy": 0.9226562466472388, "epoch": 6.0, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5.68e-06, "loss": 0.0, "num_tokens": 1150692.0, "reward": 0.08200000673532486, "reward_std": 0.9346169531345367, "rewards/reward_fn/mean": 0.08200000673532486, "rewards/reward_fn/std": 0.9346169650554657, "step": 180, "step_time": 33.47372673979989 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.975, "completions/max_length": 400.0, "completions/max_terminated_length": 35.5, "completions/mean_length": 394.4375, "completions/mean_terminated_length": 35.5, "completions/min_length": 355.5, "completions/min_terminated_length": 35.5, "entropy": 0.9981254413723946, "epoch": 6.333333333333333, "frac_reward_zero_std": 0.975, "grad_norm": 0.0, "learning_rate": 4.880000000000001e-06, "loss": 0.0, "num_tokens": 1215133.0, "reward": -0.08775000870227814, "reward_std": 0.8883017361164093, "rewards/reward_fn/mean": -0.08775000870227814, "rewards/reward_fn/std": 0.8883017897605896, "step": 190, "step_time": 33.41868621859985 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.9875, "completions/max_length": 400.0, "completions/max_terminated_length": 30.4, "completions/mean_length": 398.8, "completions/mean_terminated_length": 30.4, "completions/min_length": 390.4, "completions/min_terminated_length": 30.4, "entropy": 0.9789559677243233, "epoch": 6.666666666666667, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 4.08e-06, "loss": 0.0, "num_tokens": 1279879.0, "reward": -0.03525000065565109, "reward_std": 0.8935358546674251, "rewards/reward_fn/mean": -0.03525000065565109, "rewards/reward_fn/std": 0.8935358554124833, "step": 200, "step_time": 33.424166100800086 } ], "logging_steps": 10, "max_steps": 250, "num_input_tokens_seen": 1279879, "num_train_epochs": 9, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }