3410 lines
120 KiB
JSON
3410 lines
120 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 1250,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 257.1,
|
|
"completions/max_terminated_length": 158.3,
|
|
"completions/mean_length": 42.925,
|
|
"completions/mean_terminated_length": 36.65708351135254,
|
|
"completions/min_length": 14.1,
|
|
"completions/min_terminated_length": 14.1,
|
|
"entropy": 0.7689311370253563,
|
|
"epoch": 0.008,
|
|
"frac_reward_zero_std": 0.35,
|
|
"grad_norm": 6.757325172424316,
|
|
"learning_rate": 4.964e-07,
|
|
"loss": -0.03516485691070557,
|
|
"num_tokens": 14928.0,
|
|
"reward": 0.7131250023841857,
|
|
"reward_std": 0.4031145960092545,
|
|
"rewards/reward_fn/mean": 0.7131250023841857,
|
|
"rewards/reward_fn/std": 0.40311461091041567,
|
|
"step": 10,
|
|
"step_time": 12.329081743443385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 124.0,
|
|
"completions/max_terminated_length": 124.0,
|
|
"completions/mean_length": 28.95,
|
|
"completions/mean_terminated_length": 28.95,
|
|
"completions/min_length": 15.3,
|
|
"completions/min_terminated_length": 15.3,
|
|
"entropy": 0.74221798684448,
|
|
"epoch": 0.016,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 24.46213722229004,
|
|
"learning_rate": 4.923999999999999e-07,
|
|
"loss": -0.03677875101566315,
|
|
"num_tokens": 27908.0,
|
|
"reward": 0.8331249713897705,
|
|
"reward_std": 0.4601421281695366,
|
|
"rewards/reward_fn/mean": 0.8331249713897705,
|
|
"rewards/reward_fn/std": 0.4601421505212784,
|
|
"step": 20,
|
|
"step_time": 6.713664122438058
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 312.5,
|
|
"completions/max_terminated_length": 312.5,
|
|
"completions/mean_length": 45.70625,
|
|
"completions/mean_terminated_length": 45.70625,
|
|
"completions/min_length": 15.9,
|
|
"completions/min_terminated_length": 15.9,
|
|
"entropy": 0.7767099749296904,
|
|
"epoch": 0.024,
|
|
"frac_reward_zero_std": 0.475,
|
|
"grad_norm": 8.362913131713867,
|
|
"learning_rate": 4.884e-07,
|
|
"loss": -0.06059606671333313,
|
|
"num_tokens": 43689.0,
|
|
"reward": 0.8893749833106994,
|
|
"reward_std": 0.3755300402641296,
|
|
"rewards/reward_fn/mean": 0.8893749833106994,
|
|
"rewards/reward_fn/std": 0.3755300521850586,
|
|
"step": 30,
|
|
"step_time": 14.446745052048936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 133.3,
|
|
"completions/max_terminated_length": 133.3,
|
|
"completions/mean_length": 29.8375,
|
|
"completions/mean_terminated_length": 29.8375,
|
|
"completions/min_length": 16.8,
|
|
"completions/min_terminated_length": 16.8,
|
|
"entropy": 0.7361419148743152,
|
|
"epoch": 0.032,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 12.163936614990234,
|
|
"learning_rate": 4.844e-07,
|
|
"loss": -0.03543267250061035,
|
|
"num_tokens": 56847.0,
|
|
"reward": 0.8531249761581421,
|
|
"reward_std": 0.35396517962217333,
|
|
"rewards/reward_fn/mean": 0.8531249761581421,
|
|
"rewards/reward_fn/std": 0.3539652034640312,
|
|
"step": 40,
|
|
"step_time": 7.049018428754062
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 108.2,
|
|
"completions/max_terminated_length": 108.2,
|
|
"completions/mean_length": 27.10625,
|
|
"completions/mean_terminated_length": 27.10625,
|
|
"completions/min_length": 16.7,
|
|
"completions/min_terminated_length": 16.7,
|
|
"entropy": 0.5995874460786581,
|
|
"epoch": 0.04,
|
|
"frac_reward_zero_std": 0.525,
|
|
"grad_norm": 8.617820739746094,
|
|
"learning_rate": 4.804e-07,
|
|
"loss": 0.062236183881759645,
|
|
"num_tokens": 69272.0,
|
|
"reward": 0.9143749713897705,
|
|
"reward_std": 0.35480276346206663,
|
|
"rewards/reward_fn/mean": 0.9143749713897705,
|
|
"rewards/reward_fn/std": 0.35480278730392456,
|
|
"step": 50,
|
|
"step_time": 6.082800254039467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 53.2,
|
|
"completions/max_terminated_length": 53.2,
|
|
"completions/mean_length": 23.50625,
|
|
"completions/mean_terminated_length": 23.50625,
|
|
"completions/min_length": 17.5,
|
|
"completions/min_terminated_length": 17.5,
|
|
"entropy": 0.6239847084507346,
|
|
"epoch": 0.048,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 8.013570785522461,
|
|
"learning_rate": 4.7639999999999995e-07,
|
|
"loss": -0.007780641317367554,
|
|
"num_tokens": 81493.0,
|
|
"reward": 0.8724999845027923,
|
|
"reward_std": 0.23893336951732635,
|
|
"rewards/reward_fn/mean": 0.8724999845027923,
|
|
"rewards/reward_fn/std": 0.23893338441848755,
|
|
"step": 60,
|
|
"step_time": 3.830377937294543
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 47.5,
|
|
"completions/max_terminated_length": 47.5,
|
|
"completions/mean_length": 21.3125,
|
|
"completions/mean_terminated_length": 21.3125,
|
|
"completions/min_length": 17.1,
|
|
"completions/min_terminated_length": 17.1,
|
|
"entropy": 0.526786202006042,
|
|
"epoch": 0.056,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 6.611401081085205,
|
|
"learning_rate": 4.7239999999999997e-07,
|
|
"loss": 0.0007052101194858551,
|
|
"num_tokens": 92903.0,
|
|
"reward": 0.954999977350235,
|
|
"reward_std": 0.30052519142627715,
|
|
"rewards/reward_fn/mean": 0.954999977350235,
|
|
"rewards/reward_fn/std": 0.30052521228790285,
|
|
"step": 70,
|
|
"step_time": 3.6160760662984104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 201.9,
|
|
"completions/max_terminated_length": 112.8,
|
|
"completions/mean_length": 34.225,
|
|
"completions/mean_terminated_length": 28.01541690826416,
|
|
"completions/min_length": 17.3,
|
|
"completions/min_terminated_length": 17.3,
|
|
"entropy": 0.6730542730540037,
|
|
"epoch": 0.064,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 8.44511890411377,
|
|
"learning_rate": 4.684e-07,
|
|
"loss": 0.19833827018737793,
|
|
"num_tokens": 106795.0,
|
|
"reward": 0.8831249833106994,
|
|
"reward_std": 0.2491457238793373,
|
|
"rewards/reward_fn/mean": 0.8831249833106994,
|
|
"rewards/reward_fn/std": 0.24914574027061462,
|
|
"step": 80,
|
|
"step_time": 10.088793818978592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 171.4,
|
|
"completions/max_terminated_length": 171.4,
|
|
"completions/mean_length": 30.21875,
|
|
"completions/mean_terminated_length": 30.21875,
|
|
"completions/min_length": 17.7,
|
|
"completions/min_terminated_length": 17.7,
|
|
"entropy": 0.5667739075608551,
|
|
"epoch": 0.072,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.6439999999999995e-07,
|
|
"loss": 0.002179677784442902,
|
|
"num_tokens": 119930.0,
|
|
"reward": 0.9687499701976776,
|
|
"reward_std": 0.2984331727027893,
|
|
"rewards/reward_fn/mean": 0.9687499701976776,
|
|
"rewards/reward_fn/std": 0.2984331905841827,
|
|
"step": 90,
|
|
"step_time": 8.863318855362014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 74.8,
|
|
"completions/max_terminated_length": 74.8,
|
|
"completions/mean_length": 23.96875,
|
|
"completions/mean_terminated_length": 23.96875,
|
|
"completions/min_length": 17.8,
|
|
"completions/min_terminated_length": 17.8,
|
|
"entropy": 0.494637239864096,
|
|
"epoch": 0.08,
|
|
"frac_reward_zero_std": 0.925,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.6039999999999997e-07,
|
|
"loss": 0.001673000119626522,
|
|
"num_tokens": 131773.0,
|
|
"reward": 0.971874988079071,
|
|
"reward_std": 0.1934887498617172,
|
|
"rewards/reward_fn/mean": 0.971874988079071,
|
|
"rewards/reward_fn/std": 0.19348875880241395,
|
|
"step": 100,
|
|
"step_time": 4.718112504808232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 46.2,
|
|
"completions/max_terminated_length": 46.2,
|
|
"completions/mean_length": 21.71875,
|
|
"completions/mean_terminated_length": 21.71875,
|
|
"completions/min_length": 17.2,
|
|
"completions/min_terminated_length": 17.2,
|
|
"entropy": 0.44686332195997236,
|
|
"epoch": 0.088,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.5639999999999993e-07,
|
|
"loss": 0.007064198702573776,
|
|
"num_tokens": 143140.0,
|
|
"reward": 0.9249999701976777,
|
|
"reward_std": 0.2545803815126419,
|
|
"rewards/reward_fn/mean": 0.9249999701976777,
|
|
"rewards/reward_fn/std": 0.2545804023742676,
|
|
"step": 110,
|
|
"step_time": 3.6042728299740703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 25.2,
|
|
"completions/max_terminated_length": 25.2,
|
|
"completions/mean_length": 19.40625,
|
|
"completions/mean_terminated_length": 19.40625,
|
|
"completions/min_length": 17.1,
|
|
"completions/min_terminated_length": 17.1,
|
|
"entropy": 0.3366036908584647,
|
|
"epoch": 0.096,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 13.464982986450195,
|
|
"learning_rate": 4.524e-07,
|
|
"loss": -0.0006624836474657059,
|
|
"num_tokens": 154305.0,
|
|
"reward": 1.0399999618530273,
|
|
"reward_std": 0.34495194256305695,
|
|
"rewards/reward_fn/mean": 1.0399999618530273,
|
|
"rewards/reward_fn/std": 0.34495197534561156,
|
|
"step": 120,
|
|
"step_time": 2.747153246589005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 29.3,
|
|
"completions/max_terminated_length": 29.3,
|
|
"completions/mean_length": 19.65625,
|
|
"completions/mean_terminated_length": 19.65625,
|
|
"completions/min_length": 17.2,
|
|
"completions/min_terminated_length": 17.2,
|
|
"entropy": 0.36671050266595556,
|
|
"epoch": 0.104,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.484e-07,
|
|
"loss": 0.017103588581085204,
|
|
"num_tokens": 165738.0,
|
|
"reward": 0.9574999690055848,
|
|
"reward_std": 0.3057817339897156,
|
|
"rewards/reward_fn/mean": 0.9574999690055848,
|
|
"rewards/reward_fn/std": 0.305781751871109,
|
|
"step": 130,
|
|
"step_time": 2.903817686345428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 31.8,
|
|
"completions/max_terminated_length": 31.8,
|
|
"completions/mean_length": 19.84375,
|
|
"completions/mean_terminated_length": 19.84375,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.35206709057092667,
|
|
"epoch": 0.112,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.444e-07,
|
|
"loss": 0.0023837201297283173,
|
|
"num_tokens": 177529.0,
|
|
"reward": 1.003124976158142,
|
|
"reward_std": 0.27038749754428865,
|
|
"rewards/reward_fn/mean": 1.003124976158142,
|
|
"rewards/reward_fn/std": 0.27038750648498533,
|
|
"step": 140,
|
|
"step_time": 2.9909598857630044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 33.7,
|
|
"completions/max_terminated_length": 33.7,
|
|
"completions/mean_length": 20.18125,
|
|
"completions/mean_terminated_length": 20.18125,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.34009722527116537,
|
|
"epoch": 0.12,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 4.034805774688721,
|
|
"learning_rate": 4.404e-07,
|
|
"loss": 0.005604463815689087,
|
|
"num_tokens": 188954.0,
|
|
"reward": 0.9499999761581421,
|
|
"reward_std": 0.25686181485652926,
|
|
"rewards/reward_fn/mean": 0.9499999761581421,
|
|
"rewards/reward_fn/std": 0.25686182677745817,
|
|
"step": 150,
|
|
"step_time": 3.0971988524775953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 64.1,
|
|
"completions/max_terminated_length": 64.1,
|
|
"completions/mean_length": 22.725,
|
|
"completions/mean_terminated_length": 22.725,
|
|
"completions/min_length": 17.1,
|
|
"completions/min_terminated_length": 17.1,
|
|
"entropy": 0.3341391346533783,
|
|
"epoch": 0.128,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 3.904665946960449,
|
|
"learning_rate": 4.364e-07,
|
|
"loss": 0.0010127602145075798,
|
|
"num_tokens": 200770.0,
|
|
"reward": 1.0562499642372132,
|
|
"reward_std": 0.3720459073781967,
|
|
"rewards/reward_fn/mean": 1.0562499642372132,
|
|
"rewards/reward_fn/std": 0.37204593122005464,
|
|
"step": 160,
|
|
"step_time": 4.2844222981948406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 60.1,
|
|
"completions/max_terminated_length": 60.1,
|
|
"completions/mean_length": 21.50625,
|
|
"completions/mean_terminated_length": 21.50625,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.3025246943347156,
|
|
"epoch": 0.136,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 5.239154815673828,
|
|
"learning_rate": 4.324e-07,
|
|
"loss": -0.0667100191116333,
|
|
"num_tokens": 212767.0,
|
|
"reward": 0.9624999642372132,
|
|
"reward_std": 0.2936569094657898,
|
|
"rewards/reward_fn/mean": 0.9624999642372132,
|
|
"rewards/reward_fn/std": 0.2936569362878799,
|
|
"step": 170,
|
|
"step_time": 4.151584721263498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 189.7,
|
|
"completions/max_terminated_length": 189.7,
|
|
"completions/mean_length": 32.33125,
|
|
"completions/mean_terminated_length": 32.33125,
|
|
"completions/min_length": 17.6,
|
|
"completions/min_terminated_length": 17.6,
|
|
"entropy": 0.38091158121824265,
|
|
"epoch": 0.144,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 8.872406005859375,
|
|
"learning_rate": 4.284e-07,
|
|
"loss": -0.008929825574159621,
|
|
"num_tokens": 225856.0,
|
|
"reward": 1.0249999642372132,
|
|
"reward_std": 0.3340115398168564,
|
|
"rewards/reward_fn/mean": 1.0249999642372132,
|
|
"rewards/reward_fn/std": 0.33401156663894654,
|
|
"step": 180,
|
|
"step_time": 9.366582131106407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 355.7,
|
|
"completions/max_terminated_length": 355.7,
|
|
"completions/mean_length": 68.725,
|
|
"completions/mean_terminated_length": 68.725,
|
|
"completions/min_length": 17.5,
|
|
"completions/min_terminated_length": 17.5,
|
|
"entropy": 0.5331769159063697,
|
|
"epoch": 0.152,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.2439999999999996e-07,
|
|
"loss": 0.05694311261177063,
|
|
"num_tokens": 245108.0,
|
|
"reward": 0.9968749701976776,
|
|
"reward_std": 0.30177369713783264,
|
|
"rewards/reward_fn/mean": 0.9968749701976776,
|
|
"rewards/reward_fn/std": 0.3017737179994583,
|
|
"step": 190,
|
|
"step_time": 16.25606108647771
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 231.7,
|
|
"completions/max_terminated_length": 231.7,
|
|
"completions/mean_length": 39.475,
|
|
"completions/mean_terminated_length": 39.475,
|
|
"completions/min_length": 17.6,
|
|
"completions/min_terminated_length": 17.6,
|
|
"entropy": 0.32137000167858787,
|
|
"epoch": 0.16,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 6.173330307006836,
|
|
"learning_rate": 4.204e-07,
|
|
"loss": -0.10696818828582763,
|
|
"num_tokens": 259512.0,
|
|
"reward": 1.012499952316284,
|
|
"reward_std": 0.3910213738679886,
|
|
"rewards/reward_fn/mean": 1.012499952316284,
|
|
"rewards/reward_fn/std": 0.39102140367031096,
|
|
"step": 200,
|
|
"step_time": 11.044270927784964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.0,
|
|
"completions/max_terminated_length": 314.0,
|
|
"completions/mean_length": 40.30625,
|
|
"completions/mean_terminated_length": 40.30625,
|
|
"completions/min_length": 17.1,
|
|
"completions/min_terminated_length": 17.1,
|
|
"entropy": 0.3382976199500263,
|
|
"epoch": 0.168,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 12.670361518859863,
|
|
"learning_rate": 4.164e-07,
|
|
"loss": 0.05056280493736267,
|
|
"num_tokens": 274317.0,
|
|
"reward": 1.0124999582767487,
|
|
"reward_std": 0.37820068299770354,
|
|
"rewards/reward_fn/mean": 1.0124999582767487,
|
|
"rewards/reward_fn/std": 0.37820071876049044,
|
|
"step": 210,
|
|
"step_time": 14.444551136810333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 94.8,
|
|
"completions/max_terminated_length": 94.8,
|
|
"completions/mean_length": 24.9375,
|
|
"completions/mean_terminated_length": 24.9375,
|
|
"completions/min_length": 17.5,
|
|
"completions/min_terminated_length": 17.5,
|
|
"entropy": 0.2844417320564389,
|
|
"epoch": 0.176,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 11.695199966430664,
|
|
"learning_rate": 4.1239999999999996e-07,
|
|
"loss": 0.012198887765407562,
|
|
"num_tokens": 286943.0,
|
|
"reward": 0.9874999761581421,
|
|
"reward_std": 0.2448488086462021,
|
|
"rewards/reward_fn/mean": 0.9874999761581421,
|
|
"rewards/reward_fn/std": 0.24484881460666658,
|
|
"step": 220,
|
|
"step_time": 5.559161439398304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 103.9,
|
|
"completions/max_terminated_length": 103.9,
|
|
"completions/mean_length": 29.1125,
|
|
"completions/mean_terminated_length": 29.1125,
|
|
"completions/min_length": 16.6,
|
|
"completions/min_terminated_length": 16.6,
|
|
"entropy": 0.31354843971785157,
|
|
"epoch": 0.184,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 7.898475646972656,
|
|
"learning_rate": 4.084e-07,
|
|
"loss": -0.01266075372695923,
|
|
"num_tokens": 299829.0,
|
|
"reward": 0.9837499558925629,
|
|
"reward_std": 0.35773794949054716,
|
|
"rewards/reward_fn/mean": 0.9837499558925629,
|
|
"rewards/reward_fn/std": 0.3577379733324051,
|
|
"step": 230,
|
|
"step_time": 5.904518230678514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 70.4,
|
|
"completions/max_terminated_length": 70.4,
|
|
"completions/mean_length": 24.5625,
|
|
"completions/mean_terminated_length": 24.5625,
|
|
"completions/min_length": 17.6,
|
|
"completions/min_terminated_length": 17.6,
|
|
"entropy": 0.24243622907670215,
|
|
"epoch": 0.192,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.0439999999999994e-07,
|
|
"loss": -0.022087261080741882,
|
|
"num_tokens": 312139.0,
|
|
"reward": 1.081249964237213,
|
|
"reward_std": 0.3463846206665039,
|
|
"rewards/reward_fn/mean": 1.081249964237213,
|
|
"rewards/reward_fn/std": 0.3463846266269684,
|
|
"step": 240,
|
|
"step_time": 4.509387341840193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 85.0,
|
|
"completions/max_terminated_length": 85.0,
|
|
"completions/mean_length": 30.96875,
|
|
"completions/mean_terminated_length": 30.96875,
|
|
"completions/min_length": 17.7,
|
|
"completions/min_terminated_length": 17.7,
|
|
"entropy": 0.2872183081228286,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.0039999999999996e-07,
|
|
"loss": 0.0037321031093597414,
|
|
"num_tokens": 325758.0,
|
|
"reward": 0.9562499642372131,
|
|
"reward_std": 0.31391805708408355,
|
|
"rewards/reward_fn/mean": 0.9562499642372131,
|
|
"rewards/reward_fn/std": 0.3139180839061737,
|
|
"step": 250,
|
|
"step_time": 5.160763737838716
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 127.1,
|
|
"completions/max_terminated_length": 127.1,
|
|
"completions/mean_length": 29.21875,
|
|
"completions/mean_terminated_length": 29.21875,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.23362355224089698,
|
|
"epoch": 0.208,
|
|
"frac_reward_zero_std": 0.85,
|
|
"grad_norm": 6.962454795837402,
|
|
"learning_rate": 3.964e-07,
|
|
"loss": 0.028215166926383973,
|
|
"num_tokens": 338481.0,
|
|
"reward": 1.0999999582767486,
|
|
"reward_std": 0.42081114947795867,
|
|
"rewards/reward_fn/mean": 1.0999999582767486,
|
|
"rewards/reward_fn/std": 0.4208111733198166,
|
|
"step": 260,
|
|
"step_time": 6.7953305871225895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 116.8,
|
|
"completions/max_terminated_length": 116.8,
|
|
"completions/mean_length": 30.49375,
|
|
"completions/mean_terminated_length": 30.49375,
|
|
"completions/min_length": 17.5,
|
|
"completions/min_terminated_length": 17.5,
|
|
"entropy": 0.2725491218268871,
|
|
"epoch": 0.216,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 5.130438327789307,
|
|
"learning_rate": 3.924e-07,
|
|
"loss": -0.05285842418670654,
|
|
"num_tokens": 351340.0,
|
|
"reward": 0.9593749642372131,
|
|
"reward_std": 0.30626748204231263,
|
|
"rewards/reward_fn/mean": 0.9593749642372131,
|
|
"rewards/reward_fn/std": 0.3062675029039383,
|
|
"step": 270,
|
|
"step_time": 6.392075706832111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 231.2,
|
|
"completions/max_terminated_length": 231.2,
|
|
"completions/mean_length": 38.7625,
|
|
"completions/mean_terminated_length": 38.7625,
|
|
"completions/min_length": 17.2,
|
|
"completions/min_terminated_length": 17.2,
|
|
"entropy": 0.28721734539140015,
|
|
"epoch": 0.224,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.884e-07,
|
|
"loss": -0.011222265660762787,
|
|
"num_tokens": 365774.0,
|
|
"reward": 0.9218749761581421,
|
|
"reward_std": 0.25241841971874235,
|
|
"rewards/reward_fn/mean": 0.9218749761581421,
|
|
"rewards/reward_fn/std": 0.2524184435606003,
|
|
"step": 280,
|
|
"step_time": 11.114369830535725
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.9,
|
|
"completions/max_terminated_length": 255.9,
|
|
"completions/mean_length": 41.78125,
|
|
"completions/mean_terminated_length": 41.78125,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.28078931191121226,
|
|
"epoch": 0.232,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.8440000000000003e-07,
|
|
"loss": -0.07034780383110047,
|
|
"num_tokens": 380923.0,
|
|
"reward": 0.9999999582767487,
|
|
"reward_std": 0.35158316493034364,
|
|
"rewards/reward_fn/mean": 0.9999999582767487,
|
|
"rewards/reward_fn/std": 0.3515831857919693,
|
|
"step": 290,
|
|
"step_time": 11.978949176566676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.5,
|
|
"completions/max_terminated_length": 331.5,
|
|
"completions/mean_length": 78.25625,
|
|
"completions/mean_terminated_length": 78.25625,
|
|
"completions/min_length": 17.4,
|
|
"completions/min_terminated_length": 17.4,
|
|
"entropy": 0.34983569611795245,
|
|
"epoch": 0.24,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 6.399377822875977,
|
|
"learning_rate": 3.804e-07,
|
|
"loss": -0.0407560795545578,
|
|
"num_tokens": 401460.0,
|
|
"reward": 0.9374999701976776,
|
|
"reward_std": 0.25913873612880706,
|
|
"rewards/reward_fn/mean": 0.9374999701976776,
|
|
"rewards/reward_fn/std": 0.2591387540102005,
|
|
"step": 300,
|
|
"step_time": 15.129521024413407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 461.6,
|
|
"completions/max_terminated_length": 461.6,
|
|
"completions/mean_length": 151.96875,
|
|
"completions/mean_terminated_length": 151.96875,
|
|
"completions/min_length": 25.0,
|
|
"completions/min_terminated_length": 25.0,
|
|
"entropy": 0.568750799074769,
|
|
"epoch": 0.248,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.8816224336624146,
|
|
"learning_rate": 3.764e-07,
|
|
"loss": 0.01563715487718582,
|
|
"num_tokens": 434015.0,
|
|
"reward": 0.9343749523162842,
|
|
"reward_std": 0.3420647859573364,
|
|
"rewards/reward_fn/mean": 0.9343749523162842,
|
|
"rewards/reward_fn/std": 0.3420648217201233,
|
|
"step": 310,
|
|
"step_time": 21.686344171082602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 465.6,
|
|
"completions/max_terminated_length": 465.6,
|
|
"completions/mean_length": 160.825,
|
|
"completions/mean_terminated_length": 160.825,
|
|
"completions/min_length": 17.6,
|
|
"completions/min_terminated_length": 17.6,
|
|
"entropy": 0.5512657368555665,
|
|
"epoch": 0.256,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.9943861961364746,
|
|
"learning_rate": 3.7239999999999997e-07,
|
|
"loss": 0.02993807792663574,
|
|
"num_tokens": 468327.0,
|
|
"reward": 1.0312499523162841,
|
|
"reward_std": 0.4049929678440094,
|
|
"rewards/reward_fn/mean": 1.0312499523162841,
|
|
"rewards/reward_fn/std": 0.404993000626564,
|
|
"step": 320,
|
|
"step_time": 20.60081666558981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 409.4,
|
|
"completions/max_terminated_length": 409.4,
|
|
"completions/mean_length": 139.36875,
|
|
"completions/mean_terminated_length": 139.36875,
|
|
"completions/min_length": 17.9,
|
|
"completions/min_terminated_length": 17.9,
|
|
"entropy": 0.4664949773345143,
|
|
"epoch": 0.264,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 3.3355984687805176,
|
|
"learning_rate": 3.684e-07,
|
|
"loss": -0.015160781145095826,
|
|
"num_tokens": 498934.0,
|
|
"reward": 1.1312499642372131,
|
|
"reward_std": 0.3833997189998627,
|
|
"rewards/reward_fn/mean": 1.1312499642372131,
|
|
"rewards/reward_fn/std": 0.38339973986148834,
|
|
"step": 330,
|
|
"step_time": 18.231197547214105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 521.8,
|
|
"completions/max_terminated_length": 471.9,
|
|
"completions/mean_length": 219.0875,
|
|
"completions/mean_terminated_length": 209.3812515258789,
|
|
"completions/min_length": 26.7,
|
|
"completions/min_terminated_length": 26.7,
|
|
"entropy": 0.5834914448671042,
|
|
"epoch": 0.272,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 2.5507986545562744,
|
|
"learning_rate": 3.644e-07,
|
|
"loss": 0.030872175097465517,
|
|
"num_tokens": 541844.0,
|
|
"reward": 1.0062499642372131,
|
|
"reward_std": 0.35398963987827303,
|
|
"rewards/reward_fn/mean": 1.0062499642372131,
|
|
"rewards/reward_fn/std": 0.35398967415094373,
|
|
"step": 340,
|
|
"step_time": 23.158983804937453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 589.9,
|
|
"completions/max_terminated_length": 589.9,
|
|
"completions/mean_length": 213.15,
|
|
"completions/mean_terminated_length": 213.15,
|
|
"completions/min_length": 19.7,
|
|
"completions/min_terminated_length": 19.7,
|
|
"entropy": 0.5913283064961433,
|
|
"epoch": 0.28,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.6039999999999997e-07,
|
|
"loss": 0.01863671690225601,
|
|
"num_tokens": 584100.0,
|
|
"reward": 0.974999976158142,
|
|
"reward_std": 0.29154602289199827,
|
|
"rewards/reward_fn/mean": 0.974999976158142,
|
|
"rewards/reward_fn/std": 0.29154603481292723,
|
|
"step": 350,
|
|
"step_time": 25.847512384923174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 596.0,
|
|
"completions/max_terminated_length": 553.3,
|
|
"completions/mean_length": 166.5875,
|
|
"completions/mean_terminated_length": 161.59166717529297,
|
|
"completions/min_length": 18.1,
|
|
"completions/min_terminated_length": 18.1,
|
|
"entropy": 0.5305257711559535,
|
|
"epoch": 0.288,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.6829164028167725,
|
|
"learning_rate": 3.564e-07,
|
|
"loss": 0.009494951367378235,
|
|
"num_tokens": 619034.0,
|
|
"reward": 0.9218749582767487,
|
|
"reward_std": 0.30490350127220156,
|
|
"rewards/reward_fn/mean": 0.9218749582767487,
|
|
"rewards/reward_fn/std": 0.30490352809429166,
|
|
"step": 360,
|
|
"step_time": 26.364935595309362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 623.8,
|
|
"completions/max_terminated_length": 538.2,
|
|
"completions/mean_length": 224.5,
|
|
"completions/mean_terminated_length": 214.15750122070312,
|
|
"completions/min_length": 51.2,
|
|
"completions/min_terminated_length": 51.2,
|
|
"entropy": 0.5600679079070687,
|
|
"epoch": 0.296,
|
|
"frac_reward_zero_std": 0.575,
|
|
"grad_norm": 2.4058868885040283,
|
|
"learning_rate": 3.5239999999999995e-07,
|
|
"loss": 0.059442996978759766,
|
|
"num_tokens": 662946.0,
|
|
"reward": 1.0337499618530273,
|
|
"reward_std": 0.3860698252916336,
|
|
"rewards/reward_fn/mean": 1.0337499618530273,
|
|
"rewards/reward_fn/std": 0.38606984317302706,
|
|
"step": 370,
|
|
"step_time": 27.650598523486405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 471.4,
|
|
"completions/max_terminated_length": 471.4,
|
|
"completions/mean_length": 211.99375,
|
|
"completions/mean_terminated_length": 211.99375,
|
|
"completions/min_length": 39.4,
|
|
"completions/min_terminated_length": 39.4,
|
|
"entropy": 0.523330201394856,
|
|
"epoch": 0.304,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 1.523726463317871,
|
|
"learning_rate": 3.4839999999999997e-07,
|
|
"loss": -0.017889173328876497,
|
|
"num_tokens": 705285.0,
|
|
"reward": 0.9249999701976777,
|
|
"reward_std": 0.25987376272678375,
|
|
"rewards/reward_fn/mean": 0.9249999701976777,
|
|
"rewards/reward_fn/std": 0.2598737746477127,
|
|
"step": 380,
|
|
"step_time": 20.896892397897318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 584.2,
|
|
"completions/max_terminated_length": 520.2,
|
|
"completions/mean_length": 251.1375,
|
|
"completions/mean_terminated_length": 246.00750122070312,
|
|
"completions/min_length": 62.2,
|
|
"completions/min_terminated_length": 62.2,
|
|
"entropy": 0.5894505500793457,
|
|
"epoch": 0.312,
|
|
"frac_reward_zero_std": 0.575,
|
|
"grad_norm": 2.648047685623169,
|
|
"learning_rate": 3.444e-07,
|
|
"loss": 0.056687426567077634,
|
|
"num_tokens": 753319.0,
|
|
"reward": 1.0999999523162842,
|
|
"reward_std": 0.4478457897901535,
|
|
"rewards/reward_fn/mean": 1.0999999523162842,
|
|
"rewards/reward_fn/std": 0.4478458106517792,
|
|
"step": 390,
|
|
"step_time": 25.831602280260995
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.9,
|
|
"completions/max_terminated_length": 555.9,
|
|
"completions/mean_length": 243.40625,
|
|
"completions/mean_terminated_length": 243.40625,
|
|
"completions/min_length": 40.4,
|
|
"completions/min_terminated_length": 40.4,
|
|
"entropy": 0.5201437229756266,
|
|
"epoch": 0.32,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.4039999999999995e-07,
|
|
"loss": 0.021798035502433775,
|
|
"num_tokens": 800348.0,
|
|
"reward": 1.0062499582767486,
|
|
"reward_std": 0.37394005358219146,
|
|
"rewards/reward_fn/mean": 1.0062499582767486,
|
|
"rewards/reward_fn/std": 0.3739400714635849,
|
|
"step": 400,
|
|
"step_time": 24.329527226556092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 613.3,
|
|
"completions/max_terminated_length": 534.9,
|
|
"completions/mean_length": 244.41875,
|
|
"completions/mean_terminated_length": 234.8495864868164,
|
|
"completions/min_length": 35.6,
|
|
"completions/min_terminated_length": 35.6,
|
|
"entropy": 0.5379184504970909,
|
|
"epoch": 0.328,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.274796962738037,
|
|
"learning_rate": 3.3639999999999997e-07,
|
|
"loss": 0.06314390301704406,
|
|
"num_tokens": 847859.0,
|
|
"reward": 0.9687499582767487,
|
|
"reward_std": 0.35331138372421267,
|
|
"rewards/reward_fn/mean": 0.9687499582767487,
|
|
"rewards/reward_fn/std": 0.35331140756607055,
|
|
"step": 410,
|
|
"step_time": 27.114492582622916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 594.1,
|
|
"completions/max_terminated_length": 562.2,
|
|
"completions/mean_length": 268.15,
|
|
"completions/mean_terminated_length": 264.0933349609375,
|
|
"completions/min_length": 31.0,
|
|
"completions/min_terminated_length": 31.0,
|
|
"entropy": 0.5545548873022199,
|
|
"epoch": 0.336,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 3.0214762687683105,
|
|
"learning_rate": 3.3239999999999993e-07,
|
|
"loss": 0.0016927286982536317,
|
|
"num_tokens": 899211.0,
|
|
"reward": 1.0531249523162842,
|
|
"reward_std": 0.4457359969615936,
|
|
"rewards/reward_fn/mean": 1.0531249523162842,
|
|
"rewards/reward_fn/std": 0.44573602378368377,
|
|
"step": 420,
|
|
"step_time": 26.21310900649987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 566.0,
|
|
"completions/max_terminated_length": 566.0,
|
|
"completions/mean_length": 237.33125,
|
|
"completions/mean_terminated_length": 237.33125,
|
|
"completions/min_length": 43.4,
|
|
"completions/min_terminated_length": 43.4,
|
|
"entropy": 0.4608824389986694,
|
|
"epoch": 0.344,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.376124858856201,
|
|
"learning_rate": 3.284e-07,
|
|
"loss": 0.002047058567404747,
|
|
"num_tokens": 945568.0,
|
|
"reward": 1.2124999523162843,
|
|
"reward_std": 0.4772857129573822,
|
|
"rewards/reward_fn/mean": 1.2124999523162843,
|
|
"rewards/reward_fn/std": 0.47728572487831117,
|
|
"step": 430,
|
|
"step_time": 24.91375301098451
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 620.7,
|
|
"completions/max_terminated_length": 519.3,
|
|
"completions/mean_length": 221.89375,
|
|
"completions/mean_terminated_length": 212.0312515258789,
|
|
"completions/min_length": 18.3,
|
|
"completions/min_terminated_length": 18.3,
|
|
"entropy": 0.45621285401284695,
|
|
"epoch": 0.352,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.244e-07,
|
|
"loss": 0.03366280496120453,
|
|
"num_tokens": 989283.0,
|
|
"reward": 1.0374999582767486,
|
|
"reward_std": 0.37986487746238706,
|
|
"rewards/reward_fn/mean": 1.0374999582767486,
|
|
"rewards/reward_fn/std": 0.37986490726470945,
|
|
"step": 440,
|
|
"step_time": 27.54319058242254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 526.3,
|
|
"completions/max_terminated_length": 470.2,
|
|
"completions/mean_length": 237.4625,
|
|
"completions/mean_terminated_length": 232.41791839599608,
|
|
"completions/min_length": 55.9,
|
|
"completions/min_terminated_length": 55.9,
|
|
"entropy": 0.4685343712568283,
|
|
"epoch": 0.36,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 2.073087692260742,
|
|
"learning_rate": 3.204e-07,
|
|
"loss": 0.008559707552194595,
|
|
"num_tokens": 1035581.0,
|
|
"reward": 1.0281249642372132,
|
|
"reward_std": 0.3265856146812439,
|
|
"rewards/reward_fn/mean": 1.0281249642372132,
|
|
"rewards/reward_fn/std": 0.32658563554286957,
|
|
"step": 450,
|
|
"step_time": 23.314190701860934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 491.8,
|
|
"completions/max_terminated_length": 491.8,
|
|
"completions/mean_length": 263.15,
|
|
"completions/mean_terminated_length": 263.15,
|
|
"completions/min_length": 88.6,
|
|
"completions/min_terminated_length": 88.6,
|
|
"entropy": 0.5305897884070874,
|
|
"epoch": 0.368,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 1.9565300941467285,
|
|
"learning_rate": 3.164e-07,
|
|
"loss": 0.028019136190414427,
|
|
"num_tokens": 1086409.0,
|
|
"reward": 1.093749964237213,
|
|
"reward_std": 0.3530050367116928,
|
|
"rewards/reward_fn/mean": 1.093749964237213,
|
|
"rewards/reward_fn/std": 0.35300505757331846,
|
|
"step": 460,
|
|
"step_time": 21.788672981457786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 585.6,
|
|
"completions/max_terminated_length": 549.8,
|
|
"completions/mean_length": 269.5,
|
|
"completions/mean_terminated_length": 264.71583557128906,
|
|
"completions/min_length": 81.6,
|
|
"completions/min_terminated_length": 81.6,
|
|
"entropy": 0.5114285530522465,
|
|
"epoch": 0.376,
|
|
"frac_reward_zero_std": 0.525,
|
|
"grad_norm": 2.9088966846466064,
|
|
"learning_rate": 3.124e-07,
|
|
"loss": 0.05687993168830872,
|
|
"num_tokens": 1137953.0,
|
|
"reward": 1.1312499582767486,
|
|
"reward_std": 0.42567238211631775,
|
|
"rewards/reward_fn/mean": 1.1312499582767486,
|
|
"rewards/reward_fn/std": 0.4256723940372467,
|
|
"step": 470,
|
|
"step_time": 25.771493053948507
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 545.3,
|
|
"completions/max_terminated_length": 545.3,
|
|
"completions/mean_length": 252.63125,
|
|
"completions/mean_terminated_length": 252.63125,
|
|
"completions/min_length": 44.2,
|
|
"completions/min_terminated_length": 44.2,
|
|
"entropy": 0.5523815616965294,
|
|
"epoch": 0.384,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.084e-07,
|
|
"loss": 0.023237675428390503,
|
|
"num_tokens": 1186658.0,
|
|
"reward": 1.0437499582767487,
|
|
"reward_std": 0.38032626211643217,
|
|
"rewards/reward_fn/mean": 1.0437499582767487,
|
|
"rewards/reward_fn/std": 0.3803262859582901,
|
|
"step": 480,
|
|
"step_time": 23.983928591478616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.6,
|
|
"completions/max_terminated_length": 523.6,
|
|
"completions/mean_length": 243.5875,
|
|
"completions/mean_terminated_length": 243.5875,
|
|
"completions/min_length": 57.4,
|
|
"completions/min_terminated_length": 57.4,
|
|
"entropy": 0.5340582400560379,
|
|
"epoch": 0.392,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 5.369383335113525,
|
|
"learning_rate": 3.044e-07,
|
|
"loss": 0.031129142642021178,
|
|
"num_tokens": 1234168.0,
|
|
"reward": 1.1312499701976777,
|
|
"reward_std": 0.2945299968123436,
|
|
"rewards/reward_fn/mean": 1.1312499701976777,
|
|
"rewards/reward_fn/std": 0.29453000277280805,
|
|
"step": 490,
|
|
"step_time": 23.040811748150738
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 554.0,
|
|
"completions/max_terminated_length": 554.0,
|
|
"completions/mean_length": 229.5125,
|
|
"completions/mean_terminated_length": 229.5125,
|
|
"completions/min_length": 25.8,
|
|
"completions/min_terminated_length": 25.8,
|
|
"entropy": 0.5556849464774132,
|
|
"epoch": 0.4,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.0039999999999996e-07,
|
|
"loss": 0.012525177001953125,
|
|
"num_tokens": 1279254.0,
|
|
"reward": 1.0749999642372132,
|
|
"reward_std": 0.32434508502483367,
|
|
"rewards/reward_fn/mean": 1.0749999642372132,
|
|
"rewards/reward_fn/std": 0.3243451029062271,
|
|
"step": 500,
|
|
"step_time": 24.467384714726357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 610.0,
|
|
"completions/max_terminated_length": 546.3,
|
|
"completions/mean_length": 225.625,
|
|
"completions/mean_terminated_length": 220.79625091552734,
|
|
"completions/min_length": 41.8,
|
|
"completions/min_terminated_length": 41.8,
|
|
"entropy": 0.5509622530080378,
|
|
"epoch": 0.408,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.964e-07,
|
|
"loss": 0.06597378849983215,
|
|
"num_tokens": 1323630.0,
|
|
"reward": 0.9843749582767487,
|
|
"reward_std": 0.35697369575500487,
|
|
"rewards/reward_fn/mean": 0.9843749582767487,
|
|
"rewards/reward_fn/std": 0.3569737136363983,
|
|
"step": 510,
|
|
"step_time": 26.97945318124257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 546.9,
|
|
"completions/max_terminated_length": 496.5,
|
|
"completions/mean_length": 213.04375,
|
|
"completions/mean_terminated_length": 208.10416717529296,
|
|
"completions/min_length": 28.9,
|
|
"completions/min_terminated_length": 28.9,
|
|
"entropy": 0.5530080372467637,
|
|
"epoch": 0.416,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.924e-07,
|
|
"loss": 0.020942620933055878,
|
|
"num_tokens": 1365517.0,
|
|
"reward": 0.9343749701976776,
|
|
"reward_std": 0.27816103398799896,
|
|
"rewards/reward_fn/mean": 0.9343749701976776,
|
|
"rewards/reward_fn/std": 0.2781610548496246,
|
|
"step": 520,
|
|
"step_time": 24.255431303568184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 568.6,
|
|
"completions/max_terminated_length": 523.8,
|
|
"completions/mean_length": 182.9625,
|
|
"completions/mean_terminated_length": 173.29708557128907,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.5604467433411628,
|
|
"epoch": 0.424,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 5.100892066955566,
|
|
"learning_rate": 2.8839999999999996e-07,
|
|
"loss": 0.050272423028945926,
|
|
"num_tokens": 1402911.0,
|
|
"reward": 0.8937499821186066,
|
|
"reward_std": 0.20596824288368226,
|
|
"rewards/reward_fn/mean": 0.8937499821186066,
|
|
"rewards/reward_fn/std": 0.2059682548046112,
|
|
"step": 530,
|
|
"step_time": 25.345189223485068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 388.3,
|
|
"completions/max_terminated_length": 388.3,
|
|
"completions/mean_length": 162.04375,
|
|
"completions/mean_terminated_length": 162.04375,
|
|
"completions/min_length": 36.9,
|
|
"completions/min_terminated_length": 36.9,
|
|
"entropy": 0.5442286553792656,
|
|
"epoch": 0.432,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 4.153268814086914,
|
|
"learning_rate": 2.844e-07,
|
|
"loss": -0.00437091588973999,
|
|
"num_tokens": 1437314.0,
|
|
"reward": 1.093749964237213,
|
|
"reward_std": 0.3850394904613495,
|
|
"rewards/reward_fn/mean": 1.093749964237213,
|
|
"rewards/reward_fn/std": 0.3850395202636719,
|
|
"step": 540,
|
|
"step_time": 17.517217593453825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 478.8,
|
|
"completions/max_terminated_length": 478.8,
|
|
"completions/mean_length": 183.76875,
|
|
"completions/mean_terminated_length": 183.76875,
|
|
"completions/min_length": 18.3,
|
|
"completions/min_terminated_length": 18.3,
|
|
"entropy": 0.5666915670037269,
|
|
"epoch": 0.44,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 3.4264976978302,
|
|
"learning_rate": 2.804e-07,
|
|
"loss": 0.027844130992889404,
|
|
"num_tokens": 1475125.0,
|
|
"reward": 0.9687499701976776,
|
|
"reward_std": 0.2758553087711334,
|
|
"rewards/reward_fn/mean": 0.9687499701976776,
|
|
"rewards/reward_fn/std": 0.27585532069206237,
|
|
"step": 550,
|
|
"step_time": 21.227726350305602
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 416.8,
|
|
"completions/max_terminated_length": 416.8,
|
|
"completions/mean_length": 147.075,
|
|
"completions/mean_terminated_length": 147.075,
|
|
"completions/min_length": 17.8,
|
|
"completions/min_terminated_length": 17.8,
|
|
"entropy": 0.49057656023651364,
|
|
"epoch": 0.448,
|
|
"frac_reward_zero_std": 0.85,
|
|
"grad_norm": 3.256361722946167,
|
|
"learning_rate": 2.7639999999999996e-07,
|
|
"loss": 0.02265160083770752,
|
|
"num_tokens": 1506917.0,
|
|
"reward": 0.9812499761581421,
|
|
"reward_std": 0.2361401915550232,
|
|
"rewards/reward_fn/mean": 0.9812499761581421,
|
|
"rewards/reward_fn/std": 0.2361402153968811,
|
|
"step": 560,
|
|
"step_time": 18.685880808066578
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 384.4,
|
|
"completions/max_terminated_length": 384.4,
|
|
"completions/mean_length": 160.1875,
|
|
"completions/mean_terminated_length": 160.1875,
|
|
"completions/min_length": 18.3,
|
|
"completions/min_terminated_length": 18.3,
|
|
"entropy": 0.5117679107002914,
|
|
"epoch": 0.456,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 2.30198073387146,
|
|
"learning_rate": 2.724e-07,
|
|
"loss": -0.01834808886051178,
|
|
"num_tokens": 1540791.0,
|
|
"reward": 1.0062499523162842,
|
|
"reward_std": 0.3968144774436951,
|
|
"rewards/reward_fn/mean": 1.0062499523162842,
|
|
"rewards/reward_fn/std": 0.39681450724601747,
|
|
"step": 570,
|
|
"step_time": 17.398948775697498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 472.1,
|
|
"completions/max_terminated_length": 472.1,
|
|
"completions/mean_length": 161.95,
|
|
"completions/mean_terminated_length": 161.95,
|
|
"completions/min_length": 18.1,
|
|
"completions/min_terminated_length": 18.1,
|
|
"entropy": 0.47344469791278243,
|
|
"epoch": 0.464,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.3436312675476074,
|
|
"learning_rate": 2.684e-07,
|
|
"loss": -0.026393899321556093,
|
|
"num_tokens": 1575619.0,
|
|
"reward": 0.9874999761581421,
|
|
"reward_std": 0.2700622081756592,
|
|
"rewards/reward_fn/mean": 0.9874999761581421,
|
|
"rewards/reward_fn/std": 0.27006221413612364,
|
|
"step": 580,
|
|
"step_time": 20.999267899850384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 601.7,
|
|
"completions/max_terminated_length": 546.1,
|
|
"completions/mean_length": 233.43125,
|
|
"completions/mean_terminated_length": 228.44708404541015,
|
|
"completions/min_length": 47.7,
|
|
"completions/min_terminated_length": 47.7,
|
|
"entropy": 0.5758602514863014,
|
|
"epoch": 0.472,
|
|
"frac_reward_zero_std": 0.65,
|
|
"grad_norm": 2.945976734161377,
|
|
"learning_rate": 2.644e-07,
|
|
"loss": 0.0019540391862392426,
|
|
"num_tokens": 1621388.0,
|
|
"reward": 1.0218749582767486,
|
|
"reward_std": 0.3570388913154602,
|
|
"rewards/reward_fn/mean": 1.0218749582767486,
|
|
"rewards/reward_fn/std": 0.3570389151573181,
|
|
"step": 590,
|
|
"step_time": 26.790125040663405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 509.8,
|
|
"completions/max_terminated_length": 509.8,
|
|
"completions/mean_length": 223.8,
|
|
"completions/mean_terminated_length": 223.8,
|
|
"completions/min_length": 50.6,
|
|
"completions/min_terminated_length": 50.6,
|
|
"entropy": 0.5903411463834345,
|
|
"epoch": 0.48,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 2.2895243167877197,
|
|
"learning_rate": 2.6040000000000003e-07,
|
|
"loss": 0.01556304395198822,
|
|
"num_tokens": 1665464.0,
|
|
"reward": 1.031249964237213,
|
|
"reward_std": 0.3282184362411499,
|
|
"rewards/reward_fn/mean": 1.031249964237213,
|
|
"rewards/reward_fn/std": 0.32821846306324004,
|
|
"step": 600,
|
|
"step_time": 22.54900577166118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 446.0,
|
|
"completions/max_terminated_length": 446.0,
|
|
"completions/mean_length": 232.44375,
|
|
"completions/mean_terminated_length": 232.44375,
|
|
"completions/min_length": 68.2,
|
|
"completions/min_terminated_length": 68.2,
|
|
"entropy": 0.6003101659938693,
|
|
"epoch": 0.488,
|
|
"frac_reward_zero_std": 0.65,
|
|
"grad_norm": 2.4930214881896973,
|
|
"learning_rate": 2.564e-07,
|
|
"loss": 0.044041958451271054,
|
|
"num_tokens": 1711303.0,
|
|
"reward": 1.0312499523162841,
|
|
"reward_std": 0.3602029472589493,
|
|
"rewards/reward_fn/mean": 1.0312499523162841,
|
|
"rewards/reward_fn/std": 0.3602029770612717,
|
|
"step": 610,
|
|
"step_time": 19.94395455373451
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 619.6,
|
|
"completions/max_terminated_length": 619.6,
|
|
"completions/mean_length": 262.50625,
|
|
"completions/mean_terminated_length": 262.50625,
|
|
"completions/min_length": 36.9,
|
|
"completions/min_terminated_length": 36.9,
|
|
"entropy": 0.625312153622508,
|
|
"epoch": 0.496,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.5237791538238525,
|
|
"learning_rate": 2.524e-07,
|
|
"loss": -0.01255713552236557,
|
|
"num_tokens": 1761532.0,
|
|
"reward": 0.9999999582767487,
|
|
"reward_std": 0.3505753219127655,
|
|
"rewards/reward_fn/mean": 0.9999999582767487,
|
|
"rewards/reward_fn/std": 0.3505753517150879,
|
|
"step": 620,
|
|
"step_time": 27.551811824087054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 514.6,
|
|
"completions/max_terminated_length": 514.6,
|
|
"completions/mean_length": 243.85,
|
|
"completions/mean_terminated_length": 243.85,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.5866754692047834,
|
|
"epoch": 0.504,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.4999687671661377,
|
|
"learning_rate": 2.484e-07,
|
|
"loss": 0.015758931636810303,
|
|
"num_tokens": 1808952.0,
|
|
"reward": 1.0937499582767487,
|
|
"reward_std": 0.39699949622154235,
|
|
"rewards/reward_fn/mean": 1.0937499582767487,
|
|
"rewards/reward_fn/std": 0.39699951112270354,
|
|
"step": 630,
|
|
"step_time": 22.812671538256108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 531.3,
|
|
"completions/max_terminated_length": 485.5,
|
|
"completions/mean_length": 242.3,
|
|
"completions/mean_terminated_length": 238.0504180908203,
|
|
"completions/min_length": 64.9,
|
|
"completions/min_terminated_length": 64.9,
|
|
"entropy": 0.6033073195256293,
|
|
"epoch": 0.512,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.444e-07,
|
|
"loss": 0.010235734283924103,
|
|
"num_tokens": 1856144.0,
|
|
"reward": 0.9718749761581421,
|
|
"reward_std": 0.26936398148536683,
|
|
"rewards/reward_fn/mean": 0.9718749761581421,
|
|
"rewards/reward_fn/std": 0.2693639874458313,
|
|
"step": 640,
|
|
"step_time": 23.65593868405558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 539.9,
|
|
"completions/max_terminated_length": 539.9,
|
|
"completions/mean_length": 242.63125,
|
|
"completions/mean_terminated_length": 242.63125,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"entropy": 0.5526138002052903,
|
|
"epoch": 0.52,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.245793342590332,
|
|
"learning_rate": 2.404e-07,
|
|
"loss": -0.002822137624025345,
|
|
"num_tokens": 1903209.0,
|
|
"reward": 1.0781249523162841,
|
|
"reward_std": 0.43106013536453247,
|
|
"rewards/reward_fn/mean": 1.0781249523162841,
|
|
"rewards/reward_fn/std": 0.4310601681470871,
|
|
"step": 650,
|
|
"step_time": 24.121668337099255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 592.5,
|
|
"completions/max_terminated_length": 592.5,
|
|
"completions/mean_length": 243.65,
|
|
"completions/mean_terminated_length": 243.65,
|
|
"completions/min_length": 63.2,
|
|
"completions/min_terminated_length": 63.2,
|
|
"entropy": 0.557934966403991,
|
|
"epoch": 0.528,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 2.7416722774505615,
|
|
"learning_rate": 2.364e-07,
|
|
"loss": 0.009367964416742324,
|
|
"num_tokens": 1950505.0,
|
|
"reward": 0.9874999642372131,
|
|
"reward_std": 0.3327379524707794,
|
|
"rewards/reward_fn/mean": 0.9874999642372131,
|
|
"rewards/reward_fn/std": 0.33273797333240507,
|
|
"step": 660,
|
|
"step_time": 26.20483476021327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 606.8,
|
|
"completions/max_terminated_length": 487.6,
|
|
"completions/mean_length": 262.125,
|
|
"completions/mean_terminated_length": 252.27166748046875,
|
|
"completions/min_length": 99.2,
|
|
"completions/min_terminated_length": 99.2,
|
|
"entropy": 0.5527632829733193,
|
|
"epoch": 0.536,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.324e-07,
|
|
"loss": 0.03527545630931854,
|
|
"num_tokens": 2000653.0,
|
|
"reward": 1.0562499582767486,
|
|
"reward_std": 0.3691807985305786,
|
|
"rewards/reward_fn/mean": 1.0562499582767486,
|
|
"rewards/reward_fn/std": 0.3691808253526688,
|
|
"step": 670,
|
|
"step_time": 27.034338617976754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 584.8,
|
|
"completions/max_terminated_length": 584.8,
|
|
"completions/mean_length": 285.025,
|
|
"completions/mean_terminated_length": 285.025,
|
|
"completions/min_length": 126.6,
|
|
"completions/min_terminated_length": 126.6,
|
|
"entropy": 0.5860134104266763,
|
|
"epoch": 0.544,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.2839999999999998e-07,
|
|
"loss": 0.01230028122663498,
|
|
"num_tokens": 2054357.0,
|
|
"reward": 1.0124999582767487,
|
|
"reward_std": 0.37343316674232485,
|
|
"rewards/reward_fn/mean": 1.0124999582767487,
|
|
"rewards/reward_fn/std": 0.3734331876039505,
|
|
"step": 680,
|
|
"step_time": 25.73272733730264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 535.7,
|
|
"completions/max_terminated_length": 535.7,
|
|
"completions/mean_length": 258.66875,
|
|
"completions/mean_terminated_length": 258.66875,
|
|
"completions/min_length": 102.7,
|
|
"completions/min_terminated_length": 102.7,
|
|
"entropy": 0.5561068987473845,
|
|
"epoch": 0.552,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 2.3947577476501465,
|
|
"learning_rate": 2.2439999999999997e-07,
|
|
"loss": -0.0206025630235672,
|
|
"num_tokens": 2104000.0,
|
|
"reward": 1.0562499582767486,
|
|
"reward_std": 0.40473316311836244,
|
|
"rewards/reward_fn/mean": 1.0562499582767486,
|
|
"rewards/reward_fn/std": 0.4047331750392914,
|
|
"step": 690,
|
|
"step_time": 23.7531999821309
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 543.3,
|
|
"completions/max_terminated_length": 508.4,
|
|
"completions/mean_length": 255.71875,
|
|
"completions/mean_terminated_length": 251.499169921875,
|
|
"completions/min_length": 79.4,
|
|
"completions/min_terminated_length": 79.4,
|
|
"entropy": 0.5383795527275652,
|
|
"epoch": 0.56,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.0125722885131836,
|
|
"learning_rate": 2.2040000000000001e-07,
|
|
"loss": 0.030927711725234987,
|
|
"num_tokens": 2153891.0,
|
|
"reward": 0.9687499642372132,
|
|
"reward_std": 0.31985312402248384,
|
|
"rewards/reward_fn/mean": 0.9687499642372132,
|
|
"rewards/reward_fn/std": 0.31985313892364503,
|
|
"step": 700,
|
|
"step_time": 24.29951238576323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 624.0,
|
|
"completions/max_terminated_length": 602.9,
|
|
"completions/mean_length": 278.2875,
|
|
"completions/mean_terminated_length": 274.3729187011719,
|
|
"completions/min_length": 91.9,
|
|
"completions/min_terminated_length": 91.9,
|
|
"entropy": 0.527317856438458,
|
|
"epoch": 0.568,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 1.9104549884796143,
|
|
"learning_rate": 2.164e-07,
|
|
"loss": 0.03166455924510956,
|
|
"num_tokens": 2206773.0,
|
|
"reward": 1.090624952316284,
|
|
"reward_std": 0.4429534524679184,
|
|
"rewards/reward_fn/mean": 1.090624952316284,
|
|
"rewards/reward_fn/std": 0.4429534673690796,
|
|
"step": 710,
|
|
"step_time": 27.915992458211257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 548.3,
|
|
"completions/max_terminated_length": 527.2,
|
|
"completions/mean_length": 273.175,
|
|
"completions/mean_terminated_length": 269.2104187011719,
|
|
"completions/min_length": 99.8,
|
|
"completions/min_terminated_length": 99.8,
|
|
"entropy": 0.5130317708477378,
|
|
"epoch": 0.576,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.303006410598755,
|
|
"learning_rate": 2.124e-07,
|
|
"loss": 0.011933594197034835,
|
|
"num_tokens": 2258889.0,
|
|
"reward": 1.0312499582767487,
|
|
"reward_std": 0.39556107819080355,
|
|
"rewards/reward_fn/mean": 1.0312499582767487,
|
|
"rewards/reward_fn/std": 0.39556109607219697,
|
|
"step": 720,
|
|
"step_time": 24.53683318160474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 579.1,
|
|
"completions/max_terminated_length": 536.6,
|
|
"completions/mean_length": 264.69375,
|
|
"completions/mean_terminated_length": 259.7870849609375,
|
|
"completions/min_length": 95.5,
|
|
"completions/min_terminated_length": 95.5,
|
|
"entropy": 0.5283560438081623,
|
|
"epoch": 0.584,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.0839999999999999e-07,
|
|
"loss": 0.049981740117073056,
|
|
"num_tokens": 2309684.0,
|
|
"reward": 1.0218749642372131,
|
|
"reward_std": 0.34613644182682035,
|
|
"rewards/reward_fn/mean": 1.0218749642372131,
|
|
"rewards/reward_fn/std": 0.34613647162914274,
|
|
"step": 730,
|
|
"step_time": 25.811987762106583
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 547.0,
|
|
"completions/max_terminated_length": 547.0,
|
|
"completions/mean_length": 241.6,
|
|
"completions/mean_terminated_length": 241.6,
|
|
"completions/min_length": 64.1,
|
|
"completions/min_terminated_length": 64.1,
|
|
"entropy": 0.49383773263543845,
|
|
"epoch": 0.592,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.562471628189087,
|
|
"learning_rate": 2.0439999999999998e-07,
|
|
"loss": 0.028516930341720582,
|
|
"num_tokens": 2356300.0,
|
|
"reward": 1.0249999701976775,
|
|
"reward_std": 0.3171436250209808,
|
|
"rewards/reward_fn/mean": 1.0249999701976775,
|
|
"rewards/reward_fn/std": 0.317143639922142,
|
|
"step": 740,
|
|
"step_time": 24.14920071642846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 462.0,
|
|
"completions/max_terminated_length": 462.0,
|
|
"completions/mean_length": 239.65625,
|
|
"completions/mean_terminated_length": 239.65625,
|
|
"completions/min_length": 74.7,
|
|
"completions/min_terminated_length": 74.7,
|
|
"entropy": 0.5035146079957485,
|
|
"epoch": 0.6,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.004e-07,
|
|
"loss": -0.014082185924053192,
|
|
"num_tokens": 2402669.0,
|
|
"reward": 0.9781249642372132,
|
|
"reward_std": 0.34400319755077363,
|
|
"rewards/reward_fn/mean": 0.9781249642372132,
|
|
"rewards/reward_fn/std": 0.3440032213926315,
|
|
"step": 750,
|
|
"step_time": 20.69694092241116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 517.6,
|
|
"completions/max_terminated_length": 517.6,
|
|
"completions/mean_length": 266.775,
|
|
"completions/mean_terminated_length": 266.775,
|
|
"completions/min_length": 83.0,
|
|
"completions/min_terminated_length": 83.0,
|
|
"entropy": 0.5417108541354537,
|
|
"epoch": 0.608,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.9890964031219482,
|
|
"learning_rate": 1.9639999999999999e-07,
|
|
"loss": -0.026402422785758974,
|
|
"num_tokens": 2453809.0,
|
|
"reward": 0.9462499558925629,
|
|
"reward_std": 0.3611013382673264,
|
|
"rewards/reward_fn/mean": 0.9462499558925629,
|
|
"rewards/reward_fn/std": 0.361101371049881,
|
|
"step": 760,
|
|
"step_time": 22.986824012873694
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 590.4,
|
|
"completions/max_terminated_length": 532.4,
|
|
"completions/mean_length": 252.18125,
|
|
"completions/mean_terminated_length": 247.59083557128906,
|
|
"completions/min_length": 101.1,
|
|
"completions/min_terminated_length": 101.1,
|
|
"entropy": 0.4916321000084281,
|
|
"epoch": 0.616,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.9239999999999998e-07,
|
|
"loss": 0.030572971701622008,
|
|
"num_tokens": 2502362.0,
|
|
"reward": 1.0468749582767487,
|
|
"reward_std": 0.415197890996933,
|
|
"rewards/reward_fn/mean": 1.0468749582767487,
|
|
"rewards/reward_fn/std": 0.41519791185855864,
|
|
"step": 770,
|
|
"step_time": 26.27881493680179
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 477.3,
|
|
"completions/max_terminated_length": 477.3,
|
|
"completions/mean_length": 238.68125,
|
|
"completions/mean_terminated_length": 238.68125,
|
|
"completions/min_length": 94.1,
|
|
"completions/min_terminated_length": 94.1,
|
|
"entropy": 0.518075543269515,
|
|
"epoch": 0.624,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.884e-07,
|
|
"loss": -0.016722193360328673,
|
|
"num_tokens": 2548967.0,
|
|
"reward": 1.0437499642372132,
|
|
"reward_std": 0.35573128461837766,
|
|
"rewards/reward_fn/mean": 1.0437499642372132,
|
|
"rewards/reward_fn/std": 0.3557313114404678,
|
|
"step": 780,
|
|
"step_time": 21.362713638367133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 519.0,
|
|
"completions/max_terminated_length": 445.2,
|
|
"completions/mean_length": 248.11875,
|
|
"completions/mean_terminated_length": 239.45292053222656,
|
|
"completions/min_length": 66.9,
|
|
"completions/min_terminated_length": 66.9,
|
|
"entropy": 0.5092946726828813,
|
|
"epoch": 0.632,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.420226573944092,
|
|
"learning_rate": 1.844e-07,
|
|
"loss": 0.04115874171257019,
|
|
"num_tokens": 2596974.0,
|
|
"reward": 1.0187499582767487,
|
|
"reward_std": 0.3817029446363449,
|
|
"rewards/reward_fn/mean": 1.0187499582767487,
|
|
"rewards/reward_fn/std": 0.3817029595375061,
|
|
"step": 790,
|
|
"step_time": 23.515300380950794
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 524.6,
|
|
"completions/max_terminated_length": 524.6,
|
|
"completions/mean_length": 266.76875,
|
|
"completions/mean_terminated_length": 266.76875,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"entropy": 0.5524313434958458,
|
|
"epoch": 0.64,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 2.9757630825042725,
|
|
"learning_rate": 1.804e-07,
|
|
"loss": -0.0031020037829875948,
|
|
"num_tokens": 2648317.0,
|
|
"reward": 1.0437499642372132,
|
|
"reward_std": 0.3630165934562683,
|
|
"rewards/reward_fn/mean": 1.0437499642372132,
|
|
"rewards/reward_fn/std": 0.3630166083574295,
|
|
"step": 800,
|
|
"step_time": 23.433115491084756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 594.1,
|
|
"completions/max_terminated_length": 556.5,
|
|
"completions/mean_length": 261.55,
|
|
"completions/mean_terminated_length": 257.0808349609375,
|
|
"completions/min_length": 94.1,
|
|
"completions/min_terminated_length": 94.1,
|
|
"entropy": 0.5408615570515394,
|
|
"epoch": 0.648,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.764e-07,
|
|
"loss": 0.005869099497795105,
|
|
"num_tokens": 2698233.0,
|
|
"reward": 1.0718749642372132,
|
|
"reward_std": 0.3241104021668434,
|
|
"rewards/reward_fn/mean": 1.0718749642372132,
|
|
"rewards/reward_fn/std": 0.32411042004823687,
|
|
"step": 810,
|
|
"step_time": 26.462639589840546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 514.8,
|
|
"completions/max_terminated_length": 464.8,
|
|
"completions/mean_length": 250.34375,
|
|
"completions/mean_terminated_length": 245.92958374023436,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.5274556184187531,
|
|
"epoch": 0.656,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 3.2795684337615967,
|
|
"learning_rate": 1.7239999999999998e-07,
|
|
"loss": 0.03866562247276306,
|
|
"num_tokens": 2746656.0,
|
|
"reward": 1.015624964237213,
|
|
"reward_std": 0.36523938179016113,
|
|
"rewards/reward_fn/mean": 1.015624964237213,
|
|
"rewards/reward_fn/std": 0.3652394026517868,
|
|
"step": 820,
|
|
"step_time": 23.215507409302518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 604.7,
|
|
"completions/max_terminated_length": 557.0,
|
|
"completions/mean_length": 284.25,
|
|
"completions/mean_terminated_length": 279.5695861816406,
|
|
"completions/min_length": 98.3,
|
|
"completions/min_terminated_length": 98.3,
|
|
"entropy": 0.585433507245034,
|
|
"epoch": 0.664,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.278695583343506,
|
|
"learning_rate": 1.684e-07,
|
|
"loss": 0.027391403913497925,
|
|
"num_tokens": 2800636.0,
|
|
"reward": 0.9749999582767487,
|
|
"reward_std": 0.3571206539869308,
|
|
"rewards/reward_fn/mean": 0.9749999582767487,
|
|
"rewards/reward_fn/std": 0.357120668888092,
|
|
"step": 830,
|
|
"step_time": 27.087428363552316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 605.3,
|
|
"completions/max_terminated_length": 541.8,
|
|
"completions/mean_length": 256.64375,
|
|
"completions/mean_terminated_length": 251.76333465576172,
|
|
"completions/min_length": 78.1,
|
|
"completions/min_terminated_length": 78.1,
|
|
"entropy": 0.5444993877783417,
|
|
"epoch": 0.672,
|
|
"frac_reward_zero_std": 0.65,
|
|
"grad_norm": 2.2607951164245605,
|
|
"learning_rate": 1.644e-07,
|
|
"loss": 0.022679784893989564,
|
|
"num_tokens": 2850327.0,
|
|
"reward": 1.034374964237213,
|
|
"reward_std": 0.35396216809749603,
|
|
"rewards/reward_fn/mean": 1.034374964237213,
|
|
"rewards/reward_fn/std": 0.35396219193935397,
|
|
"step": 840,
|
|
"step_time": 27.1366524099838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 542.0,
|
|
"completions/max_terminated_length": 542.0,
|
|
"completions/mean_length": 250.46875,
|
|
"completions/mean_terminated_length": 250.46875,
|
|
"completions/min_length": 94.7,
|
|
"completions/min_terminated_length": 94.7,
|
|
"entropy": 0.5348553240299225,
|
|
"epoch": 0.68,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.5916006565093994,
|
|
"learning_rate": 1.6039999999999998e-07,
|
|
"loss": -0.023145222663879396,
|
|
"num_tokens": 2899042.0,
|
|
"reward": 1.0624999582767487,
|
|
"reward_std": 0.38501180410385133,
|
|
"rewards/reward_fn/mean": 1.0624999582767487,
|
|
"rewards/reward_fn/std": 0.3850118160247803,
|
|
"step": 850,
|
|
"step_time": 24.06133564542979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 492.0,
|
|
"completions/max_terminated_length": 492.0,
|
|
"completions/mean_length": 248.4,
|
|
"completions/mean_terminated_length": 248.4,
|
|
"completions/min_length": 76.7,
|
|
"completions/min_terminated_length": 76.7,
|
|
"entropy": 0.5445443953387439,
|
|
"epoch": 0.688,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.564e-07,
|
|
"loss": 0.008287916332483292,
|
|
"num_tokens": 2946790.0,
|
|
"reward": 1.0093749642372132,
|
|
"reward_std": 0.35188313722610476,
|
|
"rewards/reward_fn/mean": 1.0093749642372132,
|
|
"rewards/reward_fn/std": 0.3518831551074982,
|
|
"step": 860,
|
|
"step_time": 21.985819199867546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 594.7,
|
|
"completions/max_terminated_length": 532.9,
|
|
"completions/mean_length": 261.8625,
|
|
"completions/mean_terminated_length": 256.7483337402344,
|
|
"completions/min_length": 92.5,
|
|
"completions/min_terminated_length": 92.5,
|
|
"entropy": 0.5800149150192737,
|
|
"epoch": 0.696,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.524e-07,
|
|
"loss": 0.04065183401107788,
|
|
"num_tokens": 2997048.0,
|
|
"reward": 0.9968749701976776,
|
|
"reward_std": 0.30479497015476226,
|
|
"rewards/reward_fn/mean": 0.9968749701976776,
|
|
"rewards/reward_fn/std": 0.30479499995708464,
|
|
"step": 870,
|
|
"step_time": 26.580935311084612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 513.3,
|
|
"completions/max_terminated_length": 513.3,
|
|
"completions/mean_length": 253.06875,
|
|
"completions/mean_terminated_length": 253.06875,
|
|
"completions/min_length": 77.6,
|
|
"completions/min_terminated_length": 77.6,
|
|
"entropy": 0.5305031981319189,
|
|
"epoch": 0.704,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 4.454606533050537,
|
|
"learning_rate": 1.484e-07,
|
|
"loss": 0.022549983859062196,
|
|
"num_tokens": 3045711.0,
|
|
"reward": 1.0312499582767487,
|
|
"reward_std": 0.38086153864860534,
|
|
"rewards/reward_fn/mean": 1.0312499582767487,
|
|
"rewards/reward_fn/std": 0.3808615684509277,
|
|
"step": 880,
|
|
"step_time": 23.01209013015032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 502.9,
|
|
"completions/max_terminated_length": 502.9,
|
|
"completions/mean_length": 256.13125,
|
|
"completions/mean_terminated_length": 256.13125,
|
|
"completions/min_length": 66.3,
|
|
"completions/min_terminated_length": 66.3,
|
|
"entropy": 0.5786763843148947,
|
|
"epoch": 0.712,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.9759247303009033,
|
|
"learning_rate": 1.444e-07,
|
|
"loss": -0.02140951305627823,
|
|
"num_tokens": 3094856.0,
|
|
"reward": 1.0437499582767487,
|
|
"reward_std": 0.3924266636371613,
|
|
"rewards/reward_fn/mean": 1.0437499582767487,
|
|
"rewards/reward_fn/std": 0.39242667853832247,
|
|
"step": 890,
|
|
"step_time": 22.476798066869378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 487.1,
|
|
"completions/max_terminated_length": 487.1,
|
|
"completions/mean_length": 236.9625,
|
|
"completions/mean_terminated_length": 236.9625,
|
|
"completions/min_length": 78.6,
|
|
"completions/min_terminated_length": 78.6,
|
|
"entropy": 0.5970848154276609,
|
|
"epoch": 0.72,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.90382719039917,
|
|
"learning_rate": 1.4039999999999999e-07,
|
|
"loss": 0.013022461533546447,
|
|
"num_tokens": 3140922.0,
|
|
"reward": 1.0562499523162843,
|
|
"reward_std": 0.4089065968990326,
|
|
"rewards/reward_fn/mean": 1.0562499523162843,
|
|
"rewards/reward_fn/std": 0.40890662670135497,
|
|
"step": 900,
|
|
"step_time": 21.815543547831474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 585.1,
|
|
"completions/max_terminated_length": 585.1,
|
|
"completions/mean_length": 267.65625,
|
|
"completions/mean_terminated_length": 267.65625,
|
|
"completions/min_length": 83.9,
|
|
"completions/min_terminated_length": 83.9,
|
|
"entropy": 0.5671312633901835,
|
|
"epoch": 0.728,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 3.2715682983398438,
|
|
"learning_rate": 1.3639999999999998e-07,
|
|
"loss": -0.008570893108844757,
|
|
"num_tokens": 3192063.0,
|
|
"reward": 1.0312499582767487,
|
|
"reward_std": 0.385887947678566,
|
|
"rewards/reward_fn/mean": 1.0312499582767487,
|
|
"rewards/reward_fn/std": 0.3858879655599594,
|
|
"step": 910,
|
|
"step_time": 25.983504464896395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 597.8,
|
|
"completions/max_terminated_length": 572.6,
|
|
"completions/mean_length": 280.96875,
|
|
"completions/mean_terminated_length": 276.9291687011719,
|
|
"completions/min_length": 112.6,
|
|
"completions/min_terminated_length": 112.6,
|
|
"entropy": 0.5547854236327112,
|
|
"epoch": 0.736,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.8137001991271973,
|
|
"learning_rate": 1.324e-07,
|
|
"loss": -0.0006064340472221374,
|
|
"num_tokens": 3245226.0,
|
|
"reward": 1.0593749642372132,
|
|
"reward_std": 0.3496703714132309,
|
|
"rewards/reward_fn/mean": 1.0593749642372132,
|
|
"rewards/reward_fn/std": 0.3496703922748566,
|
|
"step": 920,
|
|
"step_time": 26.724192412989215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 521.3,
|
|
"completions/max_terminated_length": 521.3,
|
|
"completions/mean_length": 258.66875,
|
|
"completions/mean_terminated_length": 258.66875,
|
|
"completions/min_length": 73.2,
|
|
"completions/min_terminated_length": 73.2,
|
|
"entropy": 0.55043915361166,
|
|
"epoch": 0.744,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.9904998540878296,
|
|
"learning_rate": 1.2839999999999999e-07,
|
|
"loss": 0.002445448562502861,
|
|
"num_tokens": 3295165.0,
|
|
"reward": 1.0499999582767487,
|
|
"reward_std": 0.40058289766311644,
|
|
"rewards/reward_fn/mean": 1.0499999582767487,
|
|
"rewards/reward_fn/std": 0.40058292746543883,
|
|
"step": 930,
|
|
"step_time": 23.23095205714926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 649.2,
|
|
"completions/max_terminated_length": 649.2,
|
|
"completions/mean_length": 293.1125,
|
|
"completions/mean_terminated_length": 293.1125,
|
|
"completions/min_length": 63.0,
|
|
"completions/min_terminated_length": 63.0,
|
|
"entropy": 0.5630233686417341,
|
|
"epoch": 0.752,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.8717433214187622,
|
|
"learning_rate": 1.244e-07,
|
|
"loss": 0.015195921063423157,
|
|
"num_tokens": 3350455.0,
|
|
"reward": 0.9562499582767486,
|
|
"reward_std": 0.3392513394355774,
|
|
"rewards/reward_fn/mean": 0.9562499582767486,
|
|
"rewards/reward_fn/std": 0.33925136625766755,
|
|
"step": 940,
|
|
"step_time": 28.78155018389225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 566.8,
|
|
"completions/max_terminated_length": 566.8,
|
|
"completions/mean_length": 279.64375,
|
|
"completions/mean_terminated_length": 279.64375,
|
|
"completions/min_length": 92.6,
|
|
"completions/min_terminated_length": 92.6,
|
|
"entropy": 0.5822894293814898,
|
|
"epoch": 0.76,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 1.3468974828720093,
|
|
"learning_rate": 1.204e-07,
|
|
"loss": -0.006339100748300552,
|
|
"num_tokens": 3403710.0,
|
|
"reward": 1.0437499642372132,
|
|
"reward_std": 0.3522812038660049,
|
|
"rewards/reward_fn/mean": 1.0437499642372132,
|
|
"rewards/reward_fn/std": 0.35228122770786285,
|
|
"step": 950,
|
|
"step_time": 25.218540608556943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 550.5,
|
|
"completions/max_terminated_length": 550.5,
|
|
"completions/mean_length": 267.8625,
|
|
"completions/mean_terminated_length": 267.8625,
|
|
"completions/min_length": 101.9,
|
|
"completions/min_terminated_length": 101.9,
|
|
"entropy": 0.5490182695910335,
|
|
"epoch": 0.768,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 2.347418785095215,
|
|
"learning_rate": 1.164e-07,
|
|
"loss": -0.018277975916862487,
|
|
"num_tokens": 3454788.0,
|
|
"reward": 1.0874999523162843,
|
|
"reward_std": 0.4399394065141678,
|
|
"rewards/reward_fn/mean": 1.0874999523162843,
|
|
"rewards/reward_fn/std": 0.43993942737579345,
|
|
"step": 960,
|
|
"step_time": 24.56708482489921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 563.2,
|
|
"completions/max_terminated_length": 563.2,
|
|
"completions/mean_length": 286.8625,
|
|
"completions/mean_terminated_length": 286.8625,
|
|
"completions/min_length": 111.1,
|
|
"completions/min_terminated_length": 111.1,
|
|
"entropy": 0.5671119350939989,
|
|
"epoch": 0.776,
|
|
"frac_reward_zero_std": 0.825,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.124e-07,
|
|
"loss": 0.001966666430234909,
|
|
"num_tokens": 3508722.0,
|
|
"reward": 0.9843749582767487,
|
|
"reward_std": 0.3477096170186996,
|
|
"rewards/reward_fn/mean": 0.9843749582767487,
|
|
"rewards/reward_fn/std": 0.3477096438407898,
|
|
"step": 970,
|
|
"step_time": 25.104713304387406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01875,
|
|
"completions/max_length": 572.8,
|
|
"completions/max_terminated_length": 515.7,
|
|
"completions/mean_length": 258.20625,
|
|
"completions/mean_terminated_length": 244.45547790527343,
|
|
"completions/min_length": 68.6,
|
|
"completions/min_terminated_length": 68.6,
|
|
"entropy": 0.4816724480129778,
|
|
"epoch": 0.784,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.33716082572937,
|
|
"learning_rate": 1.0839999999999999e-07,
|
|
"loss": 0.021231548488140108,
|
|
"num_tokens": 3558171.0,
|
|
"reward": 1.0968749582767487,
|
|
"reward_std": 0.3681450128555298,
|
|
"rewards/reward_fn/mean": 1.0968749582767487,
|
|
"rewards/reward_fn/std": 0.3681450396776199,
|
|
"step": 980,
|
|
"step_time": 25.863722542859612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 474.4,
|
|
"completions/max_terminated_length": 474.4,
|
|
"completions/mean_length": 241.725,
|
|
"completions/mean_terminated_length": 241.725,
|
|
"completions/min_length": 102.0,
|
|
"completions/min_terminated_length": 102.0,
|
|
"entropy": 0.5186899159103632,
|
|
"epoch": 0.792,
|
|
"frac_reward_zero_std": 0.775,
|
|
"grad_norm": 2.8566486835479736,
|
|
"learning_rate": 1.0440000000000001e-07,
|
|
"loss": -0.013251829147338866,
|
|
"num_tokens": 3604823.0,
|
|
"reward": 1.0312499582767487,
|
|
"reward_std": 0.3475317031145096,
|
|
"rewards/reward_fn/mean": 1.0312499582767487,
|
|
"rewards/reward_fn/std": 0.347531720995903,
|
|
"step": 990,
|
|
"step_time": 21.354597127251328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 513.8,
|
|
"completions/max_terminated_length": 472.6,
|
|
"completions/mean_length": 276.25625,
|
|
"completions/mean_terminated_length": 271.7895843505859,
|
|
"completions/min_length": 129.9,
|
|
"completions/min_terminated_length": 129.9,
|
|
"entropy": 0.5287249894812703,
|
|
"epoch": 0.8,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 1.7595624923706055,
|
|
"learning_rate": 1.004e-07,
|
|
"loss": 0.030878221988677977,
|
|
"num_tokens": 3657228.0,
|
|
"reward": 1.0406249701976775,
|
|
"reward_std": 0.31898270547389984,
|
|
"rewards/reward_fn/mean": 1.0406249701976775,
|
|
"rewards/reward_fn/std": 0.31898272335529326,
|
|
"step": 1000,
|
|
"step_time": 23.230659662559628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 557.3,
|
|
"completions/max_terminated_length": 491.5,
|
|
"completions/mean_length": 266.30625,
|
|
"completions/mean_terminated_length": 261.2725006103516,
|
|
"completions/min_length": 75.9,
|
|
"completions/min_terminated_length": 75.9,
|
|
"entropy": 0.5200971383601427,
|
|
"epoch": 0.808,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 2.3148114681243896,
|
|
"learning_rate": 9.639999999999999e-08,
|
|
"loss": 0.003333679959177971,
|
|
"num_tokens": 3708309.0,
|
|
"reward": 1.0343749582767487,
|
|
"reward_std": 0.3490926504135132,
|
|
"rewards/reward_fn/mean": 1.0343749582767487,
|
|
"rewards/reward_fn/std": 0.34909267723560333,
|
|
"step": 1010,
|
|
"step_time": 25.057548108976334
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 537.4,
|
|
"completions/max_terminated_length": 537.4,
|
|
"completions/mean_length": 272.03125,
|
|
"completions/mean_terminated_length": 272.03125,
|
|
"completions/min_length": 110.4,
|
|
"completions/min_terminated_length": 110.4,
|
|
"entropy": 0.5065994596108794,
|
|
"epoch": 0.816,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 3.423994541168213,
|
|
"learning_rate": 9.24e-08,
|
|
"loss": 0.023444755375385283,
|
|
"num_tokens": 3760274.0,
|
|
"reward": 1.0218749582767486,
|
|
"reward_std": 0.35771805346012114,
|
|
"rewards/reward_fn/mean": 1.0218749582767486,
|
|
"rewards/reward_fn/std": 0.35771807432174685,
|
|
"step": 1020,
|
|
"step_time": 23.9467576073017
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 510.8,
|
|
"completions/max_terminated_length": 510.8,
|
|
"completions/mean_length": 283.375,
|
|
"completions/mean_terminated_length": 283.375,
|
|
"completions/min_length": 129.7,
|
|
"completions/min_terminated_length": 129.7,
|
|
"entropy": 0.5147501945495605,
|
|
"epoch": 0.824,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.1647582054138184,
|
|
"learning_rate": 8.84e-08,
|
|
"loss": 0.0007140228524804115,
|
|
"num_tokens": 3813922.0,
|
|
"reward": 1.199999952316284,
|
|
"reward_std": 0.491314160823822,
|
|
"rewards/reward_fn/mean": 1.199999952316284,
|
|
"rewards/reward_fn/std": 0.49131418466567994,
|
|
"step": 1030,
|
|
"step_time": 23.01038688295521
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 521.3,
|
|
"completions/max_terminated_length": 521.3,
|
|
"completions/mean_length": 264.31875,
|
|
"completions/mean_terminated_length": 264.31875,
|
|
"completions/min_length": 121.5,
|
|
"completions/min_terminated_length": 121.5,
|
|
"entropy": 0.5070869445800781,
|
|
"epoch": 0.832,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 2.585664749145508,
|
|
"learning_rate": 8.44e-08,
|
|
"loss": 0.028041335940361022,
|
|
"num_tokens": 3864485.0,
|
|
"reward": 1.0937499582767487,
|
|
"reward_std": 0.4248550355434418,
|
|
"rewards/reward_fn/mean": 1.0937499582767487,
|
|
"rewards/reward_fn/std": 0.42485505938529966,
|
|
"step": 1040,
|
|
"step_time": 23.34787617237307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 549.5,
|
|
"completions/max_terminated_length": 502.5,
|
|
"completions/mean_length": 267.49375,
|
|
"completions/mean_terminated_length": 262.67791748046875,
|
|
"completions/min_length": 93.5,
|
|
"completions/min_terminated_length": 93.5,
|
|
"entropy": 0.5002012770622969,
|
|
"epoch": 0.84,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.039999999999999e-08,
|
|
"loss": 0.04210644662380218,
|
|
"num_tokens": 3915900.0,
|
|
"reward": 1.0968749582767487,
|
|
"reward_std": 0.41755713522434235,
|
|
"rewards/reward_fn/mean": 1.0968749582767487,
|
|
"rewards/reward_fn/std": 0.4175571441650391,
|
|
"step": 1050,
|
|
"step_time": 24.77414979697205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 627.7,
|
|
"completions/max_terminated_length": 555.7,
|
|
"completions/mean_length": 305.38125,
|
|
"completions/mean_terminated_length": 289.6212219238281,
|
|
"completions/min_length": 135.6,
|
|
"completions/min_terminated_length": 135.6,
|
|
"entropy": 0.5611137403175235,
|
|
"epoch": 0.848,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 2.05741024017334,
|
|
"learning_rate": 7.64e-08,
|
|
"loss": 0.0378756046295166,
|
|
"num_tokens": 3973093.0,
|
|
"reward": 0.9687499642372132,
|
|
"reward_std": 0.332451206445694,
|
|
"rewards/reward_fn/mean": 0.9687499642372132,
|
|
"rewards/reward_fn/std": 0.33245123326778414,
|
|
"step": 1060,
|
|
"step_time": 28.287785303639247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 746.1,
|
|
"completions/max_terminated_length": 658.4,
|
|
"completions/mean_length": 300.1625,
|
|
"completions/mean_terminated_length": 290.8487518310547,
|
|
"completions/min_length": 116.9,
|
|
"completions/min_terminated_length": 116.9,
|
|
"entropy": 0.5516405990347266,
|
|
"epoch": 0.856,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.4637022018432617,
|
|
"learning_rate": 7.24e-08,
|
|
"loss": 0.02026980072259903,
|
|
"num_tokens": 4029403.0,
|
|
"reward": 0.9187499821186066,
|
|
"reward_std": 0.22319530844688415,
|
|
"rewards/reward_fn/mean": 0.9187499821186066,
|
|
"rewards/reward_fn/std": 0.22319531738758086,
|
|
"step": 1070,
|
|
"step_time": 33.521394540695475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 534.6,
|
|
"completions/max_terminated_length": 488.9,
|
|
"completions/mean_length": 263.6125,
|
|
"completions/mean_terminated_length": 258.89708557128904,
|
|
"completions/min_length": 100.4,
|
|
"completions/min_terminated_length": 100.4,
|
|
"entropy": 0.5176247822120785,
|
|
"epoch": 0.864,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 3.2136309146881104,
|
|
"learning_rate": 6.84e-08,
|
|
"loss": 0.035635238885879515,
|
|
"num_tokens": 4080145.0,
|
|
"reward": 1.015624964237213,
|
|
"reward_std": 0.3429378718137741,
|
|
"rewards/reward_fn/mean": 1.015624964237213,
|
|
"rewards/reward_fn/std": 0.34293788969516753,
|
|
"step": 1080,
|
|
"step_time": 24.050806782906875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 542.3,
|
|
"completions/max_terminated_length": 542.3,
|
|
"completions/mean_length": 260.6125,
|
|
"completions/mean_terminated_length": 260.6125,
|
|
"completions/min_length": 112.2,
|
|
"completions/min_terminated_length": 112.2,
|
|
"entropy": 0.5131400337442755,
|
|
"epoch": 0.872,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.44e-08,
|
|
"loss": 0.0009937494993209838,
|
|
"num_tokens": 4129863.0,
|
|
"reward": 0.9937499642372132,
|
|
"reward_std": 0.3179366230964661,
|
|
"rewards/reward_fn/mean": 0.9937499642372132,
|
|
"rewards/reward_fn/std": 0.31793664693832396,
|
|
"step": 1090,
|
|
"step_time": 24.295423823781313
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01875,
|
|
"completions/max_length": 593.4,
|
|
"completions/max_terminated_length": 483.5,
|
|
"completions/mean_length": 250.2875,
|
|
"completions/mean_terminated_length": 235.8367889404297,
|
|
"completions/min_length": 98.2,
|
|
"completions/min_terminated_length": 98.2,
|
|
"entropy": 0.4774348322302103,
|
|
"epoch": 0.88,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.04e-08,
|
|
"loss": 0.05956764221191406,
|
|
"num_tokens": 4178285.0,
|
|
"reward": 1.0656249523162842,
|
|
"reward_std": 0.4110028088092804,
|
|
"rewards/reward_fn/mean": 1.0656249523162842,
|
|
"rewards/reward_fn/std": 0.4110028326511383,
|
|
"step": 1100,
|
|
"step_time": 26.840810445183887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 591.2,
|
|
"completions/max_terminated_length": 540.6,
|
|
"completions/mean_length": 274.45625,
|
|
"completions/mean_terminated_length": 269.875,
|
|
"completions/min_length": 124.3,
|
|
"completions/min_terminated_length": 124.3,
|
|
"entropy": 0.5233237653970718,
|
|
"epoch": 0.888,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 4.72261381149292,
|
|
"learning_rate": 5.6399999999999995e-08,
|
|
"loss": -0.01764392852783203,
|
|
"num_tokens": 4230406.0,
|
|
"reward": 1.0343749582767487,
|
|
"reward_std": 0.40663672983646393,
|
|
"rewards/reward_fn/mean": 1.0343749582767487,
|
|
"rewards/reward_fn/std": 0.4066367566585541,
|
|
"step": 1110,
|
|
"step_time": 26.58696360127069
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 544.6,
|
|
"completions/max_terminated_length": 486.7,
|
|
"completions/mean_length": 264.1125,
|
|
"completions/mean_terminated_length": 259.6304168701172,
|
|
"completions/min_length": 124.9,
|
|
"completions/min_terminated_length": 124.9,
|
|
"entropy": 0.49998724516481163,
|
|
"epoch": 0.896,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.24e-08,
|
|
"loss": 0.03018750548362732,
|
|
"num_tokens": 4280868.0,
|
|
"reward": 1.0843749582767486,
|
|
"reward_std": 0.4210015803575516,
|
|
"rewards/reward_fn/mean": 1.0843749582767486,
|
|
"rewards/reward_fn/std": 0.421001598238945,
|
|
"step": 1120,
|
|
"step_time": 24.553724389290437
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 616.2,
|
|
"completions/max_terminated_length": 564.2,
|
|
"completions/mean_length": 266.13125,
|
|
"completions/mean_terminated_length": 261.57250061035154,
|
|
"completions/min_length": 93.2,
|
|
"completions/min_terminated_length": 93.2,
|
|
"entropy": 0.5146013794466853,
|
|
"epoch": 0.904,
|
|
"frac_reward_zero_std": 0.65,
|
|
"grad_norm": 3.1288528442382812,
|
|
"learning_rate": 4.8399999999999997e-08,
|
|
"loss": 0.03101418912410736,
|
|
"num_tokens": 4331401.0,
|
|
"reward": 1.0718749582767486,
|
|
"reward_std": 0.37411039769649507,
|
|
"rewards/reward_fn/mean": 1.0718749582767486,
|
|
"rewards/reward_fn/std": 0.3741104155778885,
|
|
"step": 1130,
|
|
"step_time": 27.615588352596387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00625,
|
|
"completions/max_length": 624.7,
|
|
"completions/max_terminated_length": 591.7,
|
|
"completions/mean_length": 291.43125,
|
|
"completions/mean_terminated_length": 286.9612518310547,
|
|
"completions/min_length": 109.7,
|
|
"completions/min_terminated_length": 109.7,
|
|
"entropy": 0.528622365463525,
|
|
"epoch": 0.912,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.9404244422912598,
|
|
"learning_rate": 4.44e-08,
|
|
"loss": 0.027211108803749086,
|
|
"num_tokens": 4386302.0,
|
|
"reward": 1.0781249582767487,
|
|
"reward_std": 0.3990673363208771,
|
|
"rewards/reward_fn/mean": 1.0781249582767487,
|
|
"rewards/reward_fn/std": 0.3990673542022705,
|
|
"step": 1140,
|
|
"step_time": 27.958656183769925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 527.2,
|
|
"completions/max_terminated_length": 527.2,
|
|
"completions/mean_length": 286.13125,
|
|
"completions/mean_terminated_length": 286.13125,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 0.513075502589345,
|
|
"epoch": 0.92,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 1.4120060205459595,
|
|
"learning_rate": 4.04e-08,
|
|
"loss": -0.000885472446680069,
|
|
"num_tokens": 4440643.0,
|
|
"reward": 1.062499964237213,
|
|
"reward_std": 0.3273422926664352,
|
|
"rewards/reward_fn/mean": 1.062499964237213,
|
|
"rewards/reward_fn/std": 0.3273423135280609,
|
|
"step": 1150,
|
|
"step_time": 23.63183649624698
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 599.6,
|
|
"completions/max_terminated_length": 599.6,
|
|
"completions/mean_length": 280.80625,
|
|
"completions/mean_terminated_length": 280.80625,
|
|
"completions/min_length": 125.5,
|
|
"completions/min_terminated_length": 125.5,
|
|
"entropy": 0.5326275195926428,
|
|
"epoch": 0.928,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.777073860168457,
|
|
"learning_rate": 3.64e-08,
|
|
"loss": 0.021264398097991945,
|
|
"num_tokens": 4493740.0,
|
|
"reward": 1.0749999582767487,
|
|
"reward_std": 0.4012425780296326,
|
|
"rewards/reward_fn/mean": 1.0749999582767487,
|
|
"rewards/reward_fn/std": 0.40124259889125824,
|
|
"step": 1160,
|
|
"step_time": 27.038261169008912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 626.3,
|
|
"completions/max_terminated_length": 570.1,
|
|
"completions/mean_length": 281.4875,
|
|
"completions/mean_terminated_length": 272.0464294433594,
|
|
"completions/min_length": 88.5,
|
|
"completions/min_terminated_length": 88.5,
|
|
"entropy": 0.5435919009149075,
|
|
"epoch": 0.936,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.009503126144409,
|
|
"learning_rate": 3.24e-08,
|
|
"loss": 0.014678403735160828,
|
|
"num_tokens": 4547326.0,
|
|
"reward": 0.9749999642372131,
|
|
"reward_std": 0.3376161724328995,
|
|
"rewards/reward_fn/mean": 0.9749999642372131,
|
|
"rewards/reward_fn/std": 0.33761619329452514,
|
|
"step": 1170,
|
|
"step_time": 28.00773431826383
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 540.3,
|
|
"completions/max_terminated_length": 540.3,
|
|
"completions/mean_length": 281.43125,
|
|
"completions/mean_terminated_length": 281.43125,
|
|
"completions/min_length": 134.6,
|
|
"completions/min_terminated_length": 134.6,
|
|
"entropy": 0.5330241896212101,
|
|
"epoch": 0.944,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 2.01973557472229,
|
|
"learning_rate": 2.84e-08,
|
|
"loss": 0.010066453367471695,
|
|
"num_tokens": 4600699.0,
|
|
"reward": 0.9749999701976776,
|
|
"reward_std": 0.30032309591770173,
|
|
"rewards/reward_fn/mean": 0.9749999701976776,
|
|
"rewards/reward_fn/std": 0.3003231108188629,
|
|
"step": 1180,
|
|
"step_time": 24.195893923100083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 531.7,
|
|
"completions/max_terminated_length": 531.7,
|
|
"completions/mean_length": 278.15,
|
|
"completions/mean_terminated_length": 278.15,
|
|
"completions/min_length": 110.2,
|
|
"completions/min_terminated_length": 110.2,
|
|
"entropy": 0.5508426167070866,
|
|
"epoch": 0.952,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.44e-08,
|
|
"loss": 0.009711582958698273,
|
|
"num_tokens": 4653463.0,
|
|
"reward": 1.031249964237213,
|
|
"reward_std": 0.34981620609760283,
|
|
"rewards/reward_fn/mean": 1.031249964237213,
|
|
"rewards/reward_fn/std": 0.34981622397899625,
|
|
"step": 1190,
|
|
"step_time": 23.97323694275692
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01875,
|
|
"completions/max_length": 658.7,
|
|
"completions/max_terminated_length": 541.0,
|
|
"completions/mean_length": 285.43125,
|
|
"completions/mean_terminated_length": 271.64125671386716,
|
|
"completions/min_length": 99.9,
|
|
"completions/min_terminated_length": 99.9,
|
|
"entropy": 0.5076606808230281,
|
|
"epoch": 0.96,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 2.406820774078369,
|
|
"learning_rate": 2.04e-08,
|
|
"loss": 0.07293225526809692,
|
|
"num_tokens": 4707780.0,
|
|
"reward": 0.9531249642372132,
|
|
"reward_std": 0.3397530823945999,
|
|
"rewards/reward_fn/mean": 0.9531249642372132,
|
|
"rewards/reward_fn/std": 0.3397530972957611,
|
|
"step": 1200,
|
|
"step_time": 29.918243083078416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 778.4,
|
|
"completions/max_terminated_length": 598.6,
|
|
"completions/mean_length": 298.01875,
|
|
"completions/mean_terminated_length": 279.6341720581055,
|
|
"completions/min_length": 107.9,
|
|
"completions/min_terminated_length": 107.9,
|
|
"entropy": 0.4813551393337548,
|
|
"epoch": 0.968,
|
|
"frac_reward_zero_std": 0.675,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.64e-08,
|
|
"loss": 0.07352781295776367,
|
|
"num_tokens": 4763607.0,
|
|
"reward": 1.0562499523162843,
|
|
"reward_std": 0.444338920712471,
|
|
"rewards/reward_fn/mean": 1.0562499523162843,
|
|
"rewards/reward_fn/std": 0.44433894753456116,
|
|
"step": 1210,
|
|
"step_time": 35.2928637356963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 542.4,
|
|
"completions/max_terminated_length": 542.4,
|
|
"completions/mean_length": 273.9375,
|
|
"completions/mean_terminated_length": 273.9375,
|
|
"completions/min_length": 112.5,
|
|
"completions/min_terminated_length": 112.5,
|
|
"entropy": 0.5351916439831257,
|
|
"epoch": 0.976,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.6699888706207275,
|
|
"learning_rate": 1.2399999999999999e-08,
|
|
"loss": 0.020609369874000548,
|
|
"num_tokens": 4815925.0,
|
|
"reward": 1.043749952316284,
|
|
"reward_std": 0.409377720952034,
|
|
"rewards/reward_fn/mean": 1.043749952316284,
|
|
"rewards/reward_fn/std": 0.4093777507543564,
|
|
"step": 1220,
|
|
"step_time": 24.284022097615527
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 555.8,
|
|
"completions/max_terminated_length": 555.8,
|
|
"completions/mean_length": 295.23125,
|
|
"completions/mean_terminated_length": 295.23125,
|
|
"completions/min_length": 116.1,
|
|
"completions/min_terminated_length": 116.1,
|
|
"entropy": 0.5482900742441416,
|
|
"epoch": 0.984,
|
|
"frac_reward_zero_std": 0.725,
|
|
"grad_norm": 1.7745944261550903,
|
|
"learning_rate": 8.399999999999999e-09,
|
|
"loss": 0.003481149673461914,
|
|
"num_tokens": 4871670.0,
|
|
"reward": 0.9249999582767486,
|
|
"reward_std": 0.30596340596675875,
|
|
"rewards/reward_fn/mean": 0.9249999582767486,
|
|
"rewards/reward_fn/std": 0.30596343576908114,
|
|
"step": 1230,
|
|
"step_time": 24.858759614871815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 614.1,
|
|
"completions/max_terminated_length": 498.9,
|
|
"completions/mean_length": 273.8625,
|
|
"completions/mean_terminated_length": 264.3100021362305,
|
|
"completions/min_length": 131.8,
|
|
"completions/min_terminated_length": 131.8,
|
|
"entropy": 0.5003731534816325,
|
|
"epoch": 0.992,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 2.952770471572876,
|
|
"learning_rate": 4.4e-09,
|
|
"loss": 0.05144896507263184,
|
|
"num_tokens": 4923236.0,
|
|
"reward": 1.0687499582767486,
|
|
"reward_std": 0.3899788945913315,
|
|
"rewards/reward_fn/mean": 1.0687499582767486,
|
|
"rewards/reward_fn/std": 0.3899789124727249,
|
|
"step": 1240,
|
|
"step_time": 27.80349059002474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 493.7,
|
|
"completions/max_terminated_length": 493.7,
|
|
"completions/mean_length": 250.225,
|
|
"completions/mean_terminated_length": 250.225,
|
|
"completions/min_length": 87.9,
|
|
"completions/min_terminated_length": 87.9,
|
|
"entropy": 0.5107808768749237,
|
|
"epoch": 1.0,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4e-10,
|
|
"loss": 0.010706515610218048,
|
|
"num_tokens": 4971516.0,
|
|
"reward": 1.0812499701976777,
|
|
"reward_std": 0.31392850875854494,
|
|
"rewards/reward_fn/mean": 1.0812499701976777,
|
|
"rewards/reward_fn/std": 0.31392852365970614,
|
|
"step": 1250,
|
|
"step_time": 22.25712463380769
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1250,
|
|
"num_input_tokens_seen": 4971516,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|