8435 lines
314 KiB
JSON
8435 lines
314 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.9620667102681493,
|
|
"eval_steps": 500,
|
|
"global_step": 3000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.49000000953674316,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 372.8,
|
|
"completions/mean_length": 291.7250061035156,
|
|
"completions/mean_terminated_length": 204.1207077026367,
|
|
"completions/min_length": 15.7,
|
|
"completions/min_terminated_length": 15.7,
|
|
"entropy": 1.5640255227684974,
|
|
"epoch": 0.006540222367560497,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5129233002662659,
|
|
"kl": 0.0005587212668615394,
|
|
"learning_rate": 4.985284499672989e-07,
|
|
"loss": -0.04552017748355865,
|
|
"num_tokens": 324400.0,
|
|
"reward": -2.107620294111712e-06,
|
|
"reward_std": 0.9998070120811462,
|
|
"rewards/rnd_reward/mean": -2.107620250058062e-06,
|
|
"rewards/rnd_reward/std": 0.9998070061206817,
|
|
"step": 10,
|
|
"step_time": 9.577712386706844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5312500178813935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 374.2,
|
|
"completions/mean_length": 293.2887481689453,
|
|
"completions/mean_terminated_length": 190.75759887695312,
|
|
"completions/min_length": 18.8,
|
|
"completions/min_terminated_length": 18.8,
|
|
"entropy": 1.451838418841362,
|
|
"epoch": 0.013080444735120994,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.47466912865638733,
|
|
"kl": 0.000643831241177395,
|
|
"learning_rate": 4.968933943754087e-07,
|
|
"loss": 0.00032075121998786926,
|
|
"num_tokens": 647161.0,
|
|
"reward": -1.490712192975252e-06,
|
|
"reward_std": 0.9998168766498565,
|
|
"rewards/rnd_reward/mean": -1.4907121759222264e-06,
|
|
"rewards/rnd_reward/std": 0.9998168766498565,
|
|
"step": 20,
|
|
"step_time": 9.337269740481862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5450000017881393,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 368.6,
|
|
"completions/mean_length": 298.80250549316406,
|
|
"completions/mean_terminated_length": 196.99718322753907,
|
|
"completions/min_length": 20.3,
|
|
"completions/min_terminated_length": 20.3,
|
|
"entropy": 1.5065956979990005,
|
|
"epoch": 0.01962066710268149,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5856828689575195,
|
|
"kl": 0.0006675069424090907,
|
|
"learning_rate": 4.952583387835186e-07,
|
|
"loss": 0.02223682850599289,
|
|
"num_tokens": 982698.0,
|
|
"reward": -1.2338162438396694e-07,
|
|
"reward_std": 0.9998676717281342,
|
|
"rewards/rnd_reward/mean": -1.233816414369926e-07,
|
|
"rewards/rnd_reward/std": 0.9998676896095275,
|
|
"step": 30,
|
|
"step_time": 9.97233030446805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5125000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 371.2,
|
|
"completions/mean_length": 287.2925048828125,
|
|
"completions/mean_terminated_length": 185.24396667480468,
|
|
"completions/min_length": 10.5,
|
|
"completions/min_terminated_length": 10.5,
|
|
"entropy": 1.6533239781856537,
|
|
"epoch": 0.026160889470241987,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5665533542633057,
|
|
"kl": 0.0007774221347062848,
|
|
"learning_rate": 4.936232831916285e-07,
|
|
"loss": 0.06135546565055847,
|
|
"num_tokens": 1303862.0,
|
|
"reward": -6.012618625206301e-07,
|
|
"reward_std": 0.9998966813087463,
|
|
"rewards/rnd_reward/mean": -6.012618571915596e-07,
|
|
"rewards/rnd_reward/std": 0.9998966932296753,
|
|
"step": 40,
|
|
"step_time": 9.489675145596266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.46375001072883604,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 377.3,
|
|
"completions/mean_length": 269.5550064086914,
|
|
"completions/mean_terminated_length": 170.72807769775392,
|
|
"completions/min_length": 7.2,
|
|
"completions/min_terminated_length": 7.2,
|
|
"entropy": 1.7108511567115783,
|
|
"epoch": 0.032701111837802485,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7200465798377991,
|
|
"kl": 0.0023742157864035107,
|
|
"learning_rate": 4.919882275997384e-07,
|
|
"loss": 0.11103466749191285,
|
|
"num_tokens": 1618921.0,
|
|
"reward": -8.426606612488285e-08,
|
|
"reward_std": 0.9999245047569275,
|
|
"rewards/rnd_reward/mean": -8.426606257216918e-08,
|
|
"rewards/rnd_reward/std": 0.999924510717392,
|
|
"step": 50,
|
|
"step_time": 10.167311429185792
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4012500047683716,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.1,
|
|
"completions/mean_length": 223.41500549316407,
|
|
"completions/mean_terminated_length": 115.63493576049805,
|
|
"completions/min_length": 2.8,
|
|
"completions/min_terminated_length": 2.8,
|
|
"entropy": 2.521006014943123,
|
|
"epoch": 0.03924133420536298,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.70113205909729,
|
|
"kl": 0.020581040543038397,
|
|
"learning_rate": 4.903531720078482e-07,
|
|
"loss": 0.17053321599960328,
|
|
"num_tokens": 1894883.0,
|
|
"reward": -1.1384487108578867e-07,
|
|
"reward_std": 0.9999224483966828,
|
|
"rewards/rnd_reward/mean": -1.1384487108578867e-07,
|
|
"rewards/rnd_reward/std": 0.9999224364757537,
|
|
"step": 60,
|
|
"step_time": 9.91588868314866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3012500047683716,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 373.8,
|
|
"completions/mean_length": 183.6437515258789,
|
|
"completions/mean_terminated_length": 97.22235260009765,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7396299123764036,
|
|
"epoch": 0.04578155657292348,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7759609222412109,
|
|
"kl": 0.0463468745816499,
|
|
"learning_rate": 4.887181164159581e-07,
|
|
"loss": 0.14934378862380981,
|
|
"num_tokens": 2138653.0,
|
|
"reward": -4.6938660425155375e-08,
|
|
"reward_std": 0.9998835921287537,
|
|
"rewards/rnd_reward/mean": -4.6938658648798536e-08,
|
|
"rewards/rnd_reward/std": 0.9998836040496826,
|
|
"step": 70,
|
|
"step_time": 10.02495912197046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2825000062584877,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.8,
|
|
"completions/mean_length": 176.12750549316405,
|
|
"completions/mean_terminated_length": 94.6318603515625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7325789600610735,
|
|
"epoch": 0.052321778940483975,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8319016695022583,
|
|
"kl": 0.04594853101298213,
|
|
"learning_rate": 4.87083060824068e-07,
|
|
"loss": 0.018015310168266296,
|
|
"num_tokens": 2376420.0,
|
|
"reward": -4.187226378959963e-08,
|
|
"reward_std": 0.9998487293720245,
|
|
"rewards/rnd_reward/mean": -4.1872263079056894e-08,
|
|
"rewards/rnd_reward/std": 0.9998487293720245,
|
|
"step": 80,
|
|
"step_time": 9.701193272857926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.34750000238418577,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.4,
|
|
"completions/mean_length": 207.52375335693358,
|
|
"completions/mean_terminated_length": 114.01394119262696,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.342811721563339,
|
|
"epoch": 0.058862001308044476,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5935761332511902,
|
|
"kl": 0.019986469263676553,
|
|
"learning_rate": 4.854480052321779e-07,
|
|
"loss": 0.010216768831014633,
|
|
"num_tokens": 2641029.0,
|
|
"reward": -2.0414591972439666e-08,
|
|
"reward_std": 0.9998277425765991,
|
|
"rewards/rnd_reward/mean": -2.0414591972439666e-08,
|
|
"rewards/rnd_reward/std": 0.9998277485370636,
|
|
"step": 90,
|
|
"step_time": 10.468820211617276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4150000065565109,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.7,
|
|
"completions/mean_length": 237.4625045776367,
|
|
"completions/mean_terminated_length": 134.2795196533203,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 2.1418308436870577,
|
|
"epoch": 0.06540222367560497,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5613327622413635,
|
|
"kl": 0.006808469316456467,
|
|
"learning_rate": 4.838129496402878e-07,
|
|
"loss": 0.04351484477519989,
|
|
"num_tokens": 2930639.0,
|
|
"reward": 8.493661918240747e-09,
|
|
"reward_std": 0.9997802138328552,
|
|
"rewards/rnd_reward/mean": 8.493660852426644e-09,
|
|
"rewards/rnd_reward/std": 0.9997801959514618,
|
|
"step": 100,
|
|
"step_time": 10.307341906265355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.38875000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 370.8,
|
|
"completions/mean_length": 234.60625610351562,
|
|
"completions/mean_terminated_length": 139.61359405517578,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.0046547770500185,
|
|
"epoch": 0.07194244604316546,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7055774927139282,
|
|
"kl": 0.006759516580495983,
|
|
"learning_rate": 4.821778940483976e-07,
|
|
"loss": 0.0377700001001358,
|
|
"num_tokens": 3215169.0,
|
|
"reward": -2.920627686364696e-08,
|
|
"reward_std": 0.9997233927249909,
|
|
"rewards/rnd_reward/mean": -2.9206276153104228e-08,
|
|
"rewards/rnd_reward/std": 0.9997233927249909,
|
|
"step": 110,
|
|
"step_time": 9.7759924543323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.38250001072883605,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 364.8,
|
|
"completions/mean_length": 221.19375457763672,
|
|
"completions/mean_terminated_length": 120.60059127807617,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.0983517467975616,
|
|
"epoch": 0.07848266841072596,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5727905631065369,
|
|
"kl": 0.009989026037510484,
|
|
"learning_rate": 4.805428384565075e-07,
|
|
"loss": 0.09626840949058532,
|
|
"num_tokens": 3496559.0,
|
|
"reward": 1.654028882747127e-08,
|
|
"reward_std": 0.9997243106365203,
|
|
"rewards/rnd_reward/mean": 1.654028882747127e-08,
|
|
"rewards/rnd_reward/std": 0.9997243225574494,
|
|
"step": 120,
|
|
"step_time": 9.981053173542023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.34750000536441805,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 370.4,
|
|
"completions/mean_length": 207.67625274658204,
|
|
"completions/mean_terminated_length": 114.74080200195313,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2568650126457213,
|
|
"epoch": 0.08502289077828647,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8493079543113708,
|
|
"kl": 0.014032764779403805,
|
|
"learning_rate": 4.789077828646174e-07,
|
|
"loss": -0.015352001786231995,
|
|
"num_tokens": 3759245.0,
|
|
"reward": -5.06639494801675e-09,
|
|
"reward_std": 0.9996613085269928,
|
|
"rewards/rnd_reward/mean": -5.066393526931279e-09,
|
|
"rewards/rnd_reward/std": 0.9996613144874573,
|
|
"step": 130,
|
|
"step_time": 10.069910727906972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3137500062584877,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 373.4,
|
|
"completions/mean_length": 196.95000305175782,
|
|
"completions/mean_terminated_length": 111.34912643432617,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.305853408575058,
|
|
"epoch": 0.09156311314584696,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7982010245323181,
|
|
"kl": 0.016896341042593122,
|
|
"learning_rate": 4.772727272727273e-07,
|
|
"loss": 0.02038821578025818,
|
|
"num_tokens": 4010260.0,
|
|
"reward": -6.914138950264714e-08,
|
|
"reward_std": 0.9996093094348908,
|
|
"rewards/rnd_reward/mean": -6.914138808156168e-08,
|
|
"rewards/rnd_reward/std": 0.9996093094348908,
|
|
"step": 140,
|
|
"step_time": 9.98037786844652
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3700000047683716,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.3,
|
|
"completions/mean_length": 212.5687515258789,
|
|
"completions/mean_terminated_length": 111.88678207397462,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2448606371879576,
|
|
"epoch": 0.09810333551340746,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6012561321258545,
|
|
"kl": 0.01302192104049027,
|
|
"learning_rate": 4.756376716808371e-07,
|
|
"loss": 0.041692107915878296,
|
|
"num_tokens": 4270360.0,
|
|
"reward": -6.854534007061375e-09,
|
|
"reward_std": 0.9995533764362335,
|
|
"rewards/rnd_reward/mean": -6.854535428146846e-09,
|
|
"rewards/rnd_reward/std": 0.9995533883571625,
|
|
"step": 150,
|
|
"step_time": 9.479559903079643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3687500059604645,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 373.0,
|
|
"completions/mean_length": 221.59500274658203,
|
|
"completions/mean_terminated_length": 127.51908645629882,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2110244154930117,
|
|
"epoch": 0.10464355788096795,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7059744000434875,
|
|
"kl": 0.009931368473917245,
|
|
"learning_rate": 4.74002616088947e-07,
|
|
"loss": 0.0420710027217865,
|
|
"num_tokens": 4538671.0,
|
|
"reward": -6.467104074481256e-08,
|
|
"reward_std": 0.9995002686977387,
|
|
"rewards/rnd_reward/mean": -6.467103932372709e-08,
|
|
"rewards/rnd_reward/std": 0.9995002567768096,
|
|
"step": 160,
|
|
"step_time": 9.463879713765346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.38750000596046447,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 366.5,
|
|
"completions/mean_length": 224.09500274658203,
|
|
"completions/mean_terminated_length": 123.22678070068359,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.105703830718994,
|
|
"epoch": 0.11118378024852844,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6353176236152649,
|
|
"kl": 0.009572444634977728,
|
|
"learning_rate": 4.723675604970569e-07,
|
|
"loss": 0.057880669832229614,
|
|
"num_tokens": 4810167.0,
|
|
"reward": -1.3113021868349506e-08,
|
|
"reward_std": 0.9995246767997742,
|
|
"rewards/rnd_reward/mean": -1.3113020802535403e-08,
|
|
"rewards/rnd_reward/std": 0.9995246648788452,
|
|
"step": 170,
|
|
"step_time": 9.689974679402075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3425000041723251,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.8,
|
|
"completions/mean_length": 207.19125213623047,
|
|
"completions/mean_terminated_length": 115.52461242675781,
|
|
"completions/min_length": 2.7,
|
|
"completions/min_terminated_length": 2.7,
|
|
"entropy": 2.3119001001119615,
|
|
"epoch": 0.11772400261608895,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6543518900871277,
|
|
"kl": 0.01470447750762105,
|
|
"learning_rate": 4.7073250490516674e-07,
|
|
"loss": 0.05609138607978821,
|
|
"num_tokens": 5069705.0,
|
|
"reward": -3.21865089580875e-08,
|
|
"reward_std": 0.9993573248386383,
|
|
"rewards/rnd_reward/mean": -3.218650753700203e-08,
|
|
"rewards/rnd_reward/std": 0.9993573546409606,
|
|
"step": 180,
|
|
"step_time": 9.860351232904941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.29625000804662704,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.4,
|
|
"completions/mean_length": 186.85875396728517,
|
|
"completions/mean_terminated_length": 103.74458618164063,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.3917055457830427,
|
|
"epoch": 0.12426422498364945,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6832460165023804,
|
|
"kl": 0.02189517344813794,
|
|
"learning_rate": 4.690974493132766e-07,
|
|
"loss": 0.0940218448638916,
|
|
"num_tokens": 5313337.0,
|
|
"reward": -2.801418297337932e-08,
|
|
"reward_std": 0.9993866741657257,
|
|
"rewards/rnd_reward/mean": -2.801418386155774e-08,
|
|
"rewards/rnd_reward/std": 0.9993866682052612,
|
|
"step": 190,
|
|
"step_time": 10.32580083089415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3087500035762787,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.0,
|
|
"completions/mean_length": 195.6250030517578,
|
|
"completions/mean_terminated_length": 111.06491088867188,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.365246319770813,
|
|
"epoch": 0.13080444735120994,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7736798524856567,
|
|
"kl": 0.018843610235489906,
|
|
"learning_rate": 4.674623937213865e-07,
|
|
"loss": 0.09085838794708252,
|
|
"num_tokens": 5564832.0,
|
|
"reward": -3.784895152847412e-08,
|
|
"reward_std": 0.9993082165718079,
|
|
"rewards/rnd_reward/mean": -3.7848948331031805e-08,
|
|
"rewards/rnd_reward/std": 0.9993082165718079,
|
|
"step": 200,
|
|
"step_time": 10.074973361729644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2800000086426735,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 364.2,
|
|
"completions/mean_length": 183.75125274658203,
|
|
"completions/mean_terminated_length": 105.59480438232421,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3340355485677717,
|
|
"epoch": 0.13734466971877043,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7310492992401123,
|
|
"kl": 0.019957996485754848,
|
|
"learning_rate": 4.6582733812949637e-07,
|
|
"loss": 0.09393368363380432,
|
|
"num_tokens": 5808593.0,
|
|
"reward": -7.003545956507651e-08,
|
|
"reward_std": 0.9992451012134552,
|
|
"rewards/rnd_reward/mean": -7.003545814399103e-08,
|
|
"rewards/rnd_reward/std": 0.9992450892925262,
|
|
"step": 210,
|
|
"step_time": 9.864037699718029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2600000038743019,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 367.5,
|
|
"completions/mean_length": 182.30500335693358,
|
|
"completions/mean_terminated_length": 111.45323791503907,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3039004147052764,
|
|
"epoch": 0.14388489208633093,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7140376567840576,
|
|
"kl": 0.020338369254022836,
|
|
"learning_rate": 4.641922825376063e-07,
|
|
"loss": 0.03314664363861084,
|
|
"num_tokens": 6039847.0,
|
|
"reward": -4.202127490415819e-08,
|
|
"reward_std": 0.9992108285427094,
|
|
"rewards/rnd_reward/mean": -4.202127419361546e-08,
|
|
"rewards/rnd_reward/std": 0.9992108345031738,
|
|
"step": 220,
|
|
"step_time": 9.367073794314638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2750000059604645,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.2,
|
|
"completions/mean_length": 176.20750274658204,
|
|
"completions/mean_terminated_length": 97.27564849853516,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.2686150133609773,
|
|
"epoch": 0.15042511445389142,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8464763760566711,
|
|
"kl": 0.021972918952815234,
|
|
"learning_rate": 4.6255722694571613e-07,
|
|
"loss": 0.04784287512302399,
|
|
"num_tokens": 6270078.0,
|
|
"reward": -1.1757016302738065e-07,
|
|
"reward_std": 0.9992310464382171,
|
|
"rewards/rnd_reward/mean": -1.1757016231683792e-07,
|
|
"rewards/rnd_reward/std": 0.9992310464382171,
|
|
"step": 230,
|
|
"step_time": 9.406287361658178
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.23750000298023224,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.0,
|
|
"completions/mean_length": 164.8237548828125,
|
|
"completions/mean_terminated_length": 96.10623779296876,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4642169535160066,
|
|
"epoch": 0.15696533682145192,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.874134361743927,
|
|
"kl": 0.027495546033605933,
|
|
"learning_rate": 4.60922171353826e-07,
|
|
"loss": 0.09257912039756774,
|
|
"num_tokens": 6507582.0,
|
|
"reward": -2.2351741080228748e-08,
|
|
"reward_std": 0.9989596605300903,
|
|
"rewards/rnd_reward/mean": -2.2351743211856957e-08,
|
|
"rewards/rnd_reward/std": 0.9989596605300903,
|
|
"step": 240,
|
|
"step_time": 10.88162840725854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.26375000476837157,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 368.7,
|
|
"completions/mean_length": 179.21375274658203,
|
|
"completions/mean_terminated_length": 105.97551345825195,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.3090430706739427,
|
|
"epoch": 0.16350555918901244,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7480731010437012,
|
|
"kl": 0.022477348684333265,
|
|
"learning_rate": 4.592871157619359e-07,
|
|
"loss": 0.0661897599697113,
|
|
"num_tokens": 6746233.0,
|
|
"reward": -8.285045609568442e-08,
|
|
"reward_std": 0.9989501237869263,
|
|
"rewards/rnd_reward/mean": -8.285045680622715e-08,
|
|
"rewards/rnd_reward/std": 0.9989501178264618,
|
|
"step": 250,
|
|
"step_time": 9.837229801248759
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.23000000417232513,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.9,
|
|
"completions/mean_length": 166.79125366210937,
|
|
"completions/mean_terminated_length": 101.81034469604492,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3430103093385695,
|
|
"epoch": 0.17004578155657293,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7986056804656982,
|
|
"kl": 0.026151024643331765,
|
|
"learning_rate": 4.5765206017004576e-07,
|
|
"loss": 0.1205091118812561,
|
|
"num_tokens": 6976091.0,
|
|
"reward": -7.808208817294826e-08,
|
|
"reward_std": 0.9988620162010193,
|
|
"rewards/rnd_reward/mean": -7.808208604132006e-08,
|
|
"rewards/rnd_reward/std": 0.9988620281219482,
|
|
"step": 260,
|
|
"step_time": 9.811681813630276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.23125000298023224,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.4,
|
|
"completions/mean_length": 162.19500274658202,
|
|
"completions/mean_terminated_length": 95.31229019165039,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2510120779275895,
|
|
"epoch": 0.17658600392413343,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.6842794418334961,
|
|
"kl": 0.025852510519325734,
|
|
"learning_rate": 4.5601700457815564e-07,
|
|
"loss": 0.07729284763336182,
|
|
"num_tokens": 7199527.0,
|
|
"reward": -7.122755136812487e-08,
|
|
"reward_std": 0.9989409029483796,
|
|
"rewards/rnd_reward/mean": -7.122754994703939e-08,
|
|
"rewards/rnd_reward/std": 0.9989409029483796,
|
|
"step": 270,
|
|
"step_time": 9.695072391931898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.24750000536441802,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.6,
|
|
"completions/mean_length": 165.32125244140624,
|
|
"completions/mean_terminated_length": 93.36570816040039,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2813161820173264,
|
|
"epoch": 0.18312622629169392,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8117495775222778,
|
|
"kl": 0.02381342255976051,
|
|
"learning_rate": 4.543819489862655e-07,
|
|
"loss": 0.08756870031356812,
|
|
"num_tokens": 7423314.0,
|
|
"reward": -3.010034372863402e-08,
|
|
"reward_std": 0.9987568318843841,
|
|
"rewards/rnd_reward/mean": -3.0100344261541065e-08,
|
|
"rewards/rnd_reward/std": 0.9987568318843841,
|
|
"step": 280,
|
|
"step_time": 9.538212020602078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.27250000536441804,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.5,
|
|
"completions/mean_length": 171.89125366210936,
|
|
"completions/mean_terminated_length": 91.7330307006836,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.2805801182985306,
|
|
"epoch": 0.18966644865925442,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7356550693511963,
|
|
"kl": 0.021292086993344127,
|
|
"learning_rate": 4.527468933943754e-07,
|
|
"loss": 0.0787746787071228,
|
|
"num_tokens": 7649017.0,
|
|
"reward": -2.2798775312082853e-08,
|
|
"reward_std": 0.9987796723842621,
|
|
"rewards/rnd_reward/mean": -2.279877513444717e-08,
|
|
"rewards/rnd_reward/std": 0.99877969622612,
|
|
"step": 290,
|
|
"step_time": 9.4448504867265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 368.4,
|
|
"completions/mean_length": 172.06375274658203,
|
|
"completions/mean_terminated_length": 101.12049560546875,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.062102970480919,
|
|
"epoch": 0.1962066710268149,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.678175151348114,
|
|
"kl": 0.021230081818066536,
|
|
"learning_rate": 4.511118378024852e-07,
|
|
"loss": 0.10422745943069459,
|
|
"num_tokens": 7893248.0,
|
|
"reward": 1.7285347131412722e-08,
|
|
"reward_std": 0.9987858593463897,
|
|
"rewards/rnd_reward/mean": 1.7285347841955455e-08,
|
|
"rewards/rnd_reward/std": 0.9987858593463897,
|
|
"step": 300,
|
|
"step_time": 11.542911725770683
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.23125,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.3,
|
|
"completions/mean_length": 155.94375305175782,
|
|
"completions/mean_terminated_length": 87.63837699890136,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.110625982284546,
|
|
"epoch": 0.2027468933943754,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8758037090301514,
|
|
"kl": 0.025008385512046517,
|
|
"learning_rate": 4.4947678221059515e-07,
|
|
"loss": 0.0913781225681305,
|
|
"num_tokens": 8105838.0,
|
|
"reward": -6.72042411054008e-08,
|
|
"reward_std": 0.9986489951610565,
|
|
"rewards/rnd_reward/mean": -6.720423826322985e-08,
|
|
"rewards/rnd_reward/std": 0.9986490130424499,
|
|
"step": 310,
|
|
"step_time": 9.397380503430032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.19500000327825545,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.1,
|
|
"completions/mean_length": 141.38000259399413,
|
|
"completions/mean_terminated_length": 82.59085083007812,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.311973440647125,
|
|
"epoch": 0.2092871157619359,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9461565613746643,
|
|
"kl": 0.03633355665951967,
|
|
"learning_rate": 4.4784172661870503e-07,
|
|
"loss": 0.08147850632667542,
|
|
"num_tokens": 8325122.0,
|
|
"reward": -9.238719300697084e-09,
|
|
"reward_std": 0.9986236810684204,
|
|
"rewards/rnd_reward/mean": -9.238719300697084e-09,
|
|
"rewards/rnd_reward/std": 0.9986236929893494,
|
|
"step": 320,
|
|
"step_time": 11.082472172076814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.20000000447034835,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.7,
|
|
"completions/mean_length": 141.80000228881835,
|
|
"completions/mean_terminated_length": 81.46792144775391,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3696460843086244,
|
|
"epoch": 0.2158273381294964,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8129741549491882,
|
|
"kl": 0.034535441920161246,
|
|
"learning_rate": 4.462066710268149e-07,
|
|
"loss": 0.06801753044128418,
|
|
"num_tokens": 8535267.0,
|
|
"reward": -8.24779284869237e-08,
|
|
"reward_std": 0.9986066579818725,
|
|
"rewards/rnd_reward/mean": -8.24779284869237e-08,
|
|
"rewards/rnd_reward/std": 0.9986066579818725,
|
|
"step": 330,
|
|
"step_time": 9.98545232352335
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2100000038743019,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 364.4,
|
|
"completions/mean_length": 141.52500228881837,
|
|
"completions/mean_terminated_length": 77.51832160949706,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3277006804943086,
|
|
"epoch": 0.2223675604970569,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8912848234176636,
|
|
"kl": 0.0404030725825578,
|
|
"learning_rate": 4.445716154349248e-07,
|
|
"loss": 0.09486417174339294,
|
|
"num_tokens": 8741817.0,
|
|
"reward": -1.6838312300038183e-08,
|
|
"reward_std": 0.9984602987766266,
|
|
"rewards/rnd_reward/mean": -1.683831301058092e-08,
|
|
"rewards/rnd_reward/std": 0.9984602928161621,
|
|
"step": 340,
|
|
"step_time": 9.806188964121976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1912500038743019,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.6,
|
|
"completions/mean_length": 139.37500076293946,
|
|
"completions/mean_terminated_length": 81.56527061462403,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.3621277123689652,
|
|
"epoch": 0.2289077828646174,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9253888130187988,
|
|
"kl": 0.03642263817600906,
|
|
"learning_rate": 4.429365598430346e-07,
|
|
"loss": 0.09143019318580628,
|
|
"num_tokens": 8945182.0,
|
|
"reward": -2.4735924242236252e-08,
|
|
"reward_std": 0.9984571993350982,
|
|
"rewards/rnd_reward/mean": -2.4735929926578137e-08,
|
|
"rewards/rnd_reward/std": 0.9984571933746338,
|
|
"step": 350,
|
|
"step_time": 9.939818266802467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.18750000447034837,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 371.7,
|
|
"completions/mean_length": 138.01250076293945,
|
|
"completions/mean_terminated_length": 81.37003707885742,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.270884335041046,
|
|
"epoch": 0.2354480052321779,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8366574645042419,
|
|
"kl": 0.03530883879866451,
|
|
"learning_rate": 4.4130150425114454e-07,
|
|
"loss": 0.10644701719284058,
|
|
"num_tokens": 9147587.0,
|
|
"reward": -1.15185978266652e-07,
|
|
"reward_std": 0.9983633041381836,
|
|
"rewards/rnd_reward/mean": -1.1518597684556652e-07,
|
|
"rewards/rnd_reward/std": 0.9983633100986481,
|
|
"step": 360,
|
|
"step_time": 9.722514500608668
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.17000000029802323,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 366.5,
|
|
"completions/mean_length": 130.59125213623048,
|
|
"completions/mean_terminated_length": 79.17214775085449,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3677036464214325,
|
|
"epoch": 0.2419882275997384,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.964614987373352,
|
|
"kl": 0.043067421624436976,
|
|
"learning_rate": 4.3966644865925437e-07,
|
|
"loss": 0.08349081873893738,
|
|
"num_tokens": 9340010.0,
|
|
"reward": -3.0696392450124675e-08,
|
|
"reward_std": 0.9982355415821076,
|
|
"rewards/rnd_reward/mean": -3.0696389607953736e-08,
|
|
"rewards/rnd_reward/std": 0.9982355415821076,
|
|
"step": 370,
|
|
"step_time": 9.710164245427586
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1887500010430813,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 367.2,
|
|
"completions/mean_length": 138.41875305175782,
|
|
"completions/mean_terminated_length": 81.47324066162109,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.258895403146744,
|
|
"epoch": 0.2485284499672989,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7839703559875488,
|
|
"kl": 0.039375259960070255,
|
|
"learning_rate": 4.3803139306736425e-07,
|
|
"loss": 0.08618925213813781,
|
|
"num_tokens": 9545205.0,
|
|
"reward": -1.0460615040841503e-07,
|
|
"reward_std": 0.998303747177124,
|
|
"rewards/rnd_reward/mean": -1.046061518295005e-07,
|
|
"rewards/rnd_reward/std": 0.9983037531375885,
|
|
"step": 380,
|
|
"step_time": 9.776164471404627
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1625000037252903,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 342.9,
|
|
"completions/mean_length": 123.14125137329101,
|
|
"completions/mean_terminated_length": 72.26187210083008,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2043879717588424,
|
|
"epoch": 0.25506867233485936,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.013533353805542,
|
|
"kl": 0.045199727965518834,
|
|
"learning_rate": 4.363963374754742e-07,
|
|
"loss": 0.08801217675209046,
|
|
"num_tokens": 9732323.0,
|
|
"reward": -1.3418495639072603e-07,
|
|
"reward_std": 0.9981010854244232,
|
|
"rewards/rnd_reward/mean": -1.3418495496964057e-07,
|
|
"rewards/rnd_reward/std": 0.9981010735034943,
|
|
"step": 390,
|
|
"step_time": 9.470399480173365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16500000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.9,
|
|
"completions/mean_length": 126.54125213623047,
|
|
"completions/mean_terminated_length": 75.62271652221679,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.1946812689304354,
|
|
"epoch": 0.2616088947024199,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9289289712905884,
|
|
"kl": 0.043896416388452056,
|
|
"learning_rate": 4.34761281883584e-07,
|
|
"loss": 0.06013514399528504,
|
|
"num_tokens": 9923251.0,
|
|
"reward": -1.1950731622079046e-07,
|
|
"reward_std": 0.9980400562286377,
|
|
"rewards/rnd_reward/mean": -1.19507314799705e-07,
|
|
"rewards/rnd_reward/std": 0.9980400741100312,
|
|
"step": 400,
|
|
"step_time": 9.578276086249389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.19000000283122062,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.4,
|
|
"completions/mean_length": 142.2825019836426,
|
|
"completions/mean_terminated_length": 85.19309997558594,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3042476952075956,
|
|
"epoch": 0.2681491170699804,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8114489316940308,
|
|
"kl": 0.039571146154776214,
|
|
"learning_rate": 4.331262262916939e-07,
|
|
"loss": 0.08236328959465027,
|
|
"num_tokens": 10132682.0,
|
|
"reward": -1.4752151500374566e-08,
|
|
"reward_std": 0.9982209146022797,
|
|
"rewards/rnd_reward/mean": -1.4752152210917303e-08,
|
|
"rewards/rnd_reward/std": 0.9982209086418152,
|
|
"step": 410,
|
|
"step_time": 9.807740885205567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.20625000149011613,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 149.2062530517578,
|
|
"completions/mean_terminated_length": 88.13671951293945,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.38392159640789,
|
|
"epoch": 0.27468933943754087,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7539530396461487,
|
|
"kl": 0.03540583737194538,
|
|
"learning_rate": 4.3149117069980376e-07,
|
|
"loss": 0.08869684934616089,
|
|
"num_tokens": 10345522.0,
|
|
"reward": -1.3530254712179611e-07,
|
|
"reward_std": 0.9981609523296356,
|
|
"rewards/rnd_reward/mean": -1.3530254356908243e-07,
|
|
"rewards/rnd_reward/std": 0.9981609761714936,
|
|
"step": 420,
|
|
"step_time": 9.619695667061023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.18750000223517418,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.4,
|
|
"completions/mean_length": 139.9700035095215,
|
|
"completions/mean_terminated_length": 83.52344436645508,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.225811779499054,
|
|
"epoch": 0.2812295618051014,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7882169485092163,
|
|
"kl": 0.031479011243209246,
|
|
"learning_rate": 4.2985611510791364e-07,
|
|
"loss": 0.09766495227813721,
|
|
"num_tokens": 10563093.0,
|
|
"reward": -1.4021993024471158e-07,
|
|
"reward_std": 0.998052579164505,
|
|
"rewards/rnd_reward/mean": -1.4021992598145516e-07,
|
|
"rewards/rnd_reward/std": 0.9980525553226471,
|
|
"step": 430,
|
|
"step_time": 11.129851599340327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.18875000178813933,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.8,
|
|
"completions/mean_length": 143.57625198364258,
|
|
"completions/mean_terminated_length": 87.31694107055664,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.1123261958360673,
|
|
"epoch": 0.28776978417266186,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7737316489219666,
|
|
"kl": 0.0280194781254977,
|
|
"learning_rate": 4.2822105951602357e-07,
|
|
"loss": 0.10439527034759521,
|
|
"num_tokens": 10766159.0,
|
|
"reward": -4.2766332430943524e-08,
|
|
"reward_std": 0.998040771484375,
|
|
"rewards/rnd_reward/mean": -4.276633118749373e-08,
|
|
"rewards/rnd_reward/std": 0.998040747642517,
|
|
"step": 440,
|
|
"step_time": 9.481722956942395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16625000312924385,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.1,
|
|
"completions/mean_length": 131.25125427246093,
|
|
"completions/mean_terminated_length": 80.89927139282227,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.2791686803102493,
|
|
"epoch": 0.2943100065402224,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.834428608417511,
|
|
"kl": 0.04166741594672203,
|
|
"learning_rate": 4.265860039241334e-07,
|
|
"loss": 0.07646016478538513,
|
|
"num_tokens": 10965315.0,
|
|
"reward": -1.0013580258316779e-07,
|
|
"reward_std": 0.998044753074646,
|
|
"rewards/rnd_reward/mean": -1.0013580116208232e-07,
|
|
"rewards/rnd_reward/std": 0.9980447709560394,
|
|
"step": 450,
|
|
"step_time": 10.756261567212642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13750000298023224,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.9,
|
|
"completions/mean_length": 118.16625137329102,
|
|
"completions/mean_terminated_length": 75.56828384399414,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.3162699341773987,
|
|
"epoch": 0.30085022890778285,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1823770999908447,
|
|
"kl": 0.04914104142226279,
|
|
"learning_rate": 4.249509483322433e-07,
|
|
"loss": 0.09161773324012756,
|
|
"num_tokens": 11151658.0,
|
|
"reward": 1.4454126073815132e-08,
|
|
"reward_std": 0.9979428768157959,
|
|
"rewards/rnd_reward/mean": 1.4454126073815132e-08,
|
|
"rewards/rnd_reward/std": 0.9979428887367249,
|
|
"step": 460,
|
|
"step_time": 9.690541778854094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15625000596046448,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.0,
|
|
"completions/mean_length": 129.17875442504882,
|
|
"completions/mean_terminated_length": 81.90925521850586,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.250484657287598,
|
|
"epoch": 0.30739045127534337,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7849432229995728,
|
|
"kl": 0.044866825267672536,
|
|
"learning_rate": 4.2331589274035315e-07,
|
|
"loss": 0.1366190195083618,
|
|
"num_tokens": 11350266.0,
|
|
"reward": -5.528331037396583e-08,
|
|
"reward_std": 0.997910988330841,
|
|
"rewards/rnd_reward/mean": -5.528330859760899e-08,
|
|
"rewards/rnd_reward/std": 0.997911012172699,
|
|
"step": 470,
|
|
"step_time": 10.094013441796415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14375000149011613,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.1,
|
|
"completions/mean_length": 121.7000015258789,
|
|
"completions/mean_terminated_length": 77.64777450561523,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.356913906335831,
|
|
"epoch": 0.31393067364290383,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7464061975479126,
|
|
"kl": 0.047594004729762676,
|
|
"learning_rate": 4.2168083714846303e-07,
|
|
"loss": 0.093714839220047,
|
|
"num_tokens": 11546191.0,
|
|
"reward": -2.1904707381281696e-08,
|
|
"reward_std": 0.9980335652828216,
|
|
"rewards/rnd_reward/mean": -2.1904707381281696e-08,
|
|
"rewards/rnd_reward/std": 0.9980335593223572,
|
|
"step": 480,
|
|
"step_time": 10.49183724487666
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1312500022351742,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.0,
|
|
"completions/mean_length": 110.71375274658203,
|
|
"completions/mean_terminated_length": 69.09722747802735,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.3989109843969345,
|
|
"epoch": 0.32047089601046436,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9596973061561584,
|
|
"kl": 0.059090607427060605,
|
|
"learning_rate": 4.200457815565729e-07,
|
|
"loss": 0.061995261907577516,
|
|
"num_tokens": 11729672.0,
|
|
"reward": -8.441507906020718e-08,
|
|
"reward_std": 0.9977342307567596,
|
|
"rewards/rnd_reward/mean": -8.441507870493581e-08,
|
|
"rewards/rnd_reward/std": 0.9977342247962951,
|
|
"step": 490,
|
|
"step_time": 9.824798973021098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13000000193715094,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.2,
|
|
"completions/mean_length": 110.2125015258789,
|
|
"completions/mean_terminated_length": 69.26982688903809,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.335902637243271,
|
|
"epoch": 0.3270111183780249,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0347245931625366,
|
|
"kl": 0.06009262367151678,
|
|
"learning_rate": 4.184107259646828e-07,
|
|
"loss": 0.011750234663486481,
|
|
"num_tokens": 11908552.0,
|
|
"reward": -7.688999374977356e-08,
|
|
"reward_std": 0.9977168381214142,
|
|
"rewards/rnd_reward/mean": -7.688999019705988e-08,
|
|
"rewards/rnd_reward/std": 0.9977168321609498,
|
|
"step": 500,
|
|
"step_time": 9.731353901792318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1450000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.2,
|
|
"completions/mean_length": 118.08375244140625,
|
|
"completions/mean_terminated_length": 73.00995483398438,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.557803976535797,
|
|
"epoch": 0.33355134074558535,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9074205160140991,
|
|
"kl": 0.06632269411347806,
|
|
"learning_rate": 4.1677567037279267e-07,
|
|
"loss": 0.08373542428016663,
|
|
"num_tokens": 12103254.0,
|
|
"reward": -1.1697411828048843e-07,
|
|
"reward_std": 0.9978465795516968,
|
|
"rewards/rnd_reward/mean": -1.1697411828048843e-07,
|
|
"rewards/rnd_reward/std": 0.9978465795516968,
|
|
"step": 510,
|
|
"step_time": 10.207280423282645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15875000208616258,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.2,
|
|
"completions/mean_length": 123.88500137329102,
|
|
"completions/mean_terminated_length": 74.63431396484376,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.437187620997429,
|
|
"epoch": 0.34009156311314587,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8857874870300293,
|
|
"kl": 0.06048318287357688,
|
|
"learning_rate": 4.151406147809025e-07,
|
|
"loss": 0.037069684267044066,
|
|
"num_tokens": 12302092.0,
|
|
"reward": -3.874301874873254e-08,
|
|
"reward_std": 0.9979513227939606,
|
|
"rewards/rnd_reward/mean": -3.874301874873254e-08,
|
|
"rewards/rnd_reward/std": 0.9979513168334961,
|
|
"step": 520,
|
|
"step_time": 10.34260532106273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.17000000253319741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.1,
|
|
"completions/mean_length": 132.52125396728516,
|
|
"completions/mean_terminated_length": 80.95349197387695,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4997961223125458,
|
|
"epoch": 0.34663178548070633,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9028640985488892,
|
|
"kl": 0.05214450396597385,
|
|
"learning_rate": 4.135055591890124e-07,
|
|
"loss": 0.07458721995353698,
|
|
"num_tokens": 12499434.0,
|
|
"reward": -2.2247433655309124e-07,
|
|
"reward_std": 0.9978251278400421,
|
|
"rewards/rnd_reward/mean": -2.2247433655309124e-07,
|
|
"rewards/rnd_reward/std": 0.9978251218795776,
|
|
"step": 530,
|
|
"step_time": 9.664499558508396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15125000178813935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.4,
|
|
"completions/mean_length": 120.7087516784668,
|
|
"completions/mean_terminated_length": 73.79461898803712,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.547806602716446,
|
|
"epoch": 0.35317200784826686,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0128388404846191,
|
|
"kl": 0.05931666204705834,
|
|
"learning_rate": 4.118705035971223e-07,
|
|
"loss": 0.11692253351211548,
|
|
"num_tokens": 12683096.0,
|
|
"reward": -8.344649238267721e-09,
|
|
"reward_std": 0.9977494418621063,
|
|
"rewards/rnd_reward/mean": -8.344651369895927e-09,
|
|
"rewards/rnd_reward/std": 0.9977494418621063,
|
|
"step": 540,
|
|
"step_time": 9.593502512201667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13500000387430192,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.6,
|
|
"completions/mean_length": 111.18750305175782,
|
|
"completions/mean_terminated_length": 68.47658958435059,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.681893822550774,
|
|
"epoch": 0.3597122302158273,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1282111406326294,
|
|
"kl": 0.06902076229453087,
|
|
"learning_rate": 4.1023544800523213e-07,
|
|
"loss": 0.023469969630241394,
|
|
"num_tokens": 12862906.0,
|
|
"reward": -5.677342400645102e-08,
|
|
"reward_std": 0.9978136360645294,
|
|
"rewards/rnd_reward/mean": -5.6773422585365554e-08,
|
|
"rewards/rnd_reward/std": 0.997813630104065,
|
|
"step": 550,
|
|
"step_time": 9.778036265238189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12000000327825547,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.0,
|
|
"completions/mean_length": 104.53875045776367,
|
|
"completions/mean_terminated_length": 66.3183536529541,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.5951172381639482,
|
|
"epoch": 0.36625245258338784,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1439512968063354,
|
|
"kl": 0.07057485273107886,
|
|
"learning_rate": 4.0860039241334206e-07,
|
|
"loss": 0.1278893232345581,
|
|
"num_tokens": 13049132.0,
|
|
"reward": -1.1786818543413347e-07,
|
|
"reward_std": 0.9977734804153442,
|
|
"rewards/rnd_reward/mean": -1.17868184013048e-07,
|
|
"rewards/rnd_reward/std": 0.9977734982967377,
|
|
"step": 560,
|
|
"step_time": 10.389736549765804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10500000044703484,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.3,
|
|
"completions/mean_length": 99.66875076293945,
|
|
"completions/mean_terminated_length": 66.29649963378907,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6503134578466416,
|
|
"epoch": 0.3727926749509483,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0748413801193237,
|
|
"kl": 0.08892364026978612,
|
|
"learning_rate": 4.069653368214519e-07,
|
|
"loss": 0.13109040260314941,
|
|
"num_tokens": 13221157.0,
|
|
"reward": -1.4342367933295462e-07,
|
|
"reward_std": 0.997740238904953,
|
|
"rewards/rnd_reward/mean": -1.4342367791186916e-07,
|
|
"rewards/rnd_reward/std": 0.9977402448654175,
|
|
"step": 570,
|
|
"step_time": 9.71618521134369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12250000312924385,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 110.21125030517578,
|
|
"completions/mean_terminated_length": 71.826904296875,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.702208098769188,
|
|
"epoch": 0.37933289731850883,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0691598653793335,
|
|
"kl": 0.07306001400575042,
|
|
"learning_rate": 4.053302812295618e-07,
|
|
"loss": 0.09580587744712829,
|
|
"num_tokens": 13409746.0,
|
|
"reward": -1.6450882269758438e-07,
|
|
"reward_std": 0.9975189864635468,
|
|
"rewards/rnd_reward/mean": -1.6450882198704164e-07,
|
|
"rewards/rnd_reward/std": 0.9975189983844757,
|
|
"step": 580,
|
|
"step_time": 10.602098648622633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13250000216066837,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.2,
|
|
"completions/mean_length": 118.48125228881835,
|
|
"completions/mean_terminated_length": 77.98408432006836,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.5411072075366974,
|
|
"epoch": 0.3858731196860693,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8738215565681458,
|
|
"kl": 0.06685408288612962,
|
|
"learning_rate": 4.0369522563767164e-07,
|
|
"loss": 0.0531582236289978,
|
|
"num_tokens": 13603691.0,
|
|
"reward": -8.791685381925874e-08,
|
|
"reward_std": 0.9976041257381439,
|
|
"rewards/rnd_reward/mean": -8.791684955600231e-08,
|
|
"rewards/rnd_reward/std": 0.9976041257381439,
|
|
"step": 590,
|
|
"step_time": 10.606248078867793
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13375000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.1,
|
|
"completions/mean_length": 120.9775001525879,
|
|
"completions/mean_terminated_length": 80.35308418273925,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.454717567563057,
|
|
"epoch": 0.3924133420536298,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8597258925437927,
|
|
"kl": 0.06251584179699421,
|
|
"learning_rate": 4.020601700457815e-07,
|
|
"loss": 0.10086486339569092,
|
|
"num_tokens": 13793088.0,
|
|
"reward": -4.872679788547885e-08,
|
|
"reward_std": 0.997677344083786,
|
|
"rewards/rnd_reward/mean": -4.872679859602158e-08,
|
|
"rewards/rnd_reward/std": 0.997677332162857,
|
|
"step": 600,
|
|
"step_time": 9.838808755297213
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13875000327825546,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.2,
|
|
"completions/mean_length": 112.62250213623047,
|
|
"completions/mean_terminated_length": 69.04776878356934,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.5064212560653685,
|
|
"epoch": 0.39895356442119034,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0397368669509888,
|
|
"kl": 0.06775042302906513,
|
|
"learning_rate": 4.0042511445389145e-07,
|
|
"loss": 0.14560239315032958,
|
|
"num_tokens": 13975906.0,
|
|
"reward": -1.70022253698221e-07,
|
|
"reward_std": 0.997452187538147,
|
|
"rewards/rnd_reward/mean": -1.7002224872442184e-07,
|
|
"rewards/rnd_reward/std": 0.9974521696567535,
|
|
"step": 610,
|
|
"step_time": 9.711915594828316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11000000387430191,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.5,
|
|
"completions/mean_length": 105.6062515258789,
|
|
"completions/mean_terminated_length": 71.07123832702636,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.531564822793007,
|
|
"epoch": 0.4054937867887508,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8839622139930725,
|
|
"kl": 0.07682965183630586,
|
|
"learning_rate": 3.987900588620013e-07,
|
|
"loss": 0.06944795846939086,
|
|
"num_tokens": 14154656.0,
|
|
"reward": 5.662443069986978e-09,
|
|
"reward_std": 0.9975684463977814,
|
|
"rewards/rnd_reward/mean": 5.662442181808558e-09,
|
|
"rewards/rnd_reward/std": 0.9975684404373169,
|
|
"step": 620,
|
|
"step_time": 10.306970553193242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10375000201165677,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.6,
|
|
"completions/mean_length": 106.93500137329102,
|
|
"completions/mean_terminated_length": 74.97318267822266,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.859010672569275,
|
|
"epoch": 0.41203400915631133,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.103442907333374,
|
|
"kl": 0.08963548736646772,
|
|
"learning_rate": 3.9715500327011115e-07,
|
|
"loss": 0.033435261249542235,
|
|
"num_tokens": 14327599.0,
|
|
"reward": -1.2174248684715394e-07,
|
|
"reward_std": 0.9974590480327606,
|
|
"rewards/rnd_reward/mean": -1.2174248613661122e-07,
|
|
"rewards/rnd_reward/std": 0.9974590301513672,
|
|
"step": 630,
|
|
"step_time": 9.607878714823164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12625000327825547,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 343.2,
|
|
"completions/mean_length": 108.95125045776368,
|
|
"completions/mean_terminated_length": 69.31592407226563,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.823798879981041,
|
|
"epoch": 0.4185742315238718,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1582435369491577,
|
|
"kl": 0.09182370994240045,
|
|
"learning_rate": 3.9551994767822103e-07,
|
|
"loss": 0.06608177423477173,
|
|
"num_tokens": 14513650.0,
|
|
"reward": -1.3239681819765537e-07,
|
|
"reward_std": 0.9976047217845917,
|
|
"rewards/rnd_reward/mean": -1.3239681961874084e-07,
|
|
"rewards/rnd_reward/std": 0.9976047277450562,
|
|
"step": 640,
|
|
"step_time": 10.307288937154226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11000000163912774,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.8,
|
|
"completions/mean_length": 107.63500213623047,
|
|
"completions/mean_terminated_length": 73.39098167419434,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.695296034216881,
|
|
"epoch": 0.4251144538914323,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0280959606170654,
|
|
"kl": 0.09326308779418468,
|
|
"learning_rate": 3.938848920863309e-07,
|
|
"loss": 0.030906683206558226,
|
|
"num_tokens": 14693328.0,
|
|
"reward": -1.2964010291938167e-07,
|
|
"reward_std": 0.9975219249725342,
|
|
"rewards/rnd_reward/mean": -1.2964010327465302e-07,
|
|
"rewards/rnd_reward/std": 0.9975219309329987,
|
|
"step": 650,
|
|
"step_time": 10.01973620983772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1512500040233135,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.3,
|
|
"completions/mean_length": 129.62250213623048,
|
|
"completions/mean_terminated_length": 84.16030578613281,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.613438582420349,
|
|
"epoch": 0.4316546762589928,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9445599317550659,
|
|
"kl": 0.0699499910697341,
|
|
"learning_rate": 3.922498364944408e-07,
|
|
"loss": 0.08974806666374206,
|
|
"num_tokens": 14892686.0,
|
|
"reward": -4.194676717084178e-08,
|
|
"reward_std": 0.9975471794605255,
|
|
"rewards/rnd_reward/mean": -4.1946770057421645e-08,
|
|
"rewards/rnd_reward/std": 0.99754718542099,
|
|
"step": 660,
|
|
"step_time": 10.161354359728284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14750000312924386,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.7,
|
|
"completions/mean_length": 126.59500045776367,
|
|
"completions/mean_terminated_length": 81.99441680908203,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.7322494119405745,
|
|
"epoch": 0.4381948986265533,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9810017943382263,
|
|
"kl": 0.08053934266790748,
|
|
"learning_rate": 3.9061478090255067e-07,
|
|
"loss": 0.14365124702453613,
|
|
"num_tokens": 15083047.0,
|
|
"reward": -7.979571803318208e-08,
|
|
"reward_std": 0.9972860395908356,
|
|
"rewards/rnd_reward/mean": -7.979571803318208e-08,
|
|
"rewards/rnd_reward/std": 0.9972860395908356,
|
|
"step": 670,
|
|
"step_time": 9.683341883006506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14250000119209288,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.5,
|
|
"completions/mean_length": 125.95000381469727,
|
|
"completions/mean_terminated_length": 83.18684883117676,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6392508417367937,
|
|
"epoch": 0.4447351209941138,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9357602596282959,
|
|
"kl": 0.07438107710331679,
|
|
"learning_rate": 3.8897972531066054e-07,
|
|
"loss": 0.047136351466178894,
|
|
"num_tokens": 15272992.0,
|
|
"reward": -1.0043382800972722e-07,
|
|
"reward_std": 0.9973943531513214,
|
|
"rewards/rnd_reward/mean": -1.0043382658864175e-07,
|
|
"rewards/rnd_reward/std": 0.997394347190857,
|
|
"step": 680,
|
|
"step_time": 9.536431295378133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14375000223517417,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.2,
|
|
"completions/mean_length": 125.81125106811524,
|
|
"completions/mean_terminated_length": 82.36017532348633,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.2503296345472337,
|
|
"epoch": 0.4512753433616743,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7484872937202454,
|
|
"kl": 0.0611060356721282,
|
|
"learning_rate": 3.873446697187704e-07,
|
|
"loss": 0.09799213409423828,
|
|
"num_tokens": 15463906.0,
|
|
"reward": -6.347894991876046e-08,
|
|
"reward_std": 0.9972564399242401,
|
|
"rewards/rnd_reward/mean": -6.34789495634891e-08,
|
|
"rewards/rnd_reward/std": 0.9972564518451691,
|
|
"step": 690,
|
|
"step_time": 9.653547045495362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15500000342726708,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.6,
|
|
"completions/mean_length": 133.82500076293945,
|
|
"completions/mean_terminated_length": 87.98081855773925,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.5280281245708465,
|
|
"epoch": 0.4578155657292348,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9658915996551514,
|
|
"kl": 0.059800021536648276,
|
|
"learning_rate": 3.857096141268803e-07,
|
|
"loss": 0.12680201530456542,
|
|
"num_tokens": 15667301.0,
|
|
"reward": -1.5869737026719122e-07,
|
|
"reward_std": 0.9973293662071228,
|
|
"rewards/rnd_reward/mean": -1.5869737026719122e-07,
|
|
"rewards/rnd_reward/std": 0.9973293721675873,
|
|
"step": 700,
|
|
"step_time": 9.966963570658118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10875000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.1,
|
|
"completions/mean_length": 107.25000228881837,
|
|
"completions/mean_terminated_length": 73.4811897277832,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4395824640989305,
|
|
"epoch": 0.4643557880967953,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.049780011177063,
|
|
"kl": 0.08286946956068278,
|
|
"learning_rate": 3.840745585349902e-07,
|
|
"loss": 0.15116369724273682,
|
|
"num_tokens": 15847671.0,
|
|
"reward": -1.005828406341891e-07,
|
|
"reward_std": 0.9973811268806457,
|
|
"rewards/rnd_reward/mean": -1.0058283921310362e-07,
|
|
"rewards/rnd_reward/std": 0.9973811089992524,
|
|
"step": 710,
|
|
"step_time": 9.890902339830063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09000000115484,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.2,
|
|
"completions/mean_length": 93.01625213623046,
|
|
"completions/mean_terminated_length": 64.09797096252441,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8066621690988542,
|
|
"epoch": 0.4708960104643558,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.34576416015625,
|
|
"kl": 0.10443261573091149,
|
|
"learning_rate": 3.8243950294310006e-07,
|
|
"loss": 0.10605430603027344,
|
|
"num_tokens": 16021059.0,
|
|
"reward": -1.0505319210096787e-07,
|
|
"reward_std": 0.9972734034061432,
|
|
"rewards/rnd_reward/mean": -1.0505318357445503e-07,
|
|
"rewards/rnd_reward/std": 0.9972733795642853,
|
|
"step": 720,
|
|
"step_time": 10.098007511859759
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000074505807,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.0,
|
|
"completions/mean_length": 94.07875137329101,
|
|
"completions/mean_terminated_length": 66.15557479858398,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6466055154800414,
|
|
"epoch": 0.4774362328319163,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0772963762283325,
|
|
"kl": 0.10096871191635728,
|
|
"learning_rate": 3.8080444735120994e-07,
|
|
"loss": 0.09338142871856689,
|
|
"num_tokens": 16191222.0,
|
|
"reward": -1.777708558137192e-07,
|
|
"reward_std": 0.9971938729286194,
|
|
"rewards/rnd_reward/mean": -1.7777085012937732e-07,
|
|
"rewards/rnd_reward/std": 0.9971938788890838,
|
|
"step": 730,
|
|
"step_time": 10.299412568542175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13375000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.1,
|
|
"completions/mean_length": 126.26250228881835,
|
|
"completions/mean_terminated_length": 86.43918380737304,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.173070195317268,
|
|
"epoch": 0.4839764551994768,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7416156530380249,
|
|
"kl": 0.053471417538821696,
|
|
"learning_rate": 3.7916939175931976e-07,
|
|
"loss": 0.07696372270584106,
|
|
"num_tokens": 16393972.0,
|
|
"reward": -1.7151236602064303e-07,
|
|
"reward_std": 0.9972269594669342,
|
|
"rewards/rnd_reward/mean": -1.715123674417285e-07,
|
|
"rewards/rnd_reward/std": 0.9972269415855408,
|
|
"step": 740,
|
|
"step_time": 10.502406498114578
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.148750002682209,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.0,
|
|
"completions/mean_length": 139.12875213623047,
|
|
"completions/mean_terminated_length": 96.77134628295899,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.325762075185776,
|
|
"epoch": 0.49051667756703726,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7352995872497559,
|
|
"kl": 0.05052672061137855,
|
|
"learning_rate": 3.775343361674297e-07,
|
|
"loss": 0.09521990418434143,
|
|
"num_tokens": 16596670.0,
|
|
"reward": -2.0787120362797397e-07,
|
|
"reward_std": 0.9972488164901734,
|
|
"rewards/rnd_reward/mean": -2.0787119723308933e-07,
|
|
"rewards/rnd_reward/std": 0.9972488105297088,
|
|
"step": 750,
|
|
"step_time": 9.698242208594456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14750000238418579,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 371.2,
|
|
"completions/mean_length": 127.9612533569336,
|
|
"completions/mean_terminated_length": 83.97822761535645,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.2959645807743074,
|
|
"epoch": 0.4970568999345978,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8263393044471741,
|
|
"kl": 0.05697637004777789,
|
|
"learning_rate": 3.7589928057553957e-07,
|
|
"loss": 0.10516003370285035,
|
|
"num_tokens": 16792629.0,
|
|
"reward": -1.6942620391091622e-07,
|
|
"reward_std": 0.9972022175788879,
|
|
"rewards/rnd_reward/mean": -1.6942620106874529e-07,
|
|
"rewards/rnd_reward/std": 0.9972022175788879,
|
|
"step": 760,
|
|
"step_time": 10.044187037739903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10875000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 339.2,
|
|
"completions/mean_length": 112.1150016784668,
|
|
"completions/mean_terminated_length": 79.13344192504883,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7552184253931045,
|
|
"epoch": 0.5035971223021583,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.029379963874817,
|
|
"kl": 0.07228619959205389,
|
|
"learning_rate": 3.742642249836494e-07,
|
|
"loss": 0.03081640601158142,
|
|
"num_tokens": 16971246.0,
|
|
"reward": -8.299946501200139e-08,
|
|
"reward_std": 0.9970763862133026,
|
|
"rewards/rnd_reward/mean": -8.299946785417234e-08,
|
|
"rewards/rnd_reward/std": 0.9970763862133026,
|
|
"step": 770,
|
|
"step_time": 9.667120404168964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13125000298023223,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.9,
|
|
"completions/mean_length": 124.3275032043457,
|
|
"completions/mean_terminated_length": 85.28903274536133,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.8021608620882033,
|
|
"epoch": 0.5101373446697187,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0165283679962158,
|
|
"kl": 0.07025760738179088,
|
|
"learning_rate": 3.7262916939175933e-07,
|
|
"loss": 0.03996403813362122,
|
|
"num_tokens": 17162308.0,
|
|
"reward": -5.170702825907369e-08,
|
|
"reward_std": 0.9971189498901367,
|
|
"rewards/rnd_reward/mean": -5.170703110124464e-08,
|
|
"rewards/rnd_reward/std": 0.9971189439296723,
|
|
"step": 780,
|
|
"step_time": 9.969513631938025
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13375000283122063,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.5,
|
|
"completions/mean_length": 127.03500213623047,
|
|
"completions/mean_terminated_length": 87.38848876953125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.637522208690643,
|
|
"epoch": 0.5166775670372793,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9886835813522339,
|
|
"kl": 0.06816753121092915,
|
|
"learning_rate": 3.7099411379986915e-07,
|
|
"loss": 0.07478119730949402,
|
|
"num_tokens": 17356536.0,
|
|
"reward": -1.1831521575800253e-07,
|
|
"reward_std": 0.9971611678600312,
|
|
"rewards/rnd_reward/mean": -1.1831521575800253e-07,
|
|
"rewards/rnd_reward/std": 0.997161191701889,
|
|
"step": 790,
|
|
"step_time": 9.748413591040299
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12625000327825547,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.3,
|
|
"completions/mean_length": 120.86375045776367,
|
|
"completions/mean_terminated_length": 83.05458145141601,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6706619918346406,
|
|
"epoch": 0.5232177894048398,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8250897526741028,
|
|
"kl": 0.07074365848675371,
|
|
"learning_rate": 3.693590582079791e-07,
|
|
"loss": 0.05061581134796143,
|
|
"num_tokens": 17545417.0,
|
|
"reward": -6.467104096685716e-08,
|
|
"reward_std": 0.9972768127918243,
|
|
"rewards/rnd_reward/mean": -6.467104096685716e-08,
|
|
"rewards/rnd_reward/std": 0.9972768068313599,
|
|
"step": 800,
|
|
"step_time": 9.678878564713523
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12500000298023223,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.5,
|
|
"completions/mean_length": 121.30500183105468,
|
|
"completions/mean_terminated_length": 83.6633544921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.795678195357323,
|
|
"epoch": 0.5297580117724002,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0277215242385864,
|
|
"kl": 0.06990557787939906,
|
|
"learning_rate": 3.677240026160889e-07,
|
|
"loss": 0.059202158451080324,
|
|
"num_tokens": 17737161.0,
|
|
"reward": -1.2487173535191686e-07,
|
|
"reward_std": 0.9970629572868347,
|
|
"rewards/rnd_reward/mean": -1.2487172682540403e-07,
|
|
"rewards/rnd_reward/std": 0.9970629572868347,
|
|
"step": 810,
|
|
"step_time": 10.510143173905089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13000000268220901,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.4,
|
|
"completions/mean_length": 119.66500091552734,
|
|
"completions/mean_terminated_length": 80.23575859069824,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8059179961681364,
|
|
"epoch": 0.5362982341399608,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8309988379478455,
|
|
"kl": 0.08165676584467292,
|
|
"learning_rate": 3.660889470241988e-07,
|
|
"loss": 0.12505731582641602,
|
|
"num_tokens": 17927903.0,
|
|
"reward": -2.3901462826714236e-07,
|
|
"reward_std": 0.997013247013092,
|
|
"rewards/rnd_reward/mean": -2.3901462400388593e-07,
|
|
"rewards/rnd_reward/std": 0.9970132112503052,
|
|
"step": 820,
|
|
"step_time": 10.190648046904244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14875000417232515,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.9,
|
|
"completions/mean_length": 126.17125396728515,
|
|
"completions/mean_terminated_length": 80.60145225524903,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.938824012875557,
|
|
"epoch": 0.5428384565075213,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1022851467132568,
|
|
"kl": 0.07984373616054655,
|
|
"learning_rate": 3.644538914323087e-07,
|
|
"loss": 0.023773609101772307,
|
|
"num_tokens": 18123110.0,
|
|
"reward": -2.2649766062698973e-08,
|
|
"reward_std": 0.9969716727733612,
|
|
"rewards/rnd_reward/mean": -2.2649762509985294e-08,
|
|
"rewards/rnd_reward/std": 0.9969716787338256,
|
|
"step": 830,
|
|
"step_time": 9.89921972535085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14375000447034836,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.7,
|
|
"completions/mean_length": 127.39125366210938,
|
|
"completions/mean_terminated_length": 84.2758415222168,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8030223518610002,
|
|
"epoch": 0.5493786788750817,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9427273273468018,
|
|
"kl": 0.0753337662667036,
|
|
"learning_rate": 3.6281883584041854e-07,
|
|
"loss": 0.05915579795837402,
|
|
"num_tokens": 18319518.0,
|
|
"reward": -1.1295080284412507e-07,
|
|
"reward_std": 0.9969306707382202,
|
|
"rewards/rnd_reward/mean": -1.1295080142303959e-07,
|
|
"rewards/rnd_reward/std": 0.9969306766986847,
|
|
"step": 840,
|
|
"step_time": 9.87428822228685
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15750000178813933,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 368.2,
|
|
"completions/mean_length": 137.7587516784668,
|
|
"completions/mean_terminated_length": 91.88029098510742,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.810399681329727,
|
|
"epoch": 0.5559189012426422,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8602397441864014,
|
|
"kl": 0.0653490168042481,
|
|
"learning_rate": 3.611837802485284e-07,
|
|
"loss": 0.03035195469856262,
|
|
"num_tokens": 18526310.0,
|
|
"reward": -1.1295080284412507e-07,
|
|
"reward_std": 0.9969462454319,
|
|
"rewards/rnd_reward/mean": -1.1295080426521054e-07,
|
|
"rewards/rnd_reward/std": 0.9969462633132935,
|
|
"step": 850,
|
|
"step_time": 10.37288966330234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15250000283122062,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 343.9,
|
|
"completions/mean_length": 127.73250350952148,
|
|
"completions/mean_terminated_length": 81.79599456787109,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.83120773434639,
|
|
"epoch": 0.5624591236102028,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0732090473175049,
|
|
"kl": 0.08459939705207944,
|
|
"learning_rate": 3.595487246566383e-07,
|
|
"loss": 0.03491010665893555,
|
|
"num_tokens": 18724316.0,
|
|
"reward": -1.4752149617436318e-07,
|
|
"reward_std": 0.9969268858432769,
|
|
"rewards/rnd_reward/mean": -1.4752149759544864e-07,
|
|
"rewards/rnd_reward/std": 0.9969269037246704,
|
|
"step": 860,
|
|
"step_time": 10.52987401576247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14000000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.0,
|
|
"completions/mean_length": 131.95375213623046,
|
|
"completions/mean_terminated_length": 90.77078781127929,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.9562632203102113,
|
|
"epoch": 0.5689993459777632,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.969403862953186,
|
|
"kl": 0.08435474634170533,
|
|
"learning_rate": 3.579136690647482e-07,
|
|
"loss": 0.04807090759277344,
|
|
"num_tokens": 18926489.0,
|
|
"reward": -1.3545155894689742e-07,
|
|
"reward_std": 0.9968080937862396,
|
|
"rewards/rnd_reward/mean": -1.3545156178906837e-07,
|
|
"rewards/rnd_reward/std": 0.9968081057071686,
|
|
"step": 870,
|
|
"step_time": 9.85819350676611
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16750000193715095,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 367.2,
|
|
"completions/mean_length": 137.6675033569336,
|
|
"completions/mean_terminated_length": 88.34969863891601,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7675389796495438,
|
|
"epoch": 0.5755395683453237,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.828363835811615,
|
|
"kl": 0.07512732800096274,
|
|
"learning_rate": 3.5627861347285806e-07,
|
|
"loss": 0.010637739300727844,
|
|
"num_tokens": 19126833.0,
|
|
"reward": -7.003545690054125e-08,
|
|
"reward_std": 0.9967847049236298,
|
|
"rewards/rnd_reward/mean": -7.003545903216945e-08,
|
|
"rewards/rnd_reward/std": 0.9967847049236298,
|
|
"step": 880,
|
|
"step_time": 9.748134864587337
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.19125000312924384,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.5,
|
|
"completions/mean_length": 148.54750366210936,
|
|
"completions/mean_terminated_length": 93.13098068237305,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.6605105638504027,
|
|
"epoch": 0.5820797907128843,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8364513516426086,
|
|
"kl": 0.06730008013546467,
|
|
"learning_rate": 3.5464355788096794e-07,
|
|
"loss": 0.0366318553686142,
|
|
"num_tokens": 19338336.0,
|
|
"reward": -1.0654330893089536e-07,
|
|
"reward_std": 0.9968020260334015,
|
|
"rewards/rnd_reward/mean": -1.0654329685166886e-07,
|
|
"rewards/rnd_reward/std": 0.9968020260334015,
|
|
"step": 890,
|
|
"step_time": 9.872815124248154
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16625000312924385,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.5,
|
|
"completions/mean_length": 149.7400032043457,
|
|
"completions/mean_terminated_length": 103.11838226318359,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.535254544019699,
|
|
"epoch": 0.5886200130804448,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7181042432785034,
|
|
"kl": 0.05683262627571821,
|
|
"learning_rate": 3.530085022890778e-07,
|
|
"loss": 0.08454052209854127,
|
|
"num_tokens": 19557363.0,
|
|
"reward": -1.737475452401327e-07,
|
|
"reward_std": 0.9967290103435517,
|
|
"rewards/rnd_reward/mean": -1.737475338714489e-07,
|
|
"rewards/rnd_reward/std": 0.9967289924621582,
|
|
"step": 900,
|
|
"step_time": 10.189786232402549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1912500038743019,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.0,
|
|
"completions/mean_length": 157.67875213623046,
|
|
"completions/mean_terminated_length": 104.52761840820312,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.5941266864538193,
|
|
"epoch": 0.5951602354480052,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9426244497299194,
|
|
"kl": 0.050556823750957845,
|
|
"learning_rate": 3.513734466971877e-07,
|
|
"loss": 0.07448753714561462,
|
|
"num_tokens": 19775536.0,
|
|
"reward": -2.162158537544201e-07,
|
|
"reward_std": 0.9968621492385864,
|
|
"rewards/rnd_reward/mean": -2.1621584735953548e-07,
|
|
"rewards/rnd_reward/std": 0.9968621671199799,
|
|
"step": 910,
|
|
"step_time": 9.74868165422231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1662500023841858,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.0,
|
|
"completions/mean_length": 151.89000396728517,
|
|
"completions/mean_terminated_length": 105.61829071044922,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.508600893616676,
|
|
"epoch": 0.6017004578155657,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7371200323104858,
|
|
"kl": 0.04880232191644609,
|
|
"learning_rate": 3.4973839110529757e-07,
|
|
"loss": 0.12514564990997315,
|
|
"num_tokens": 19990298.0,
|
|
"reward": -1.93715107066339e-08,
|
|
"reward_std": 0.9970090925693512,
|
|
"rewards/rnd_reward/mean": -1.937150893027706e-08,
|
|
"rewards/rnd_reward/std": 0.9970090866088868,
|
|
"step": 920,
|
|
"step_time": 9.937099263281562
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1575000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.5,
|
|
"completions/mean_length": 138.63375244140624,
|
|
"completions/mean_terminated_length": 92.52680130004883,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4236575067043304,
|
|
"epoch": 0.6082406801831263,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9167628884315491,
|
|
"kl": 0.05787700763903558,
|
|
"learning_rate": 3.4810333551340745e-07,
|
|
"loss": 0.10835990905761719,
|
|
"num_tokens": 20187745.0,
|
|
"reward": -7.212161943215278e-08,
|
|
"reward_std": 0.996803218126297,
|
|
"rewards/rnd_reward/mean": -7.212161801106731e-08,
|
|
"rewards/rnd_reward/std": 0.9968032300472259,
|
|
"step": 930,
|
|
"step_time": 9.689079960295931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1337500013411045,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.4,
|
|
"completions/mean_length": 129.22500305175782,
|
|
"completions/mean_terminated_length": 89.93161087036133,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6605860769748686,
|
|
"epoch": 0.6147809025506867,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0838011503219604,
|
|
"kl": 0.06689512394368649,
|
|
"learning_rate": 3.4646827992151733e-07,
|
|
"loss": 0.07303244471549988,
|
|
"num_tokens": 20383990.0,
|
|
"reward": -1.1324883786301143e-08,
|
|
"reward_std": 0.9967760264873504,
|
|
"rewards/rnd_reward/mean": -1.1324883786301143e-08,
|
|
"rewards/rnd_reward/std": 0.9967760384082794,
|
|
"step": 940,
|
|
"step_time": 9.85847494918853
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16125000268220901,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 364.1,
|
|
"completions/mean_length": 134.65625534057617,
|
|
"completions/mean_terminated_length": 86.58066329956054,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7146200090646744,
|
|
"epoch": 0.6213211249182472,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9745354652404785,
|
|
"kl": 0.06564742000773549,
|
|
"learning_rate": 3.448332243296272e-07,
|
|
"loss": 0.10424284934997559,
|
|
"num_tokens": 20593190.0,
|
|
"reward": -1.0967255157368072e-07,
|
|
"reward_std": 0.9968843460083008,
|
|
"rewards/rnd_reward/mean": -1.096725451787961e-07,
|
|
"rewards/rnd_reward/std": 0.9968843400478363,
|
|
"step": 950,
|
|
"step_time": 10.862376277497969
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10875000022351741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.1,
|
|
"completions/mean_length": 113.23000411987304,
|
|
"completions/mean_terminated_length": 80.14122467041015,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8509408473968505,
|
|
"epoch": 0.6278613472858077,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9323987364768982,
|
|
"kl": 0.08510139184072614,
|
|
"learning_rate": 3.4319816873773703e-07,
|
|
"loss": 0.0554282009601593,
|
|
"num_tokens": 20776164.0,
|
|
"reward": -7.957220322651893e-08,
|
|
"reward_std": 0.9968249201774597,
|
|
"rewards/rnd_reward/mean": -7.957220180543345e-08,
|
|
"rewards/rnd_reward/std": 0.9968249082565308,
|
|
"step": 960,
|
|
"step_time": 9.989999298867769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10750000327825546,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.2,
|
|
"completions/mean_length": 112.88375091552734,
|
|
"completions/mean_terminated_length": 80.20179443359375,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.88864361345768,
|
|
"epoch": 0.6344015696533682,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8851020932197571,
|
|
"kl": 0.0875691649504006,
|
|
"learning_rate": 3.4156311314584696e-07,
|
|
"loss": 0.10126043558120727,
|
|
"num_tokens": 20958826.0,
|
|
"reward": -1.6622245713193706e-07,
|
|
"reward_std": 0.9968612968921662,
|
|
"rewards/rnd_reward/mean": -1.6622244931596696e-07,
|
|
"rewards/rnd_reward/std": 0.9968612730503082,
|
|
"step": 970,
|
|
"step_time": 10.018015099223703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11125000156462192,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.6,
|
|
"completions/mean_length": 106.1625,
|
|
"completions/mean_terminated_length": 71.38758811950683,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9656033277511598,
|
|
"epoch": 0.6409417920209287,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9377958178520203,
|
|
"kl": 0.09664760064333677,
|
|
"learning_rate": 3.3992805755395684e-07,
|
|
"loss": 0.046259531378746034,
|
|
"num_tokens": 21147866.0,
|
|
"reward": -2.499669804478799e-07,
|
|
"reward_std": 0.9968202471733093,
|
|
"rewards/rnd_reward/mean": -2.499669751188094e-07,
|
|
"rewards/rnd_reward/std": 0.9968202352523804,
|
|
"step": 980,
|
|
"step_time": 11.508573367865756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1262500025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.6,
|
|
"completions/mean_length": 122.56625289916992,
|
|
"completions/mean_terminated_length": 84.71166687011718,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.80318078994751,
|
|
"epoch": 0.6474820143884892,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9525420069694519,
|
|
"kl": 0.07310699988156558,
|
|
"learning_rate": 3.3829300196206667e-07,
|
|
"loss": 0.02322930097579956,
|
|
"num_tokens": 21338124.0,
|
|
"reward": -9.76771161731449e-08,
|
|
"reward_std": 0.9966861188411713,
|
|
"rewards/rnd_reward/mean": -9.7677110488803e-08,
|
|
"rewards/rnd_reward/std": 0.9966861248016358,
|
|
"step": 990,
|
|
"step_time": 9.876657200045884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15375000201165676,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.8,
|
|
"completions/mean_length": 137.66875076293945,
|
|
"completions/mean_terminated_length": 92.6232063293457,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9049280107021334,
|
|
"epoch": 0.6540222367560498,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8244759440422058,
|
|
"kl": 0.061428536847233775,
|
|
"learning_rate": 3.366579463701766e-07,
|
|
"loss": 0.05219428539276123,
|
|
"num_tokens": 21539579.0,
|
|
"reward": -1.6495585271059098e-07,
|
|
"reward_std": 0.9966946303844452,
|
|
"rewards/rnd_reward/mean": -1.6495585271059098e-07,
|
|
"rewards/rnd_reward/std": 0.9966946005821228,
|
|
"step": 1000,
|
|
"step_time": 9.666829459602013
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.18250000178813935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.9,
|
|
"completions/mean_length": 146.93875427246093,
|
|
"completions/mean_terminated_length": 93.79228134155274,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8016369342803955,
|
|
"epoch": 0.6605624591236102,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.766015350818634,
|
|
"kl": 0.05675441958010197,
|
|
"learning_rate": 3.350228907782864e-07,
|
|
"loss": 0.08666471242904664,
|
|
"num_tokens": 21754125.0,
|
|
"reward": -2.2858381782953073e-07,
|
|
"reward_std": 0.996807062625885,
|
|
"rewards/rnd_reward/mean": -2.2858381640844527e-07,
|
|
"rewards/rnd_reward/std": 0.9968070745468139,
|
|
"step": 1010,
|
|
"step_time": 10.005978721287102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13875000253319741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.3,
|
|
"completions/mean_length": 136.35500259399413,
|
|
"completions/mean_terminated_length": 96.46944885253906,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8755707025527952,
|
|
"epoch": 0.6671026814911707,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0061216354370117,
|
|
"kl": 0.05596274421550333,
|
|
"learning_rate": 3.3338783518639635e-07,
|
|
"loss": 0.09954084157943725,
|
|
"num_tokens": 21957634.0,
|
|
"reward": -1.9520521661320344e-07,
|
|
"reward_std": 0.9966324806213379,
|
|
"rewards/rnd_reward/mean": -1.9520520950777608e-07,
|
|
"rewards/rnd_reward/std": 0.9966325044631958,
|
|
"step": 1020,
|
|
"step_time": 10.040163300000131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15875000432133674,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.5,
|
|
"completions/mean_length": 136.7437530517578,
|
|
"completions/mean_terminated_length": 90.20843048095703,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7692076802253722,
|
|
"epoch": 0.6736429038587312,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8324238061904907,
|
|
"kl": 0.057948453910648824,
|
|
"learning_rate": 3.317527795945062e-07,
|
|
"loss": 0.07627512216567993,
|
|
"num_tokens": 22158654.0,
|
|
"reward": -3.7699937749380295e-08,
|
|
"reward_std": 0.9966008186340332,
|
|
"rewards/rnd_reward/mean": -3.769993632829482e-08,
|
|
"rewards/rnd_reward/std": 0.9966008424758911,
|
|
"step": 1030,
|
|
"step_time": 10.1797529961681
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16250000298023223,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.7,
|
|
"completions/mean_length": 139.57500534057618,
|
|
"completions/mean_terminated_length": 92.21996688842773,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.8831230252981186,
|
|
"epoch": 0.6801831262262917,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8833721876144409,
|
|
"kl": 0.060850747767835854,
|
|
"learning_rate": 3.3011772400261606e-07,
|
|
"loss": 0.10484603643417359,
|
|
"num_tokens": 22366559.0,
|
|
"reward": -1.0505318579490108e-07,
|
|
"reward_std": 0.996761429309845,
|
|
"rewards/rnd_reward/mean": -1.0505318588371893e-07,
|
|
"rewards/rnd_reward/std": 0.9967614591121674,
|
|
"step": 1040,
|
|
"step_time": 10.081386449630372
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1625000037252903,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.9,
|
|
"completions/mean_length": 137.33875198364257,
|
|
"completions/mean_terminated_length": 89.72673568725585,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.885282647609711,
|
|
"epoch": 0.6867233485938522,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9477590918540955,
|
|
"kl": 0.05932593122124672,
|
|
"learning_rate": 3.28482668410726e-07,
|
|
"loss": 0.024539683759212495,
|
|
"num_tokens": 22571575.0,
|
|
"reward": 4.9620866704458424e-08,
|
|
"reward_std": 0.996524053812027,
|
|
"rewards/rnd_reward/mean": 4.9620866704458424e-08,
|
|
"rewards/rnd_reward/std": 0.9965240597724915,
|
|
"step": 1050,
|
|
"step_time": 10.535549363144673
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16250000447034835,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.5,
|
|
"completions/mean_length": 138.3237533569336,
|
|
"completions/mean_terminated_length": 90.70624694824218,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.875243663787842,
|
|
"epoch": 0.6932635709614127,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9407637119293213,
|
|
"kl": 0.06043377108871937,
|
|
"learning_rate": 3.268476128188358e-07,
|
|
"loss": 0.06624546647071838,
|
|
"num_tokens": 22771314.0,
|
|
"reward": -5.781650713743147e-08,
|
|
"reward_std": 0.9966591358184814,
|
|
"rewards/rnd_reward/mean": -5.7816505716346e-08,
|
|
"rewards/rnd_reward/std": 0.9966591358184814,
|
|
"step": 1060,
|
|
"step_time": 10.021701269014738
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1400000039488077,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 342.1,
|
|
"completions/mean_length": 127.3300048828125,
|
|
"completions/mean_terminated_length": 85.42830429077148,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.927751249074936,
|
|
"epoch": 0.6998037933289731,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8508753180503845,
|
|
"kl": 0.0620661674067378,
|
|
"learning_rate": 3.252125572269457e-07,
|
|
"loss": 0.12821773290634156,
|
|
"num_tokens": 22965548.0,
|
|
"reward": -1.0244548391824537e-07,
|
|
"reward_std": 0.9966984510421752,
|
|
"rewards/rnd_reward/mean": -1.0244548391824537e-07,
|
|
"rewards/rnd_reward/std": 0.9966984808444976,
|
|
"step": 1070,
|
|
"step_time": 9.767576195369475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14500000327825546,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.6,
|
|
"completions/mean_length": 136.3862518310547,
|
|
"completions/mean_terminated_length": 94.67120056152343,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.6616650968790054,
|
|
"epoch": 0.7063440156965337,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.789601743221283,
|
|
"kl": 0.0545062692835927,
|
|
"learning_rate": 3.2357750163505557e-07,
|
|
"loss": 0.11497254371643066,
|
|
"num_tokens": 23170007.0,
|
|
"reward": 1.226365569095833e-07,
|
|
"reward_std": 0.9966248035430908,
|
|
"rewards/rnd_reward/mean": 1.2263655548849784e-07,
|
|
"rewards/rnd_reward/std": 0.9966248333454132,
|
|
"step": 1080,
|
|
"step_time": 9.882125865970739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14250000566244125,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.8,
|
|
"completions/mean_length": 130.01000289916993,
|
|
"completions/mean_terminated_length": 87.69976119995117,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.890427288413048,
|
|
"epoch": 0.7128842380640942,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8021807670593262,
|
|
"kl": 0.05592884449288249,
|
|
"learning_rate": 3.2194244604316545e-07,
|
|
"loss": 0.09672708511352539,
|
|
"num_tokens": 23366425.0,
|
|
"reward": 6.273389110411643e-08,
|
|
"reward_std": 0.9965825915336609,
|
|
"rewards/rnd_reward/mean": 6.273389181465916e-08,
|
|
"rewards/rnd_reward/std": 0.9965825736522674,
|
|
"step": 1090,
|
|
"step_time": 9.854898347100242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14500000178813935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 336.9,
|
|
"completions/mean_length": 131.98875274658204,
|
|
"completions/mean_terminated_length": 89.18458480834961,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7609634190797805,
|
|
"epoch": 0.7194244604316546,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0092402696609497,
|
|
"kl": 0.05501702083274722,
|
|
"learning_rate": 3.2030739045127533e-07,
|
|
"loss": 0.08695750832557678,
|
|
"num_tokens": 23567981.0,
|
|
"reward": -1.3828277736216422e-07,
|
|
"reward_std": 0.9966061592102051,
|
|
"rewards/rnd_reward/mean": -1.3828277451999326e-07,
|
|
"rewards/rnd_reward/std": 0.9966061592102051,
|
|
"step": 1100,
|
|
"step_time": 10.145819194451906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1212500013411045,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.9,
|
|
"completions/mean_length": 121.10875091552734,
|
|
"completions/mean_terminated_length": 84.89219970703125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8415472239255903,
|
|
"epoch": 0.7259646827992152,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8949585556983948,
|
|
"kl": 0.06347150038927793,
|
|
"learning_rate": 3.186723348593852e-07,
|
|
"loss": 0.08753393888473511,
|
|
"num_tokens": 23758168.0,
|
|
"reward": -1.1354684570363816e-07,
|
|
"reward_std": 0.9964610040187836,
|
|
"rewards/rnd_reward/mean": -1.1354684286146721e-07,
|
|
"rewards/rnd_reward/std": 0.9964609861373901,
|
|
"step": 1110,
|
|
"step_time": 10.006089114071802
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12125000208616257,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 369.7,
|
|
"completions/mean_length": 121.20000228881835,
|
|
"completions/mean_terminated_length": 84.96716842651367,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.9164877772331237,
|
|
"epoch": 0.7325049051667757,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1227521896362305,
|
|
"kl": 0.06196983400732279,
|
|
"learning_rate": 3.170372792674951e-07,
|
|
"loss": 0.07237975597381592,
|
|
"num_tokens": 23955023.0,
|
|
"reward": -5.5432321488524396e-08,
|
|
"reward_std": 0.9964903295040131,
|
|
"rewards/rnd_reward/mean": -5.543231935689619e-08,
|
|
"rewards/rnd_reward/std": 0.9964903354644775,
|
|
"step": 1120,
|
|
"step_time": 10.375506636733189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11875000298023224,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.2,
|
|
"completions/mean_length": 117.09375,
|
|
"completions/mean_terminated_length": 81.17416534423828,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.7361535727977753,
|
|
"epoch": 0.7390451275343362,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8529885411262512,
|
|
"kl": 0.06673006983473898,
|
|
"learning_rate": 3.1540222367560496e-07,
|
|
"loss": 0.09247955083847045,
|
|
"num_tokens": 24140203.0,
|
|
"reward": -2.6300548938706926e-08,
|
|
"reward_std": 0.9966336786746979,
|
|
"rewards/rnd_reward/mean": -2.6300551070335132e-08,
|
|
"rewards/rnd_reward/std": 0.9966336965560914,
|
|
"step": 1130,
|
|
"step_time": 10.012891574040987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11500000357627868,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.7,
|
|
"completions/mean_length": 119.6712516784668,
|
|
"completions/mean_terminated_length": 85.35453567504882,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6402025282382966,
|
|
"epoch": 0.7455853499018966,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9655517935752869,
|
|
"kl": 0.06339178960770368,
|
|
"learning_rate": 3.1376716808371484e-07,
|
|
"loss": 0.012396740168333054,
|
|
"num_tokens": 24333505.0,
|
|
"reward": -1.3798475424486866e-07,
|
|
"reward_std": 0.9963988780975341,
|
|
"rewards/rnd_reward/mean": -1.379847538895973e-07,
|
|
"rewards/rnd_reward/std": 0.9963988780975341,
|
|
"step": 1140,
|
|
"step_time": 10.81275618087966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12750000208616258,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.4,
|
|
"completions/mean_length": 130.15375289916992,
|
|
"completions/mean_terminated_length": 93.11451568603516,
|
|
"completions/min_length": 2.6,
|
|
"completions/min_terminated_length": 2.6,
|
|
"entropy": 2.7988665878772734,
|
|
"epoch": 0.7521255722694572,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9594887495040894,
|
|
"kl": 0.057864805683493614,
|
|
"learning_rate": 3.121321124918247e-07,
|
|
"loss": 0.05015655755996704,
|
|
"num_tokens": 24526988.0,
|
|
"reward": -2.7045605088815705e-08,
|
|
"reward_std": 0.9962463498115539,
|
|
"rewards/rnd_reward/mean": -2.7045605088815705e-08,
|
|
"rewards/rnd_reward/std": 0.9962463617324829,
|
|
"step": 1150,
|
|
"step_time": 9.847500793170184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13000000305473805,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.1,
|
|
"completions/mean_length": 119.13750076293945,
|
|
"completions/mean_terminated_length": 79.57122192382812,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.799566090106964,
|
|
"epoch": 0.7586657946370177,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3144031763076782,
|
|
"kl": 0.06464958414435387,
|
|
"learning_rate": 3.104970568999346e-07,
|
|
"loss": 0.08081794381141663,
|
|
"num_tokens": 24712093.0,
|
|
"reward": 2.2947787847726885e-08,
|
|
"reward_std": 0.9963953137397766,
|
|
"rewards/rnd_reward/mean": 2.2947789268812358e-08,
|
|
"rewards/rnd_reward/std": 0.9963953375816346,
|
|
"step": 1160,
|
|
"step_time": 9.831479465053416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13000000268220901,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 366.1,
|
|
"completions/mean_length": 128.67500228881835,
|
|
"completions/mean_terminated_length": 90.44079208374023,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.758173832297325,
|
|
"epoch": 0.7652060170045781,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8444108963012695,
|
|
"kl": 0.061153271794319154,
|
|
"learning_rate": 3.088620013080445e-07,
|
|
"loss": 0.06105284690856934,
|
|
"num_tokens": 24905938.0,
|
|
"reward": -9.223818580039733e-08,
|
|
"reward_std": 0.9962478995323181,
|
|
"rewards/rnd_reward/mean": -9.22381850898546e-08,
|
|
"rewards/rnd_reward/std": 0.9962479054927826,
|
|
"step": 1170,
|
|
"step_time": 9.715631593717262
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13125000149011612,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 367.7,
|
|
"completions/mean_length": 132.65000381469727,
|
|
"completions/mean_terminated_length": 94.60303192138672,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.652452102303505,
|
|
"epoch": 0.7717462393721386,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8246821761131287,
|
|
"kl": 0.05974919479340315,
|
|
"learning_rate": 3.072269457161543e-07,
|
|
"loss": 0.07230232954025269,
|
|
"num_tokens": 25111683.0,
|
|
"reward": -1.4752150079289095e-08,
|
|
"reward_std": 0.9964145243167877,
|
|
"rewards/rnd_reward/mean": -1.4752149368746359e-08,
|
|
"rewards/rnd_reward/std": 0.9964145421981812,
|
|
"step": 1180,
|
|
"step_time": 10.24143796269782
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1437500037252903,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 133.8912498474121,
|
|
"completions/mean_terminated_length": 92.00172958374023,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.534428471326828,
|
|
"epoch": 0.7782864617396992,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7489418387413025,
|
|
"kl": 0.05822144625708461,
|
|
"learning_rate": 3.0559189012426423e-07,
|
|
"loss": 0.07474024295806884,
|
|
"num_tokens": 25319036.0,
|
|
"reward": -8.940696432091499e-08,
|
|
"reward_std": 0.9965973794460297,
|
|
"rewards/rnd_reward/mean": -8.940696574200047e-08,
|
|
"rewards/rnd_reward/std": 0.9965973973274231,
|
|
"step": 1190,
|
|
"step_time": 10.355807177536189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1450000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.6,
|
|
"completions/mean_length": 138.9825012207031,
|
|
"completions/mean_terminated_length": 97.18752059936523,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8433078229427338,
|
|
"epoch": 0.7848266841072596,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8156785368919373,
|
|
"kl": 0.06143994340673089,
|
|
"learning_rate": 3.039568345323741e-07,
|
|
"loss": 0.08815957307815551,
|
|
"num_tokens": 25526837.0,
|
|
"reward": -1.0192394315211572e-07,
|
|
"reward_std": 0.996451473236084,
|
|
"rewards/rnd_reward/mean": -1.0192393888885931e-07,
|
|
"rewards/rnd_reward/std": 0.996451485157013,
|
|
"step": 1200,
|
|
"step_time": 10.298059746343643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14000000283122063,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.0,
|
|
"completions/mean_length": 127.81375274658203,
|
|
"completions/mean_terminated_length": 85.73782577514649,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7779837220907213,
|
|
"epoch": 0.7913669064748201,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.937714695930481,
|
|
"kl": 0.06527232667431235,
|
|
"learning_rate": 3.0232177894048394e-07,
|
|
"loss": 0.08186920881271362,
|
|
"num_tokens": 25721938.0,
|
|
"reward": -1.0982156268823929e-07,
|
|
"reward_std": 0.9962611317634582,
|
|
"rewards/rnd_reward/mean": -1.0982156055661107e-07,
|
|
"rewards/rnd_reward/std": 0.9962611377239228,
|
|
"step": 1210,
|
|
"step_time": 9.923247428331525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13625000193715095,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.9,
|
|
"completions/mean_length": 128.51625213623046,
|
|
"completions/mean_terminated_length": 88.076513671875,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.661186295747757,
|
|
"epoch": 0.7979071288423807,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9877737164497375,
|
|
"kl": 0.06118576750159264,
|
|
"learning_rate": 3.0068672334859387e-07,
|
|
"loss": 0.0857498049736023,
|
|
"num_tokens": 25916611.0,
|
|
"reward": -9.454786980001017e-08,
|
|
"reward_std": 0.9962615430355072,
|
|
"rewards/rnd_reward/mean": -9.454786553675376e-08,
|
|
"rewards/rnd_reward/std": 0.9962615609169007,
|
|
"step": 1220,
|
|
"step_time": 10.026552183739842
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13875000402331353,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.7,
|
|
"completions/mean_length": 127.41250381469726,
|
|
"completions/mean_terminated_length": 86.15239715576172,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.597662454843521,
|
|
"epoch": 0.8044473512099412,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9214726090431213,
|
|
"kl": 0.06120588518679142,
|
|
"learning_rate": 2.990516677567037e-07,
|
|
"loss": 0.08460624814033509,
|
|
"num_tokens": 26120156.0,
|
|
"reward": -1.5325844344715732e-07,
|
|
"reward_std": 0.9961971819400788,
|
|
"rewards/rnd_reward/mean": -1.5325843989444364e-07,
|
|
"rewards/rnd_reward/std": 0.9961971819400788,
|
|
"step": 1230,
|
|
"step_time": 10.459681713976897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12875000089406968,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.9,
|
|
"completions/mean_length": 121.05875167846679,
|
|
"completions/mean_terminated_length": 82.47761840820313,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.899898463487625,
|
|
"epoch": 0.8109875735775016,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9018652439117432,
|
|
"kl": 0.06602321406826377,
|
|
"learning_rate": 2.9741661216481357e-07,
|
|
"loss": 0.05030661225318909,
|
|
"num_tokens": 26307033.0,
|
|
"reward": -1.496076620810527e-07,
|
|
"reward_std": 0.996212637424469,
|
|
"rewards/rnd_reward/mean": -1.4960765781779628e-07,
|
|
"rewards/rnd_reward/std": 0.9962126433849334,
|
|
"step": 1240,
|
|
"step_time": 9.930500056478195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13875000253319741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.1,
|
|
"completions/mean_length": 125.87625274658203,
|
|
"completions/mean_terminated_length": 84.06948699951172,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.7101810783147813,
|
|
"epoch": 0.8175277959450621,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9933876991271973,
|
|
"kl": 0.06315780784934759,
|
|
"learning_rate": 2.9578155657292345e-07,
|
|
"loss": 0.14283275604248047,
|
|
"num_tokens": 26506699.0,
|
|
"reward": -4.701316604904804e-08,
|
|
"reward_std": 0.996234393119812,
|
|
"rewards/rnd_reward/mean": -4.7013166404319404e-08,
|
|
"rewards/rnd_reward/std": 0.9962343990802764,
|
|
"step": 1250,
|
|
"step_time": 11.001938613550738
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14875000342726707,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.5,
|
|
"completions/mean_length": 131.21625289916992,
|
|
"completions/mean_terminated_length": 87.06982650756837,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.695519116520882,
|
|
"epoch": 0.8240680183126227,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8424022793769836,
|
|
"kl": 0.06466643353924155,
|
|
"learning_rate": 2.9414650098103333e-07,
|
|
"loss": 0.08369569182395935,
|
|
"num_tokens": 26703352.0,
|
|
"reward": -2.6673080633088375e-08,
|
|
"reward_std": 0.9962631404399872,
|
|
"rewards/rnd_reward/mean": -2.6673077790917433e-08,
|
|
"rewards/rnd_reward/std": 0.9962631464004517,
|
|
"step": 1260,
|
|
"step_time": 9.866059662331827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.16375000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 368.6,
|
|
"completions/mean_length": 132.26625137329103,
|
|
"completions/mean_terminated_length": 83.00667343139648,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7214665055274962,
|
|
"epoch": 0.8306082406801831,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.061651349067688,
|
|
"kl": 0.060832435730844736,
|
|
"learning_rate": 2.9251144538914326e-07,
|
|
"loss": 0.011636599898338318,
|
|
"num_tokens": 26905195.0,
|
|
"reward": -1.28149986178272e-08,
|
|
"reward_std": 0.996177339553833,
|
|
"rewards/rnd_reward/mean": -1.2814998262555832e-08,
|
|
"rewards/rnd_reward/std": 0.9961773693561554,
|
|
"step": 1270,
|
|
"step_time": 10.226913845306262
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1462500013411045,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.4,
|
|
"completions/mean_length": 135.36000289916993,
|
|
"completions/mean_terminated_length": 92.69313278198243,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.5201783299446108,
|
|
"epoch": 0.8371484630477436,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9056648015975952,
|
|
"kl": 0.05058280224911869,
|
|
"learning_rate": 2.908763897972531e-07,
|
|
"loss": 0.08019465208053589,
|
|
"num_tokens": 27106543.0,
|
|
"reward": -1.9341707613307335e-07,
|
|
"reward_std": 0.9962323784828186,
|
|
"rewards/rnd_reward/mean": -1.9341706973818874e-07,
|
|
"rewards/rnd_reward/std": 0.9962323844432831,
|
|
"step": 1280,
|
|
"step_time": 10.01526041161269
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1225000001490116,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 373.1,
|
|
"completions/mean_length": 130.23250198364258,
|
|
"completions/mean_terminated_length": 94.88236541748047,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.623024606704712,
|
|
"epoch": 0.8436886854153042,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.783699095249176,
|
|
"kl": 0.05370175880379975,
|
|
"learning_rate": 2.8924133420536296e-07,
|
|
"loss": 0.11414051055908203,
|
|
"num_tokens": 27305554.0,
|
|
"reward": -9.253621175986382e-08,
|
|
"reward_std": 0.9961644470691681,
|
|
"rewards/rnd_reward/mean": -9.253620731897172e-08,
|
|
"rewards/rnd_reward/std": 0.9961644232273101,
|
|
"step": 1290,
|
|
"step_time": 10.230407970934175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14875000193715096,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 365.6,
|
|
"completions/mean_length": 136.5575035095215,
|
|
"completions/mean_terminated_length": 93.4459114074707,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.676475775241852,
|
|
"epoch": 0.8502289077828646,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0168393850326538,
|
|
"kl": 0.052173000667244195,
|
|
"learning_rate": 2.8760627861347284e-07,
|
|
"loss": 0.049798119068145755,
|
|
"num_tokens": 27507145.0,
|
|
"reward": -1.576542894232702e-07,
|
|
"reward_std": 0.9962258338928223,
|
|
"rewards/rnd_reward/mean": -1.5765429013381292e-07,
|
|
"rewards/rnd_reward/std": 0.9962258338928223,
|
|
"step": 1300,
|
|
"step_time": 9.98130206335336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13500000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.1,
|
|
"completions/mean_length": 128.2087516784668,
|
|
"completions/mean_terminated_length": 88.18164749145508,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.640226128697395,
|
|
"epoch": 0.8567691301504251,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8835275769233704,
|
|
"kl": 0.05669898777268827,
|
|
"learning_rate": 2.859712230215827e-07,
|
|
"loss": 0.10237141847610473,
|
|
"num_tokens": 27707452.0,
|
|
"reward": -6.869435509315736e-08,
|
|
"reward_std": 0.9961890399456024,
|
|
"rewards/rnd_reward/mean": -6.869434940881546e-08,
|
|
"rewards/rnd_reward/std": 0.99618901014328,
|
|
"step": 1310,
|
|
"step_time": 10.37953436721582
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13625000268220902,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.5,
|
|
"completions/mean_length": 128.80750274658203,
|
|
"completions/mean_terminated_length": 88.5141227722168,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.4644729048013687,
|
|
"epoch": 0.8633093525179856,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8116011619567871,
|
|
"kl": 0.05524417916312814,
|
|
"learning_rate": 2.843361674296926e-07,
|
|
"loss": 0.1121416449546814,
|
|
"num_tokens": 27900098.0,
|
|
"reward": -5.230307502657183e-08,
|
|
"reward_std": 0.9961800217628479,
|
|
"rewards/rnd_reward/mean": -5.230307715820004e-08,
|
|
"rewards/rnd_reward/std": 0.9961800217628479,
|
|
"step": 1320,
|
|
"step_time": 9.958397940779104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11625000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.3,
|
|
"completions/mean_length": 122.20125198364258,
|
|
"completions/mean_terminated_length": 87.82691497802735,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.619413223862648,
|
|
"epoch": 0.8698495748855462,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1175379753112793,
|
|
"kl": 0.06109289322048426,
|
|
"learning_rate": 2.827011118378025e-07,
|
|
"loss": 0.05299858450889587,
|
|
"num_tokens": 28083799.0,
|
|
"reward": -9.030103811369373e-08,
|
|
"reward_std": 0.9959750473499298,
|
|
"rewards/rnd_reward/mean": -9.030103793605804e-08,
|
|
"rewards/rnd_reward/std": 0.9959750413894654,
|
|
"step": 1330,
|
|
"step_time": 9.82245781556703
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11625000312924386,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 361.4,
|
|
"completions/mean_length": 122.17500152587891,
|
|
"completions/mean_terminated_length": 87.70087432861328,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.900624880194664,
|
|
"epoch": 0.8763897972531066,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.942399263381958,
|
|
"kl": 0.0663422117009759,
|
|
"learning_rate": 2.8106605624591235e-07,
|
|
"loss": 0.07738288044929505,
|
|
"num_tokens": 28270204.0,
|
|
"reward": -1.4573336244438905e-07,
|
|
"reward_std": 0.9961058497428894,
|
|
"rewards/rnd_reward/mean": -1.4573335960221812e-07,
|
|
"rewards/rnd_reward/std": 0.9961058616638183,
|
|
"step": 1340,
|
|
"step_time": 9.953787335986272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1312500022351742,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 370.4,
|
|
"completions/mean_length": 120.04500045776368,
|
|
"completions/mean_terminated_length": 80.18856277465821,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.87275829911232,
|
|
"epoch": 0.8829300196206671,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0761396884918213,
|
|
"kl": 0.07089657634496689,
|
|
"learning_rate": 2.794310006540222e-07,
|
|
"loss": 0.13924288749694824,
|
|
"num_tokens": 28461485.0,
|
|
"reward": 2.965331162840812e-08,
|
|
"reward_std": 0.9960719764232635,
|
|
"rewards/rnd_reward/mean": 2.965331162840812e-08,
|
|
"rewards/rnd_reward/std": 0.996071970462799,
|
|
"step": 1350,
|
|
"step_time": 10.60484689238947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12875000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.8,
|
|
"completions/mean_length": 115.14375228881836,
|
|
"completions/mean_terminated_length": 75.8135986328125,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 3.0420612663030626,
|
|
"epoch": 0.8894702419882276,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1144483089447021,
|
|
"kl": 0.07124942755326628,
|
|
"learning_rate": 2.777959450621321e-07,
|
|
"loss": 0.08705209493637085,
|
|
"num_tokens": 28652205.0,
|
|
"reward": -1.2695789215477759e-07,
|
|
"reward_std": 0.9961645841598511,
|
|
"rewards/rnd_reward/mean": -1.2695789215477759e-07,
|
|
"rewards/rnd_reward/std": 0.99616459608078,
|
|
"step": 1360,
|
|
"step_time": 10.374728590669111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12250000275671483,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.6,
|
|
"completions/mean_length": 116.32750015258789,
|
|
"completions/mean_terminated_length": 78.6882926940918,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8743550151586534,
|
|
"epoch": 0.8960104643557881,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.038732886314392,
|
|
"kl": 0.07000278439372778,
|
|
"learning_rate": 2.76160889470242e-07,
|
|
"loss": 0.11522959470748902,
|
|
"num_tokens": 28836007.0,
|
|
"reward": -2.1174550361990897e-07,
|
|
"reward_std": 0.9960576713085174,
|
|
"rewards/rnd_reward/mean": -2.117455032646376e-07,
|
|
"rewards/rnd_reward/std": 0.9960576832294464,
|
|
"step": 1370,
|
|
"step_time": 10.044307312183083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10750000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.4,
|
|
"completions/mean_length": 112.57625274658203,
|
|
"completions/mean_terminated_length": 79.90474357604981,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.814165809750557,
|
|
"epoch": 0.9025506867233486,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.853209912776947,
|
|
"kl": 0.07179404897615313,
|
|
"learning_rate": 2.7452583387835187e-07,
|
|
"loss": 0.11325899362564087,
|
|
"num_tokens": 29020798.0,
|
|
"reward": -4.172328260665381e-09,
|
|
"reward_std": 0.995877069234848,
|
|
"rewards/rnd_reward/mean": -4.172324707951702e-09,
|
|
"rewards/rnd_reward/std": 0.9958770573139191,
|
|
"step": 1380,
|
|
"step_time": 10.452752491505816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09500000216066837,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.1,
|
|
"completions/mean_length": 97.97000122070312,
|
|
"completions/mean_terminated_length": 68.24239921569824,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.9521768242120743,
|
|
"epoch": 0.9090909090909091,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2329504489898682,
|
|
"kl": 0.09182704174891114,
|
|
"learning_rate": 2.7289077828646175e-07,
|
|
"loss": 0.09825693368911743,
|
|
"num_tokens": 29195334.0,
|
|
"reward": 9.715557460765468e-08,
|
|
"reward_std": 0.9959101438522339,
|
|
"rewards/rnd_reward/mean": 9.715556679168457e-08,
|
|
"rewards/rnd_reward/std": 0.9959101438522339,
|
|
"step": 1390,
|
|
"step_time": 10.322968539246357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10125000234693289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.9,
|
|
"completions/mean_length": 103.2087516784668,
|
|
"completions/mean_terminated_length": 71.93345756530762,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.0182015657424928,
|
|
"epoch": 0.9156311314584696,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4850752353668213,
|
|
"kl": 0.09229131015017629,
|
|
"learning_rate": 2.7125572269457157e-07,
|
|
"loss": 0.07991544604301452,
|
|
"num_tokens": 29371486.0,
|
|
"reward": -1.5191734150654668e-07,
|
|
"reward_std": 0.995727401971817,
|
|
"rewards/rnd_reward/mean": -1.5191734008546122e-07,
|
|
"rewards/rnd_reward/std": 0.9957274079322815,
|
|
"step": 1400,
|
|
"step_time": 10.564200303750113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11875000409781933,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.6,
|
|
"completions/mean_length": 109.10750198364258,
|
|
"completions/mean_terminated_length": 72.09171180725097,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9613120049238204,
|
|
"epoch": 0.9221713538260301,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9964513778686523,
|
|
"kl": 0.0881183447316289,
|
|
"learning_rate": 2.696206671026815e-07,
|
|
"loss": 0.05779829025268555,
|
|
"num_tokens": 29547567.0,
|
|
"reward": -2.881884620276054e-07,
|
|
"reward_std": 0.9955705881118775,
|
|
"rewards/rnd_reward/mean": -2.881884591854345e-07,
|
|
"rewards/rnd_reward/std": 0.9955705881118775,
|
|
"step": 1410,
|
|
"step_time": 10.214018813963047
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000156462193,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.2,
|
|
"completions/mean_length": 102.86875228881836,
|
|
"completions/mean_terminated_length": 76.34417495727538,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.703101697564125,
|
|
"epoch": 0.9287115761935906,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8821516633033752,
|
|
"kl": 0.08714565662667155,
|
|
"learning_rate": 2.679856115107914e-07,
|
|
"loss": 0.017349694669246674,
|
|
"num_tokens": 29720227.0,
|
|
"reward": -8.821487531562867e-08,
|
|
"reward_std": 0.9958377420902252,
|
|
"rewards/rnd_reward/mean": -8.821487389454319e-08,
|
|
"rewards/rnd_reward/std": 0.9958377361297608,
|
|
"step": 1420,
|
|
"step_time": 10.117785238358191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13000000044703483,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.4,
|
|
"completions/mean_length": 115.6462516784668,
|
|
"completions/mean_terminated_length": 75.65214309692382,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.66856093108654,
|
|
"epoch": 0.935251798561151,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8306900858879089,
|
|
"kl": 0.07128150118514895,
|
|
"learning_rate": 2.663505559189012e-07,
|
|
"loss": 0.07572194337844848,
|
|
"num_tokens": 29907299.0,
|
|
"reward": -1.2844801631217705e-07,
|
|
"reward_std": 0.995860880613327,
|
|
"rewards/rnd_reward/mean": -1.2844801062783516e-07,
|
|
"rewards/rnd_reward/std": 0.9958608686923981,
|
|
"step": 1430,
|
|
"step_time": 10.222553351335227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12500000223517418,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.7,
|
|
"completions/mean_length": 121.33125076293945,
|
|
"completions/mean_terminated_length": 83.87310562133788,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.7864604860544206,
|
|
"epoch": 0.9417920209287116,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.97917640209198,
|
|
"kl": 0.06842925064265729,
|
|
"learning_rate": 2.6471550032701114e-07,
|
|
"loss": 0.11840918064117431,
|
|
"num_tokens": 30100959.0,
|
|
"reward": -1.5377998749954713e-07,
|
|
"reward_std": 0.9958663046360016,
|
|
"rewards/rnd_reward/mean": -1.5377998110466252e-07,
|
|
"rewards/rnd_reward/std": 0.9958662986755371,
|
|
"step": 1440,
|
|
"step_time": 10.766110814665444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.14125000238418578,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.3,
|
|
"completions/mean_length": 124.87875289916992,
|
|
"completions/mean_terminated_length": 82.31183624267578,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.699194425344467,
|
|
"epoch": 0.9483322432962721,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8881458640098572,
|
|
"kl": 0.06711579142138362,
|
|
"learning_rate": 2.6308044473512096e-07,
|
|
"loss": 0.10254174470901489,
|
|
"num_tokens": 30297562.0,
|
|
"reward": -6.161630330403512e-08,
|
|
"reward_std": 0.9957123577594758,
|
|
"rewards/rnd_reward/mean": -6.16162979749646e-08,
|
|
"rewards/rnd_reward/std": 0.9957123517990112,
|
|
"step": 1450,
|
|
"step_time": 10.593307360098697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1200000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.1,
|
|
"completions/mean_length": 116.06000213623047,
|
|
"completions/mean_terminated_length": 79.77308044433593,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.995149564743042,
|
|
"epoch": 0.9548724656638325,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0435981750488281,
|
|
"kl": 0.07414040835574269,
|
|
"learning_rate": 2.6144538914323084e-07,
|
|
"loss": 0.08301687836647034,
|
|
"num_tokens": 30483565.0,
|
|
"reward": -1.0631978923925089e-07,
|
|
"reward_std": 0.9958245873451232,
|
|
"rewards/rnd_reward/mean": -1.0631978462072311e-07,
|
|
"rewards/rnd_reward/std": 0.9958245933055878,
|
|
"step": 1460,
|
|
"step_time": 10.52764433112461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12750000283122062,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.3,
|
|
"completions/mean_length": 111.42625198364257,
|
|
"completions/mean_terminated_length": 71.7352523803711,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8058162331581116,
|
|
"epoch": 0.961412688031393,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9467556476593018,
|
|
"kl": 0.07855136888101696,
|
|
"learning_rate": 2.598103335513407e-07,
|
|
"loss": 0.10095235109329223,
|
|
"num_tokens": 30671516.0,
|
|
"reward": -1.529604193084566e-07,
|
|
"reward_std": 0.9958255529403687,
|
|
"rewards/rnd_reward/mean": -1.5296041966372798e-07,
|
|
"rewards/rnd_reward/std": 0.9958255529403687,
|
|
"step": 1470,
|
|
"step_time": 10.889759617857635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11125000268220901,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.2,
|
|
"completions/mean_length": 105.6962516784668,
|
|
"completions/mean_terminated_length": 70.84101905822754,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9369604349136353,
|
|
"epoch": 0.9679529103989536,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9438734650611877,
|
|
"kl": 0.0841084063053131,
|
|
"learning_rate": 2.581752779594506e-07,
|
|
"loss": 0.06790788173675537,
|
|
"num_tokens": 30852283.0,
|
|
"reward": -1.862645504502325e-08,
|
|
"reward_std": 0.9955705404281616,
|
|
"rewards/rnd_reward/mean": -1.8626451847580937e-08,
|
|
"rewards/rnd_reward/std": 0.9955705344676972,
|
|
"step": 1480,
|
|
"step_time": 10.544715784140863
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11250000223517417,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.8,
|
|
"completions/mean_length": 111.51875228881836,
|
|
"completions/mean_terminated_length": 77.10519866943359,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.871400186419487,
|
|
"epoch": 0.9744931327665141,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0710206031799316,
|
|
"kl": 0.07257421473041177,
|
|
"learning_rate": 2.5654022236756053e-07,
|
|
"loss": 0.10279649496078491,
|
|
"num_tokens": 31039368.0,
|
|
"reward": -3.039836773410798e-08,
|
|
"reward_std": 0.9956327259540558,
|
|
"rewards/rnd_reward/mean": -3.039836915519345e-08,
|
|
"rewards/rnd_reward/std": 0.9956327199935913,
|
|
"step": 1490,
|
|
"step_time": 11.004371847352013
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11750000305473804,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.9,
|
|
"completions/mean_length": 110.3587516784668,
|
|
"completions/mean_terminated_length": 74.05159416198731,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8570073008537293,
|
|
"epoch": 0.9810333551340745,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9674124717712402,
|
|
"kl": 0.07076722700148821,
|
|
"learning_rate": 2.5490516677567035e-07,
|
|
"loss": 0.09354864358901978,
|
|
"num_tokens": 31229300.0,
|
|
"reward": -7.458031374696361e-08,
|
|
"reward_std": 0.9957621216773986,
|
|
"rewards/rnd_reward/mean": -7.458031019424993e-08,
|
|
"rewards/rnd_reward/std": 0.9957621157169342,
|
|
"step": 1500,
|
|
"step_time": 10.672748254612088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09250000156462193,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 366.3,
|
|
"completions/mean_length": 109.77875213623047,
|
|
"completions/mean_terminated_length": 81.85236892700195,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8984388679265978,
|
|
"epoch": 0.9875735775016351,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.144245982170105,
|
|
"kl": 0.07264794735237956,
|
|
"learning_rate": 2.5327011118378023e-07,
|
|
"loss": 0.10363882780075073,
|
|
"num_tokens": 31410018.0,
|
|
"reward": 3.874301679474002e-09,
|
|
"reward_std": 0.9956199109554291,
|
|
"rewards/rnd_reward/mean": 3.874301679474002e-09,
|
|
"rewards/rnd_reward/std": 0.9956199288368225,
|
|
"step": 1510,
|
|
"step_time": 10.57373680251185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11250000223517417,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.4,
|
|
"completions/mean_length": 107.2912498474121,
|
|
"completions/mean_terminated_length": 71.87631187438964,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.835093083977699,
|
|
"epoch": 0.9941137998691956,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9730135798454285,
|
|
"kl": 0.06768459901213646,
|
|
"learning_rate": 2.516350555918901e-07,
|
|
"loss": 0.04406432807445526,
|
|
"num_tokens": 31584581.0,
|
|
"reward": -6.169080961626605e-08,
|
|
"reward_std": 0.9956278324127197,
|
|
"rewards/rnd_reward/mean": -6.169080748463785e-08,
|
|
"rewards/rnd_reward/std": 0.9956278324127197,
|
|
"step": 1520,
|
|
"step_time": 9.995071636443026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10000000149011612,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.7,
|
|
"completions/mean_length": 105.59500198364258,
|
|
"completions/mean_terminated_length": 74.4354133605957,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.818147099018097,
|
|
"epoch": 1.0006540222367561,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0657150745391846,
|
|
"kl": 0.0751381286419928,
|
|
"learning_rate": 2.5e-07,
|
|
"loss": 0.14666662216186524,
|
|
"num_tokens": 31762082.0,
|
|
"reward": -7.718801313671974e-08,
|
|
"reward_std": 0.9956592619419098,
|
|
"rewards/rnd_reward/mean": -7.718801242617701e-08,
|
|
"rewards/rnd_reward/std": 0.9956592619419098,
|
|
"step": 1530,
|
|
"step_time": 10.494197542965413
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.0,
|
|
"completions/mean_length": 100.37250137329102,
|
|
"completions/mean_terminated_length": 73.99925155639649,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.850705623626709,
|
|
"epoch": 1.0071942446043165,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8915496468544006,
|
|
"kl": 0.08450272670015693,
|
|
"learning_rate": 2.4836494440810987e-07,
|
|
"loss": 0.10265235900878907,
|
|
"num_tokens": 31935920.0,
|
|
"reward": -7.78958216152148e-08,
|
|
"reward_std": 0.9955736041069031,
|
|
"rewards/rnd_reward/mean": -7.789581664141565e-08,
|
|
"rewards/rnd_reward/std": 0.995573616027832,
|
|
"step": 1540,
|
|
"step_time": 10.282078485726379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09625000357627869,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 339.5,
|
|
"completions/mean_length": 100.54375,
|
|
"completions/mean_terminated_length": 70.20137939453124,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7832202523946763,
|
|
"epoch": 1.013734466971877,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9670330882072449,
|
|
"kl": 0.07673759749159217,
|
|
"learning_rate": 2.4672988881621975e-07,
|
|
"loss": 0.053245991468429565,
|
|
"num_tokens": 32105575.0,
|
|
"reward": -7.98702268323126e-08,
|
|
"reward_std": 0.9954982161521911,
|
|
"rewards/rnd_reward/mean": -7.987022256905619e-08,
|
|
"rewards/rnd_reward/std": 0.9954982340335846,
|
|
"step": 1550,
|
|
"step_time": 10.094654931733384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09625000134110451,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 356.6,
|
|
"completions/mean_length": 105.8862518310547,
|
|
"completions/mean_terminated_length": 76.32862396240235,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.806140884757042,
|
|
"epoch": 1.0202746893394374,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9716780185699463,
|
|
"kl": 0.07651977436617016,
|
|
"learning_rate": 2.450948332243296e-07,
|
|
"loss": 0.07588104605674743,
|
|
"num_tokens": 32282264.0,
|
|
"reward": -1.2878328945475914e-07,
|
|
"reward_std": 0.9953814685344696,
|
|
"rewards/rnd_reward/mean": -1.2878328448096e-07,
|
|
"rewards/rnd_reward/std": 0.9953814685344696,
|
|
"step": 1560,
|
|
"step_time": 10.249592515034601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09250000044703484,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 370.2,
|
|
"completions/mean_length": 101.42500152587891,
|
|
"completions/mean_terminated_length": 72.58811225891114,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.706285873055458,
|
|
"epoch": 1.026814911706998,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2053457498550415,
|
|
"kl": 0.07990584680810571,
|
|
"learning_rate": 2.434597776324395e-07,
|
|
"loss": 0.08478202223777771,
|
|
"num_tokens": 32458489.0,
|
|
"reward": -1.197308309830447e-07,
|
|
"reward_std": 0.9958563566207885,
|
|
"rewards/rnd_reward/mean": -1.1973082814087377e-07,
|
|
"rewards/rnd_reward/std": 0.9958563566207885,
|
|
"step": 1570,
|
|
"step_time": 10.385891513410025
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000223517418,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.3,
|
|
"completions/mean_length": 92.8375015258789,
|
|
"completions/mean_terminated_length": 65.02403182983399,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.832634910941124,
|
|
"epoch": 1.0333551340745586,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2888636589050293,
|
|
"kl": 0.08601820273324848,
|
|
"learning_rate": 2.418247220405494e-07,
|
|
"loss": 0.024559563398361205,
|
|
"num_tokens": 32626059.0,
|
|
"reward": -1.628696985278566e-07,
|
|
"reward_std": 0.9954473912715912,
|
|
"rewards/rnd_reward/mean": -1.6286969000134378e-07,
|
|
"rewards/rnd_reward/std": 0.9954474031925201,
|
|
"step": 1580,
|
|
"step_time": 10.754275933234021
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09125000201165676,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.7,
|
|
"completions/mean_length": 99.23750076293945,
|
|
"completions/mean_terminated_length": 70.4692756652832,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.9075894236564634,
|
|
"epoch": 1.039895356442119,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0340582132339478,
|
|
"kl": 0.0824001532047987,
|
|
"learning_rate": 2.4018966644865926e-07,
|
|
"loss": 0.04327432513236999,
|
|
"num_tokens": 32797839.0,
|
|
"reward": 6.616115513224941e-08,
|
|
"reward_std": 0.9953638434410095,
|
|
"rewards/rnd_reward/mean": 6.61611530006212e-08,
|
|
"rewards/rnd_reward/std": 0.9953638732433319,
|
|
"step": 1590,
|
|
"step_time": 10.39529764095787
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1075000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.2,
|
|
"completions/mean_length": 105.15875091552735,
|
|
"completions/mean_terminated_length": 71.44011192321777,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 3.068614384531975,
|
|
"epoch": 1.0464355788096795,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9280741810798645,
|
|
"kl": 0.08188259806483984,
|
|
"learning_rate": 2.3855461085676914e-07,
|
|
"loss": 0.07503924369812012,
|
|
"num_tokens": 32970486.0,
|
|
"reward": 2.354383532576776e-08,
|
|
"reward_std": 0.9955627143383026,
|
|
"rewards/rnd_reward/mean": 2.3543834259953656e-08,
|
|
"rewards/rnd_reward/std": 0.9955627143383026,
|
|
"step": 1600,
|
|
"step_time": 10.102484514610842
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12375000268220901,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.0,
|
|
"completions/mean_length": 108.67875137329102,
|
|
"completions/mean_terminated_length": 69.76623840332032,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.7116469889879227,
|
|
"epoch": 1.05297580117724,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8361508846282959,
|
|
"kl": 0.07135752588510513,
|
|
"learning_rate": 2.36919555264879e-07,
|
|
"loss": 0.08789730072021484,
|
|
"num_tokens": 33155644.0,
|
|
"reward": -7.331371434382561e-08,
|
|
"reward_std": 0.9953298509120941,
|
|
"rewards/rnd_reward/mean": -7.331371363328287e-08,
|
|
"rewards/rnd_reward/std": 0.9953298389911651,
|
|
"step": 1610,
|
|
"step_time": 10.373006676253862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10250000208616257,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.6,
|
|
"completions/mean_length": 107.08625106811523,
|
|
"completions/mean_terminated_length": 75.19281349182128,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.767005005478859,
|
|
"epoch": 1.0595160235448005,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8207447528839111,
|
|
"kl": 0.07195558175444602,
|
|
"learning_rate": 2.352844996729889e-07,
|
|
"loss": 0.07792651653289795,
|
|
"num_tokens": 33333863.0,
|
|
"reward": -7.919967437430842e-08,
|
|
"reward_std": 0.9952020287513733,
|
|
"rewards/rnd_reward/mean": -7.919966940050927e-08,
|
|
"rewards/rnd_reward/std": 0.9952020525932312,
|
|
"step": 1620,
|
|
"step_time": 10.309986177436077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12875000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.7,
|
|
"completions/mean_length": 117.29250259399414,
|
|
"completions/mean_terminated_length": 78.07871360778809,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8041924834251404,
|
|
"epoch": 1.066056245912361,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7970576882362366,
|
|
"kl": 0.07171947872266174,
|
|
"learning_rate": 2.3364944408109875e-07,
|
|
"loss": 0.07321782112121582,
|
|
"num_tokens": 33529112.0,
|
|
"reward": -7.54743808784042e-08,
|
|
"reward_std": 0.9954242706298828,
|
|
"rewards/rnd_reward/mean": -7.54743808784042e-08,
|
|
"rewards/rnd_reward/std": 0.9954242765903473,
|
|
"step": 1630,
|
|
"step_time": 11.020952883246355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10375000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 367.8,
|
|
"completions/mean_length": 109.85750122070313,
|
|
"completions/mean_terminated_length": 78.08732643127442,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7948483556509016,
|
|
"epoch": 1.0725964682799216,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0839210748672485,
|
|
"kl": 0.07083125235512852,
|
|
"learning_rate": 2.3201438848920862e-07,
|
|
"loss": 0.07660450935363769,
|
|
"num_tokens": 33711328.0,
|
|
"reward": -4.068016963287846e-08,
|
|
"reward_std": 0.9954971015453339,
|
|
"rewards/rnd_reward/mean": -4.068016963287846e-08,
|
|
"rewards/rnd_reward/std": 0.9954971194267273,
|
|
"step": 1640,
|
|
"step_time": 10.339221865893341
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09125000033527612,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.3,
|
|
"completions/mean_length": 100.59000167846679,
|
|
"completions/mean_terminated_length": 72.11472930908204,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7371392965316774,
|
|
"epoch": 1.079136690647482,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9994949698448181,
|
|
"kl": 0.08674983121454716,
|
|
"learning_rate": 2.303793328973185e-07,
|
|
"loss": 0.14760398864746094,
|
|
"num_tokens": 33894090.0,
|
|
"reward": -4.775822315394862e-08,
|
|
"reward_std": 0.9951541960239411,
|
|
"rewards/rnd_reward/mean": -4.7758222088134514e-08,
|
|
"rewards/rnd_reward/std": 0.9951542019844055,
|
|
"step": 1650,
|
|
"step_time": 11.004630160145462
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09750000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.6,
|
|
"completions/mean_length": 103.39125213623046,
|
|
"completions/mean_terminated_length": 73.03910675048829,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.719507083296776,
|
|
"epoch": 1.0856769130150425,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0636616945266724,
|
|
"kl": 0.07622466497123241,
|
|
"learning_rate": 2.2874427730542835e-07,
|
|
"loss": 0.11726703643798828,
|
|
"num_tokens": 34065638.0,
|
|
"reward": -6.273389079325397e-08,
|
|
"reward_std": 0.9954430878162384,
|
|
"rewards/rnd_reward/mean": -6.273388937216851e-08,
|
|
"rewards/rnd_reward/std": 0.9954430878162384,
|
|
"step": 1660,
|
|
"step_time": 10.205868328805082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11125000230967999,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 337.9,
|
|
"completions/mean_length": 103.16000251770019,
|
|
"completions/mean_terminated_length": 68.0124439239502,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.555436444282532,
|
|
"epoch": 1.0922171353826031,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4149969816207886,
|
|
"kl": 0.08045686418190598,
|
|
"learning_rate": 2.2710922171353826e-07,
|
|
"loss": 0.18388016223907472,
|
|
"num_tokens": 34243911.0,
|
|
"reward": -6.049871625890546e-08,
|
|
"reward_std": 0.995277750492096,
|
|
"rewards/rnd_reward/mean": -6.049871181801336e-08,
|
|
"rewards/rnd_reward/std": 0.9952777206897736,
|
|
"step": 1670,
|
|
"step_time": 10.405935624614358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09250000230967999,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.5,
|
|
"completions/mean_length": 98.8687515258789,
|
|
"completions/mean_terminated_length": 69.8191291809082,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.679945033788681,
|
|
"epoch": 1.0987573577501635,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1418001651763916,
|
|
"kl": 0.07972979731857777,
|
|
"learning_rate": 2.2547416612164814e-07,
|
|
"loss": 0.08271819949150086,
|
|
"num_tokens": 34422086.0,
|
|
"reward": -2.2441148974650105e-07,
|
|
"reward_std": 0.9950048804283143,
|
|
"rewards/rnd_reward/mean": -2.2441148654905875e-07,
|
|
"rewards/rnd_reward/std": 0.9950048506259919,
|
|
"step": 1680,
|
|
"step_time": 10.670063342596404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000141561032,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.9,
|
|
"completions/mean_length": 95.35000228881836,
|
|
"completions/mean_terminated_length": 67.31705818176269,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6938273787498472,
|
|
"epoch": 1.105297580117724,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0555897951126099,
|
|
"kl": 0.0881148618645966,
|
|
"learning_rate": 2.2383911052975802e-07,
|
|
"loss": 0.057183104753494265,
|
|
"num_tokens": 34592661.0,
|
|
"reward": -5.260109574578564e-08,
|
|
"reward_std": 0.9953494489192962,
|
|
"rewards/rnd_reward/mean": -5.260109787741385e-08,
|
|
"rewards/rnd_reward/std": 0.9953494608402252,
|
|
"step": 1690,
|
|
"step_time": 10.320013692067004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09500000197440386,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 96.38625259399414,
|
|
"completions/mean_terminated_length": 66.25210113525391,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8528232872486115,
|
|
"epoch": 1.1118378024852844,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1842913627624512,
|
|
"kl": 0.09175272611901164,
|
|
"learning_rate": 2.2220405493786787e-07,
|
|
"loss": 0.11493265628814697,
|
|
"num_tokens": 34767800.0,
|
|
"reward": -3.5762789174498266e-08,
|
|
"reward_std": 0.9952230334281922,
|
|
"rewards/rnd_reward/mean": -3.5762784911241854e-08,
|
|
"rewards/rnd_reward/std": 0.9952230513095855,
|
|
"step": 1700,
|
|
"step_time": 11.20805806266144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06625000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.7,
|
|
"completions/mean_length": 88.85,
|
|
"completions/mean_terminated_length": 67.81800193786621,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7997246354818346,
|
|
"epoch": 1.118378024852845,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3541666269302368,
|
|
"kl": 0.0883994361385703,
|
|
"learning_rate": 2.2056899934597775e-07,
|
|
"loss": 0.10645198822021484,
|
|
"num_tokens": 34940290.0,
|
|
"reward": -9.074807572062582e-08,
|
|
"reward_std": 0.9952512562274933,
|
|
"rewards/rnd_reward/mean": -9.074807181264077e-08,
|
|
"rewards/rnd_reward/std": 0.9952512562274933,
|
|
"step": 1710,
|
|
"step_time": 11.268909463495948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08375000078231096,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 341.1,
|
|
"completions/mean_length": 92.34500198364258,
|
|
"completions/mean_terminated_length": 65.63225555419922,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.5688311845064162,
|
|
"epoch": 1.1249182472204056,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.086549997329712,
|
|
"kl": 0.08725747494027018,
|
|
"learning_rate": 2.1893394375408762e-07,
|
|
"loss": 0.09306548833847046,
|
|
"num_tokens": 35107316.0,
|
|
"reward": -1.1682510461241691e-07,
|
|
"reward_std": 0.9951877236366272,
|
|
"rewards/rnd_reward/mean": -1.1682510461241691e-07,
|
|
"rewards/rnd_reward/std": 0.9951877236366272,
|
|
"step": 1720,
|
|
"step_time": 10.925348741887138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000104308128,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.1,
|
|
"completions/mean_length": 97.91875076293945,
|
|
"completions/mean_terminated_length": 70.1454303741455,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.690840741991997,
|
|
"epoch": 1.131458469587966,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0729917287826538,
|
|
"kl": 0.08789877109229564,
|
|
"learning_rate": 2.172988881621975e-07,
|
|
"loss": 0.10979057550430298,
|
|
"num_tokens": 35281881.0,
|
|
"reward": -1.907348732288483e-08,
|
|
"reward_std": 0.995078319311142,
|
|
"rewards/rnd_reward/mean": -1.907348732288483e-08,
|
|
"rewards/rnd_reward/std": 0.9950783312320709,
|
|
"step": 1730,
|
|
"step_time": 10.632008279883303
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000223517418,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.8,
|
|
"completions/mean_length": 96.91125183105468,
|
|
"completions/mean_terminated_length": 69.48632164001465,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.681867742538452,
|
|
"epoch": 1.1379986919555265,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0131276845932007,
|
|
"kl": 0.08944809632375836,
|
|
"learning_rate": 2.1566383257030738e-07,
|
|
"loss": 0.10518434047698974,
|
|
"num_tokens": 35461970.0,
|
|
"reward": 2.6673077968553116e-08,
|
|
"reward_std": 0.9949174404144288,
|
|
"rewards/rnd_reward/mean": 2.6673081521266795e-08,
|
|
"rewards/rnd_reward/std": 0.9949174463748932,
|
|
"step": 1740,
|
|
"step_time": 11.253412028984167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0787500012665987,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 338.7,
|
|
"completions/mean_length": 84.14750061035156,
|
|
"completions/mean_terminated_length": 58.63842086791992,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.567233794927597,
|
|
"epoch": 1.144538914323087,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1535600423812866,
|
|
"kl": 0.09054145393893123,
|
|
"learning_rate": 2.1402877697841726e-07,
|
|
"loss": 0.10667283535003662,
|
|
"num_tokens": 35622023.0,
|
|
"reward": -5.97536580215774e-08,
|
|
"reward_std": 0.9952092111110687,
|
|
"rewards/rnd_reward/mean": -5.975365517940645e-08,
|
|
"rewards/rnd_reward/std": 0.9952091991901397,
|
|
"step": 1750,
|
|
"step_time": 10.509924068162217
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07000000029802322,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 331.8,
|
|
"completions/mean_length": 86.03000106811524,
|
|
"completions/mean_terminated_length": 63.5996639251709,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.5665056079626085,
|
|
"epoch": 1.1510791366906474,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1171236038208008,
|
|
"kl": 0.08619214734062552,
|
|
"learning_rate": 2.1239372138652714e-07,
|
|
"loss": 0.07680293321609497,
|
|
"num_tokens": 35780972.0,
|
|
"reward": -4.529952919440916e-08,
|
|
"reward_std": 0.9950806498527527,
|
|
"rewards/rnd_reward/mean": -4.529952919440916e-08,
|
|
"rewards/rnd_reward/std": 0.9950806617736816,
|
|
"step": 1760,
|
|
"step_time": 10.277455605636352
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06875000074505806,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.5,
|
|
"completions/mean_length": 89.81625213623047,
|
|
"completions/mean_terminated_length": 68.19218559265137,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.563973236083984,
|
|
"epoch": 1.157619359058208,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1521550416946411,
|
|
"kl": 0.0870651968754828,
|
|
"learning_rate": 2.10758665794637e-07,
|
|
"loss": 0.1115831971168518,
|
|
"num_tokens": 35948090.0,
|
|
"reward": 1.3858081859829951e-08,
|
|
"reward_std": 0.9948914706707,
|
|
"rewards/rnd_reward/mean": 1.38580813269229e-08,
|
|
"rewards/rnd_reward/std": 0.9948914825916291,
|
|
"step": 1770,
|
|
"step_time": 10.7476754687028
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07750000115483999,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.2,
|
|
"completions/mean_length": 85.87875022888184,
|
|
"completions/mean_terminated_length": 60.88022689819336,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.646288427710533,
|
|
"epoch": 1.1641595814257686,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9658340811729431,
|
|
"kl": 0.09920121757313609,
|
|
"learning_rate": 2.091236102027469e-07,
|
|
"loss": 0.08330439329147339,
|
|
"num_tokens": 36114108.0,
|
|
"reward": -7.227063285597523e-08,
|
|
"reward_std": 0.9949531733989716,
|
|
"rewards/rnd_reward/mean": -7.227063250070387e-08,
|
|
"rewards/rnd_reward/std": 0.994953167438507,
|
|
"step": 1780,
|
|
"step_time": 11.134026414644904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06250000018626452,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.1,
|
|
"completions/mean_length": 84.51375122070313,
|
|
"completions/mean_terminated_length": 64.53133659362793,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.704926884174347,
|
|
"epoch": 1.170699803793329,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2755341529846191,
|
|
"kl": 0.10716334227472543,
|
|
"learning_rate": 2.0748855461085677e-07,
|
|
"loss": 0.13019881248474122,
|
|
"num_tokens": 36271504.0,
|
|
"reward": -1.305341750246214e-07,
|
|
"reward_std": 0.9949664771556854,
|
|
"rewards/rnd_reward/mean": -1.305341714719077e-07,
|
|
"rewards/rnd_reward/std": 0.9949664771556854,
|
|
"step": 1790,
|
|
"step_time": 10.397847580048255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000175088644,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 326.3,
|
|
"completions/mean_length": 87.63249969482422,
|
|
"completions/mean_terminated_length": 59.75216636657715,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.593474331498146,
|
|
"epoch": 1.1772400261608895,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.155380129814148,
|
|
"kl": 0.09049692396074534,
|
|
"learning_rate": 2.0585349901896665e-07,
|
|
"loss": 0.12421066761016845,
|
|
"num_tokens": 36439860.0,
|
|
"reward": 1.50501714202278e-08,
|
|
"reward_std": 0.9950001060962677,
|
|
"rewards/rnd_reward/mean": 1.505017284131327e-08,
|
|
"rewards/rnd_reward/std": 0.9950001001358032,
|
|
"step": 1800,
|
|
"step_time": 10.84767303718254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06250000158324838,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.8,
|
|
"completions/mean_length": 91.63375244140624,
|
|
"completions/mean_terminated_length": 72.32135162353515,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.5027853429317473,
|
|
"epoch": 1.1837802485284499,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0669174194335938,
|
|
"kl": 0.08057954236865043,
|
|
"learning_rate": 2.042184434270765e-07,
|
|
"loss": 0.09477431774139404,
|
|
"num_tokens": 36602592.0,
|
|
"reward": -1.505017284131327e-07,
|
|
"reward_std": 0.994869589805603,
|
|
"rewards/rnd_reward/mean": -1.5050172876840407e-07,
|
|
"rewards/rnd_reward/std": 0.9948696017265319,
|
|
"step": 1810,
|
|
"step_time": 10.333302733465098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000216066838,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.4,
|
|
"completions/mean_length": 92.49750137329102,
|
|
"completions/mean_terminated_length": 68.43786659240723,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.487256759405136,
|
|
"epoch": 1.1903204708960105,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.864107608795166,
|
|
"kl": 0.07850290769711137,
|
|
"learning_rate": 2.0258338783518638e-07,
|
|
"loss": 0.09753584861755371,
|
|
"num_tokens": 36768390.0,
|
|
"reward": -1.2189149920516229e-07,
|
|
"reward_std": 0.9947888195514679,
|
|
"rewards/rnd_reward/mean": -1.2189150062624775e-07,
|
|
"rewards/rnd_reward/std": 0.9947888076305389,
|
|
"step": 1820,
|
|
"step_time": 10.366325606242754
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07500000111758709,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 327.2,
|
|
"completions/mean_length": 87.97500152587891,
|
|
"completions/mean_terminated_length": 64.1132999420166,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.5121534377336503,
|
|
"epoch": 1.196860693263571,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8458340167999268,
|
|
"kl": 0.08256649980321526,
|
|
"learning_rate": 2.0094833224329626e-07,
|
|
"loss": 0.06835322976112365,
|
|
"num_tokens": 36933960.0,
|
|
"reward": -1.531839394885992e-07,
|
|
"reward_std": 0.9951117813587189,
|
|
"rewards/rnd_reward/mean": -1.5318393664642826e-07,
|
|
"rewards/rnd_reward/std": 0.9951117634773254,
|
|
"step": 1830,
|
|
"step_time": 11.365050849993713
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07375000137835741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.0,
|
|
"completions/mean_length": 94.00125045776367,
|
|
"completions/mean_terminated_length": 71.0650505065918,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.4178095728158953,
|
|
"epoch": 1.2034009156311314,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1388354301452637,
|
|
"kl": 0.0803013733588159,
|
|
"learning_rate": 1.9931327665140614e-07,
|
|
"loss": 0.12810555696487427,
|
|
"num_tokens": 37099636.0,
|
|
"reward": -4.909932687091611e-08,
|
|
"reward_std": 0.9949400007724762,
|
|
"rewards/rnd_reward/mean": -4.909932758145885e-08,
|
|
"rewards/rnd_reward/std": 0.9949400067329407,
|
|
"step": 1840,
|
|
"step_time": 10.505639822757804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06875000093132258,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 341.5,
|
|
"completions/mean_length": 91.8550018310547,
|
|
"completions/mean_terminated_length": 70.10141868591309,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.4157417356967925,
|
|
"epoch": 1.209941137998692,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0894784927368164,
|
|
"kl": 0.08061180328950286,
|
|
"learning_rate": 1.9767822105951601e-07,
|
|
"loss": 0.09716765880584717,
|
|
"num_tokens": 37269225.0,
|
|
"reward": -4.3064359189770585e-08,
|
|
"reward_std": 0.9946663022041321,
|
|
"rewards/rnd_reward/mean": -4.306435528178554e-08,
|
|
"rewards/rnd_reward/std": 0.9946663022041321,
|
|
"step": 1850,
|
|
"step_time": 11.422960984380916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000074505807,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.2,
|
|
"completions/mean_length": 93.6987518310547,
|
|
"completions/mean_terminated_length": 65.78671073913574,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.5770903497934343,
|
|
"epoch": 1.2164813603662525,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9160525798797607,
|
|
"kl": 0.08047085385769606,
|
|
"learning_rate": 1.960431654676259e-07,
|
|
"loss": 0.05968161225318909,
|
|
"num_tokens": 37440344.0,
|
|
"reward": 2.6524067830990816e-08,
|
|
"reward_std": 0.9945189356803894,
|
|
"rewards/rnd_reward/mean": 2.65240635677344e-08,
|
|
"rewards/rnd_reward/std": 0.9945189356803894,
|
|
"step": 1860,
|
|
"step_time": 10.725451961229556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000063329935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 335.1,
|
|
"completions/mean_length": 95.60625076293945,
|
|
"completions/mean_terminated_length": 68.25989952087403,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.4541951596736906,
|
|
"epoch": 1.223021582733813,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.999136209487915,
|
|
"kl": 0.07508358927443623,
|
|
"learning_rate": 1.9440810987573577e-07,
|
|
"loss": 0.13460888862609863,
|
|
"num_tokens": 37611004.0,
|
|
"reward": -1.23679642705099e-07,
|
|
"reward_std": 0.9949141681194306,
|
|
"rewards/rnd_reward/mean": -1.236796327575007e-07,
|
|
"rewards/rnd_reward/std": 0.9949141621589661,
|
|
"step": 1870,
|
|
"step_time": 10.482190745323896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 328.5,
|
|
"completions/mean_length": 89.38125152587891,
|
|
"completions/mean_terminated_length": 65.09402084350586,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.567149502038956,
|
|
"epoch": 1.2295618051013735,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2019909620285034,
|
|
"kl": 0.08660045554861426,
|
|
"learning_rate": 1.9277305428384562e-07,
|
|
"loss": 0.20026679039001466,
|
|
"num_tokens": 37786329.0,
|
|
"reward": -8.866190821521514e-08,
|
|
"reward_std": 0.9948227822780609,
|
|
"rewards/rnd_reward/mean": -8.866190999157198e-08,
|
|
"rewards/rnd_reward/std": 0.9948227941989899,
|
|
"step": 1880,
|
|
"step_time": 11.85091487034224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06375000011175871,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.9,
|
|
"completions/mean_length": 89.55750122070313,
|
|
"completions/mean_terminated_length": 69.49982872009278,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8322397589683534,
|
|
"epoch": 1.236102027468934,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1517750024795532,
|
|
"kl": 0.09399051470682025,
|
|
"learning_rate": 1.9113799869195553e-07,
|
|
"loss": 0.17799322605133056,
|
|
"num_tokens": 37947720.0,
|
|
"reward": -1.0848045732814171e-07,
|
|
"reward_std": 0.9948072552680969,
|
|
"rewards/rnd_reward/mean": -1.0848045448597076e-07,
|
|
"rewards/rnd_reward/std": 0.9948072493076324,
|
|
"step": 1890,
|
|
"step_time": 10.414563778741286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06625000014901161,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.4,
|
|
"completions/mean_length": 86.39500045776367,
|
|
"completions/mean_terminated_length": 65.30856475830078,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7242774665355682,
|
|
"epoch": 1.2426422498364944,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3484736680984497,
|
|
"kl": 0.09410189539194107,
|
|
"learning_rate": 1.895029431000654e-07,
|
|
"loss": 0.0743591845035553,
|
|
"num_tokens": 38111876.0,
|
|
"reward": -6.467104114449285e-08,
|
|
"reward_std": 0.9947612941265106,
|
|
"rewards/rnd_reward/mean": -6.467103901286464e-08,
|
|
"rewards/rnd_reward/std": 0.9947613418102265,
|
|
"step": 1900,
|
|
"step_time": 10.527994227781893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06625000108033419,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 338.0,
|
|
"completions/mean_length": 84.77500114440917,
|
|
"completions/mean_terminated_length": 63.568695068359375,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.954171320796013,
|
|
"epoch": 1.249182472204055,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9760697484016418,
|
|
"kl": 0.10021719867363572,
|
|
"learning_rate": 1.8786788750817528e-07,
|
|
"loss": 0.02971080243587494,
|
|
"num_tokens": 38279376.0,
|
|
"reward": -1.5616416861874428e-07,
|
|
"reward_std": 0.9947503983974457,
|
|
"rewards/rnd_reward/mean": -1.5616416790820153e-07,
|
|
"rewards/rnd_reward/std": 0.9947503864765167,
|
|
"step": 1910,
|
|
"step_time": 11.339212405891157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08000000193715096,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.5,
|
|
"completions/mean_length": 88.47500076293946,
|
|
"completions/mean_terminated_length": 62.80438385009766,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.846560522913933,
|
|
"epoch": 1.2557226945716153,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2370659112930298,
|
|
"kl": 0.09515976402908563,
|
|
"learning_rate": 1.8623283191628514e-07,
|
|
"loss": 0.10529450178146363,
|
|
"num_tokens": 38449416.0,
|
|
"reward": -3.65078278719011e-09,
|
|
"reward_std": 0.9944445490837097,
|
|
"rewards/rnd_reward/mean": -3.6507834977328456e-09,
|
|
"rewards/rnd_reward/std": 0.9944445431232453,
|
|
"step": 1920,
|
|
"step_time": 10.698268468677998
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000160187483,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.8,
|
|
"completions/mean_length": 87.2925018310547,
|
|
"completions/mean_terminated_length": 62.797546768188475,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.9005165725946425,
|
|
"epoch": 1.262262916939176,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3234843015670776,
|
|
"kl": 0.10119491461664439,
|
|
"learning_rate": 1.8459777632439501e-07,
|
|
"loss": 0.014341549575328827,
|
|
"num_tokens": 38616700.0,
|
|
"reward": -4.380941049930698e-08,
|
|
"reward_std": 0.9944035768508911,
|
|
"rewards/rnd_reward/mean": -4.3809414407292026e-08,
|
|
"rewards/rnd_reward/std": 0.994403600692749,
|
|
"step": 1930,
|
|
"step_time": 11.375351972319185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000111758709,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 334.7,
|
|
"completions/mean_length": 96.3550018310547,
|
|
"completions/mean_terminated_length": 68.85774459838868,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.9528768658638,
|
|
"epoch": 1.2688031393067365,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1305567026138306,
|
|
"kl": 0.09469359386712313,
|
|
"learning_rate": 1.829627207325049e-07,
|
|
"loss": 0.0362447589635849,
|
|
"num_tokens": 38788569.0,
|
|
"reward": -5.45382517813664e-08,
|
|
"reward_std": 0.9948436379432678,
|
|
"rewards/rnd_reward/mean": -5.4538249649738193e-08,
|
|
"rewards/rnd_reward/std": 0.9948436439037323,
|
|
"step": 1940,
|
|
"step_time": 10.781422062939964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10000000298023223,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 342.1,
|
|
"completions/mean_length": 93.3875015258789,
|
|
"completions/mean_terminated_length": 61.06837921142578,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7504739254713058,
|
|
"epoch": 1.2753433616742968,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.153893232345581,
|
|
"kl": 0.08643652414903044,
|
|
"learning_rate": 1.8132766514061477e-07,
|
|
"loss": 0.10290535688400268,
|
|
"num_tokens": 38954489.0,
|
|
"reward": -1.2144446595030445e-07,
|
|
"reward_std": 0.9947971343994141,
|
|
"rewards/rnd_reward/mean": -1.2144446346340487e-07,
|
|
"rewards/rnd_reward/std": 0.994797146320343,
|
|
"step": 1950,
|
|
"step_time": 10.527900594798847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0950000025331974,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 329.3,
|
|
"completions/mean_length": 100.58875122070313,
|
|
"completions/mean_terminated_length": 70.79409141540528,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.7141620010137557,
|
|
"epoch": 1.2818835840418574,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0054807662963867,
|
|
"kl": 0.08213893175125123,
|
|
"learning_rate": 1.7969260954872465e-07,
|
|
"loss": 0.07241884469985962,
|
|
"num_tokens": 39126695.0,
|
|
"reward": -1.028180136586343e-08,
|
|
"reward_std": 0.9948840856552124,
|
|
"rewards/rnd_reward/mean": -1.0281802786948902e-08,
|
|
"rewards/rnd_reward/std": 0.9948840975761414,
|
|
"step": 1960,
|
|
"step_time": 10.461326698656194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06750000081956387,
|
|
"completions/max_length": 383.2,
|
|
"completions/max_terminated_length": 350.0,
|
|
"completions/mean_length": 88.08000106811524,
|
|
"completions/mean_terminated_length": 66.5671230316162,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.7966362595558167,
|
|
"epoch": 1.2884238064094178,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1359318494796753,
|
|
"kl": 0.09009936284273863,
|
|
"learning_rate": 1.7805755395683453e-07,
|
|
"loss": 0.12549155950546265,
|
|
"num_tokens": 39294724.0,
|
|
"reward": 4.082918199088681e-08,
|
|
"reward_std": 0.9944938480854034,
|
|
"rewards/rnd_reward/mean": 4.0829182701429545e-08,
|
|
"rewards/rnd_reward/std": 0.9944938659667969,
|
|
"step": 1970,
|
|
"step_time": 10.983743568998761
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.4,
|
|
"completions/mean_length": 95.8025016784668,
|
|
"completions/mean_terminated_length": 71.99091606140136,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7956566989421843,
|
|
"epoch": 1.2949640287769784,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3064550161361694,
|
|
"kl": 0.08204388581216335,
|
|
"learning_rate": 1.764224983649444e-07,
|
|
"loss": 0.06261605024337769,
|
|
"num_tokens": 39465981.0,
|
|
"reward": -7.681548792604076e-08,
|
|
"reward_std": 0.9945617258548737,
|
|
"rewards/rnd_reward/mean": -7.681548792604076e-08,
|
|
"rewards/rnd_reward/std": 0.9945617258548737,
|
|
"step": 1980,
|
|
"step_time": 10.856887764390558
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10500000119209289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 336.7,
|
|
"completions/mean_length": 99.39125137329101,
|
|
"completions/mean_terminated_length": 66.14606056213378,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.809012603759766,
|
|
"epoch": 1.301504251144539,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0052663087844849,
|
|
"kl": 0.08238375652581453,
|
|
"learning_rate": 1.7478744277305426e-07,
|
|
"loss": 0.04824514389038086,
|
|
"num_tokens": 39642064.0,
|
|
"reward": -2.3305416743113483e-07,
|
|
"reward_std": 0.9944523334503174,
|
|
"rewards/rnd_reward/mean": -2.3305416174679294e-07,
|
|
"rewards/rnd_reward/std": 0.9944523453712464,
|
|
"step": 1990,
|
|
"step_time": 10.668248945730738
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06875000093132258,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.9,
|
|
"completions/mean_length": 94.09000167846679,
|
|
"completions/mean_terminated_length": 72.67349357604981,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.782809841632843,
|
|
"epoch": 1.3080444735120995,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.088478684425354,
|
|
"kl": 0.08519633868709206,
|
|
"learning_rate": 1.7315238718116416e-07,
|
|
"loss": 0.12635375261306764,
|
|
"num_tokens": 39807401.0,
|
|
"reward": -9.67830406706227e-08,
|
|
"reward_std": 0.9944651544094085,
|
|
"rewards/rnd_reward/mean": -9.67830406706227e-08,
|
|
"rewards/rnd_reward/std": 0.9944651544094085,
|
|
"step": 2000,
|
|
"step_time": 10.477193391043693
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08125000074505806,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 333.7,
|
|
"completions/mean_length": 97.57875061035156,
|
|
"completions/mean_terminated_length": 72.30035781860352,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.851186656951904,
|
|
"epoch": 1.3145846958796599,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0262951850891113,
|
|
"kl": 0.08686133781448006,
|
|
"learning_rate": 1.7151733158927404e-07,
|
|
"loss": 0.16627882719039916,
|
|
"num_tokens": 39979409.0,
|
|
"reward": -1.8477440306696734e-07,
|
|
"reward_std": 0.9941576004028321,
|
|
"rewards/rnd_reward/mean": -1.8477439738262548e-07,
|
|
"rewards/rnd_reward/std": 0.994157612323761,
|
|
"step": 2010,
|
|
"step_time": 10.494085661577992
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000070780516,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 332.3,
|
|
"completions/mean_length": 98.56625061035156,
|
|
"completions/mean_terminated_length": 74.01468200683594,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6910068154335023,
|
|
"epoch": 1.3211249182472204,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.180172085762024,
|
|
"kl": 0.08328518448397518,
|
|
"learning_rate": 1.6988227599738392e-07,
|
|
"loss": 0.1285926580429077,
|
|
"num_tokens": 40151192.0,
|
|
"reward": -2.4214387792653726e-07,
|
|
"reward_std": 0.9945690989494324,
|
|
"rewards/rnd_reward/mean": -2.421438694000244e-07,
|
|
"rewards/rnd_reward/std": 0.9945691049098968,
|
|
"step": 2020,
|
|
"step_time": 10.840997534058989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06375000085681677,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.1,
|
|
"completions/mean_length": 86.02375068664551,
|
|
"completions/mean_terminated_length": 65.85984230041504,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.954036074876785,
|
|
"epoch": 1.3276651406147808,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1177207231521606,
|
|
"kl": 0.09944681767374278,
|
|
"learning_rate": 1.6824722040549377e-07,
|
|
"loss": 0.07238306999206542,
|
|
"num_tokens": 40313136.0,
|
|
"reward": -2.1941960497429137e-07,
|
|
"reward_std": 0.9945330858230591,
|
|
"rewards/rnd_reward/mean": -2.1941959928994948e-07,
|
|
"rewards/rnd_reward/std": 0.9945330798625946,
|
|
"step": 2030,
|
|
"step_time": 10.803823693329468
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11000000201165676,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 326.8,
|
|
"completions/mean_length": 100.16625137329102,
|
|
"completions/mean_terminated_length": 64.80138359069824,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 3.0097627967596052,
|
|
"epoch": 1.3342053629823414,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1276546716690063,
|
|
"kl": 0.08885867483913898,
|
|
"learning_rate": 1.6661216481360365e-07,
|
|
"loss": 0.010317666083574295,
|
|
"num_tokens": 40489459.0,
|
|
"reward": -1.4454126961993552e-07,
|
|
"reward_std": 0.9944145083427429,
|
|
"rewards/rnd_reward/mean": -1.4454125896179448e-07,
|
|
"rewards/rnd_reward/std": 0.994414496421814,
|
|
"step": 2040,
|
|
"step_time": 11.046867836569437
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09500000067055225,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 329.5,
|
|
"completions/mean_length": 96.87375183105469,
|
|
"completions/mean_terminated_length": 66.7010440826416,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.999319186806679,
|
|
"epoch": 1.340745585349902,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0046088695526123,
|
|
"kl": 0.09102731477469206,
|
|
"learning_rate": 1.6497710922171353e-07,
|
|
"loss": 0.0822763979434967,
|
|
"num_tokens": 40655648.0,
|
|
"reward": -1.0967255033023094e-07,
|
|
"reward_std": 0.9943422853946686,
|
|
"rewards/rnd_reward/mean": -1.0967254464588904e-07,
|
|
"rewards/rnd_reward/std": 0.9943422734737396,
|
|
"step": 2050,
|
|
"step_time": 10.422167311538942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0912500012665987,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.1,
|
|
"completions/mean_length": 97.65625228881837,
|
|
"completions/mean_terminated_length": 68.92426071166992,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8394038140773774,
|
|
"epoch": 1.3472858077174623,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9094915986061096,
|
|
"kl": 0.08663466572761536,
|
|
"learning_rate": 1.633420536298234e-07,
|
|
"loss": 0.020597906410694124,
|
|
"num_tokens": 40822473.0,
|
|
"reward": 1.2516974123855107e-08,
|
|
"reward_std": 0.9942900657653808,
|
|
"rewards/rnd_reward/mean": 1.251697430149079e-08,
|
|
"rewards/rnd_reward/std": 0.9942900657653808,
|
|
"step": 2060,
|
|
"step_time": 10.357499032304622
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09500000141561031,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 323.5,
|
|
"completions/mean_length": 97.2625015258789,
|
|
"completions/mean_terminated_length": 67.25451889038087,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.869212657213211,
|
|
"epoch": 1.353826030085023,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9482806921005249,
|
|
"kl": 0.08622422777116298,
|
|
"learning_rate": 1.6170699803793328e-07,
|
|
"loss": 0.05238085389137268,
|
|
"num_tokens": 40987073.0,
|
|
"reward": -1.0728836219442428e-07,
|
|
"reward_std": 0.9939788460731507,
|
|
"rewards/rnd_reward/mean": -1.0728835935225333e-07,
|
|
"rewards/rnd_reward/std": 0.9939788401126861,
|
|
"step": 2070,
|
|
"step_time": 10.501573320990428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000290572643,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 334.2,
|
|
"completions/mean_length": 91.6250015258789,
|
|
"completions/mean_terminated_length": 65.58112106323242,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.9047418594360352,
|
|
"epoch": 1.3603662524525835,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.018664002418518,
|
|
"kl": 0.08782740281894803,
|
|
"learning_rate": 1.6007194244604316e-07,
|
|
"loss": 0.15487767457962037,
|
|
"num_tokens": 41152698.0,
|
|
"reward": -1.256167863061819e-07,
|
|
"reward_std": 0.9939099371433258,
|
|
"rewards/rnd_reward/mean": -1.2561678914835283e-07,
|
|
"rewards/rnd_reward/std": 0.9939099371433258,
|
|
"step": 2080,
|
|
"step_time": 10.78773567241151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09625000096857547,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.6,
|
|
"completions/mean_length": 100.0337516784668,
|
|
"completions/mean_terminated_length": 69.91814079284669,
|
|
"completions/min_length": 1.9,
|
|
"completions/min_terminated_length": 1.9,
|
|
"entropy": 2.7196634620428086,
|
|
"epoch": 1.3669064748201438,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0979881286621094,
|
|
"kl": 0.0808198869228363,
|
|
"learning_rate": 1.5843688685415304e-07,
|
|
"loss": 0.131181538105011,
|
|
"num_tokens": 41330705.0,
|
|
"reward": -1.9848346672901583e-07,
|
|
"reward_std": 0.9943111777305603,
|
|
"rewards/rnd_reward/mean": -1.9848346672901583e-07,
|
|
"rewards/rnd_reward/std": 0.9943111896514892,
|
|
"step": 2090,
|
|
"step_time": 11.131318410113455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11250000149011612,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.0,
|
|
"completions/mean_length": 100.51125030517578,
|
|
"completions/mean_terminated_length": 64.72891502380371,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.9781338214874267,
|
|
"epoch": 1.3734466971877044,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0100359916687012,
|
|
"kl": 0.08852830519899726,
|
|
"learning_rate": 1.568018312622629e-07,
|
|
"loss": 0.1010061264038086,
|
|
"num_tokens": 41512969.0,
|
|
"reward": -2.2500751484244574e-08,
|
|
"reward_std": 0.9940204381942749,
|
|
"rewards/rnd_reward/mean": -2.2500754326415517e-08,
|
|
"rewards/rnd_reward/std": 0.9940204560756684,
|
|
"step": 2100,
|
|
"step_time": 11.608714979188516
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09250000165775418,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.9,
|
|
"completions/mean_length": 99.50000114440918,
|
|
"completions/mean_terminated_length": 70.67034187316895,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.7862779945135117,
|
|
"epoch": 1.379986919555265,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.939647376537323,
|
|
"kl": 0.08671271968632936,
|
|
"learning_rate": 1.551667756703728e-07,
|
|
"loss": 0.07131583094596863,
|
|
"num_tokens": 41688714.0,
|
|
"reward": -5.654990502534929e-08,
|
|
"reward_std": 0.9939040839672089,
|
|
"rewards/rnd_reward/mean": -5.654991070969118e-08,
|
|
"rewards/rnd_reward/std": 0.9939040839672089,
|
|
"step": 2110,
|
|
"step_time": 10.6546359580243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.2,
|
|
"completions/mean_length": 105.17625045776367,
|
|
"completions/mean_terminated_length": 79.25297164916992,
|
|
"completions/min_length": 1.9,
|
|
"completions/min_terminated_length": 1.9,
|
|
"entropy": 2.7606303304433824,
|
|
"epoch": 1.3865271419228253,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.05146062374115,
|
|
"kl": 0.07692439574748278,
|
|
"learning_rate": 1.5353172007848268e-07,
|
|
"loss": 0.08600187301635742,
|
|
"num_tokens": 41867675.0,
|
|
"reward": -2.2687018805811476e-07,
|
|
"reward_std": 0.9938721656799316,
|
|
"rewards/rnd_reward/mean": -2.2687017953160193e-07,
|
|
"rewards/rnd_reward/std": 0.9938721656799316,
|
|
"step": 2120,
|
|
"step_time": 10.522320462344215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.0,
|
|
"completions/mean_length": 96.24625244140626,
|
|
"completions/mean_terminated_length": 67.99528999328614,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.848968583345413,
|
|
"epoch": 1.393067364290386,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0902374982833862,
|
|
"kl": 0.08595932517200708,
|
|
"learning_rate": 1.5189666448659253e-07,
|
|
"loss": 0.10983504056930542,
|
|
"num_tokens": 42040322.0,
|
|
"reward": -1.4990568644179803e-07,
|
|
"reward_std": 0.9940501570701599,
|
|
"rewards/rnd_reward/mean": -1.4990567649419971e-07,
|
|
"rewards/rnd_reward/std": 0.9940501391887665,
|
|
"step": 2130,
|
|
"step_time": 11.159733151923865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000098720192,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 359.4,
|
|
"completions/mean_length": 102.9712516784668,
|
|
"completions/mean_terminated_length": 77.02480850219726,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.790306252241135,
|
|
"epoch": 1.3996075866579463,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9154292941093445,
|
|
"kl": 0.077430235221982,
|
|
"learning_rate": 1.502616088947024e-07,
|
|
"loss": 0.06642908453941346,
|
|
"num_tokens": 42217629.0,
|
|
"reward": -9.521842088133781e-08,
|
|
"reward_std": 0.9938531458377838,
|
|
"rewards/rnd_reward/mean": -9.521841803916686e-08,
|
|
"rewards/rnd_reward/std": 0.9938531517982483,
|
|
"step": 2140,
|
|
"step_time": 11.171463684784248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000145286321,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.3,
|
|
"completions/mean_length": 102.10125198364258,
|
|
"completions/mean_terminated_length": 75.82226257324218,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.675621399283409,
|
|
"epoch": 1.4061478090255068,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0319130420684814,
|
|
"kl": 0.07909353226423263,
|
|
"learning_rate": 1.4862655330281228e-07,
|
|
"loss": 0.0862727165222168,
|
|
"num_tokens": 42394325.0,
|
|
"reward": -1.949071898987853e-07,
|
|
"reward_std": 0.9938407003879547,
|
|
"rewards/rnd_reward/mean": -1.9490719025405667e-07,
|
|
"rewards/rnd_reward/std": 0.9938406765460968,
|
|
"step": 2150,
|
|
"step_time": 11.259053491754457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07500000111758709,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 324.9,
|
|
"completions/mean_length": 85.76750183105469,
|
|
"completions/mean_terminated_length": 61.72400894165039,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.564385971426964,
|
|
"epoch": 1.4126880313930674,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0090827941894531,
|
|
"kl": 0.08957632025703788,
|
|
"learning_rate": 1.4699149771092216e-07,
|
|
"loss": 0.05871064066886902,
|
|
"num_tokens": 42561024.0,
|
|
"reward": -4.068017274150293e-08,
|
|
"reward_std": 0.9942023098468781,
|
|
"rewards/rnd_reward/mean": -4.068016918878925e-08,
|
|
"rewards/rnd_reward/std": 0.9942023098468781,
|
|
"step": 2160,
|
|
"step_time": 11.197773828869686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000052154064,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.5,
|
|
"completions/mean_length": 94.29875259399414,
|
|
"completions/mean_terminated_length": 67.34189720153809,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.5985243558883666,
|
|
"epoch": 1.4192282537606278,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9199637174606323,
|
|
"kl": 0.08871069457381964,
|
|
"learning_rate": 1.4535644211903204e-07,
|
|
"loss": 0.14661363363265992,
|
|
"num_tokens": 42731253.0,
|
|
"reward": -9.38773203174037e-08,
|
|
"reward_std": 0.9938823819160462,
|
|
"rewards/rnd_reward/mean": -9.387731640941865e-08,
|
|
"rewards/rnd_reward/std": 0.9938823938369751,
|
|
"step": 2170,
|
|
"step_time": 10.55815609311685
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0600000012665987,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.3,
|
|
"completions/mean_length": 83.71750259399414,
|
|
"completions/mean_terminated_length": 64.72726974487304,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.617268776893616,
|
|
"epoch": 1.4257684761281884,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0482478141784668,
|
|
"kl": 0.0882584748789668,
|
|
"learning_rate": 1.4372138652714192e-07,
|
|
"loss": 0.04273238778114319,
|
|
"num_tokens": 42891912.0,
|
|
"reward": -2.068281148126516e-07,
|
|
"reward_std": 0.9936731696128845,
|
|
"rewards/rnd_reward/mean": -2.0682811765482257e-07,
|
|
"rewards/rnd_reward/std": 0.9936731576919555,
|
|
"step": 2180,
|
|
"step_time": 10.923749357438647
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06250000074505806,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.3,
|
|
"completions/mean_length": 86.99125213623047,
|
|
"completions/mean_terminated_length": 67.24082832336425,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.775143876671791,
|
|
"epoch": 1.432308698495749,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1199995279312134,
|
|
"kl": 0.08594764936715364,
|
|
"learning_rate": 1.420863309352518e-07,
|
|
"loss": 0.09395463466644287,
|
|
"num_tokens": 43055435.0,
|
|
"reward": -7.629394431774017e-08,
|
|
"reward_std": 0.9939565777778625,
|
|
"rewards/rnd_reward/mean": -7.629394431774017e-08,
|
|
"rewards/rnd_reward/std": 0.9939565896987915,
|
|
"step": 2190,
|
|
"step_time": 10.720726100169122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05625000055879355,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.9,
|
|
"completions/mean_length": 77.54250106811523,
|
|
"completions/mean_terminated_length": 59.23452491760254,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7800998747348786,
|
|
"epoch": 1.4388489208633093,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2252848148345947,
|
|
"kl": 0.09403526708483696,
|
|
"learning_rate": 1.4045127534336168e-07,
|
|
"loss": 0.072505122423172,
|
|
"num_tokens": 43214484.0,
|
|
"reward": -2.1472573372705028e-07,
|
|
"reward_std": 0.9937775075435639,
|
|
"rewards/rnd_reward/mean": -2.1472573088487935e-07,
|
|
"rewards/rnd_reward/std": 0.9937775015830994,
|
|
"step": 2200,
|
|
"step_time": 11.38782194799278
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000104308128,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.3,
|
|
"completions/mean_length": 93.60000152587891,
|
|
"completions/mean_terminated_length": 67.61266975402832,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.5929481953382494,
|
|
"epoch": 1.4453891432308699,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0342575311660767,
|
|
"kl": 0.08299258435145021,
|
|
"learning_rate": 1.3881621975147153e-07,
|
|
"loss": 0.1498832106590271,
|
|
"num_tokens": 43386444.0,
|
|
"reward": -1.038610928816297e-07,
|
|
"reward_std": 0.9938294231891632,
|
|
"rewards/rnd_reward/mean": -1.0386109572380064e-07,
|
|
"rewards/rnd_reward/std": 0.9938294172286988,
|
|
"step": 2210,
|
|
"step_time": 11.334175962558948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08000000193715096,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.8,
|
|
"completions/mean_length": 88.12125167846679,
|
|
"completions/mean_terminated_length": 62.357414627075194,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.737210324406624,
|
|
"epoch": 1.4519293655984304,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2599236965179443,
|
|
"kl": 0.08750931257382036,
|
|
"learning_rate": 1.3718116415958143e-07,
|
|
"loss": 0.10528864860534667,
|
|
"num_tokens": 43548366.0,
|
|
"reward": -9.566545777772717e-08,
|
|
"reward_std": 0.9939428031444549,
|
|
"rewards/rnd_reward/mean": -9.566545600137033e-08,
|
|
"rewards/rnd_reward/std": 0.993942791223526,
|
|
"step": 2220,
|
|
"step_time": 10.802253967593424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.060000000335276124,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.9,
|
|
"completions/mean_length": 83.26000137329102,
|
|
"completions/mean_terminated_length": 64.11358032226562,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.655692085623741,
|
|
"epoch": 1.4584695879659908,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0874786376953125,
|
|
"kl": 0.09471626300364733,
|
|
"learning_rate": 1.355461085676913e-07,
|
|
"loss": 0.0698157787322998,
|
|
"num_tokens": 43704804.0,
|
|
"reward": -1.7568469061757242e-07,
|
|
"reward_std": 0.9932361245155334,
|
|
"rewards/rnd_reward/mean": -1.7568469061757242e-07,
|
|
"rewards/rnd_reward/std": 0.993236118555069,
|
|
"step": 2230,
|
|
"step_time": 10.654912683716976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07375000119209289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 333.5,
|
|
"completions/mean_length": 86.09000091552734,
|
|
"completions/mean_terminated_length": 62.54329261779785,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.6805451244115828,
|
|
"epoch": 1.4650098103335514,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2531136274337769,
|
|
"kl": 0.09066678332164883,
|
|
"learning_rate": 1.3391105297580116e-07,
|
|
"loss": 0.1158557415008545,
|
|
"num_tokens": 43870226.0,
|
|
"reward": -5.9008597474985435e-08,
|
|
"reward_std": 0.9934276759624481,
|
|
"rewards/rnd_reward/mean": -5.900859711971407e-08,
|
|
"rewards/rnd_reward/std": 0.9934276819229126,
|
|
"step": 2240,
|
|
"step_time": 10.659809662820772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06875000232830644,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 319.8,
|
|
"completions/mean_length": 86.3887508392334,
|
|
"completions/mean_terminated_length": 64.96298866271972,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.90979445874691,
|
|
"epoch": 1.4715500327011117,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8808140754699707,
|
|
"kl": 0.10609434265643358,
|
|
"learning_rate": 1.3227599738391104e-07,
|
|
"loss": 0.051298785209655764,
|
|
"num_tokens": 44036937.0,
|
|
"reward": -1.4841556961187052e-07,
|
|
"reward_std": 0.9932375848293304,
|
|
"rewards/rnd_reward/mean": -1.4841556605915684e-07,
|
|
"rewards/rnd_reward/std": 0.9932375907897949,
|
|
"step": 2250,
|
|
"step_time": 11.205043429811486
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07375000193715095,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 318.5,
|
|
"completions/mean_length": 82.19750175476074,
|
|
"completions/mean_terminated_length": 58.15153121948242,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 3.074227809906006,
|
|
"epoch": 1.4780902550686723,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.282700538635254,
|
|
"kl": 0.11512546790763736,
|
|
"learning_rate": 1.3064094179202092e-07,
|
|
"loss": 0.0818709373474121,
|
|
"num_tokens": 44197515.0,
|
|
"reward": -5.573034247419173e-08,
|
|
"reward_std": 0.9930948257446289,
|
|
"rewards/rnd_reward/mean": -5.573034211892036e-08,
|
|
"rewards/rnd_reward/std": 0.9930948317050934,
|
|
"step": 2260,
|
|
"step_time": 10.878628938994371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06750000081956387,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.7,
|
|
"completions/mean_length": 84.10250320434571,
|
|
"completions/mean_terminated_length": 62.43845558166504,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.938316199183464,
|
|
"epoch": 1.4846304774362329,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2573261260986328,
|
|
"kl": 0.10300196381285787,
|
|
"learning_rate": 1.290058862001308e-07,
|
|
"loss": 0.0356877863407135,
|
|
"num_tokens": 44362002.0,
|
|
"reward": -1.7605722284486093e-07,
|
|
"reward_std": 0.9934640884399414,
|
|
"rewards/rnd_reward/mean": -1.7605722000268997e-07,
|
|
"rewards/rnd_reward/std": 0.9934640944004058,
|
|
"step": 2270,
|
|
"step_time": 11.115813568769955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07750000040978193,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 343.9,
|
|
"completions/mean_length": 87.04125137329102,
|
|
"completions/mean_terminated_length": 62.220666122436526,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.1251493275165556,
|
|
"epoch": 1.4911706998037935,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3937524557113647,
|
|
"kl": 0.10200166283175349,
|
|
"learning_rate": 1.2737083060824068e-07,
|
|
"loss": 0.08271963596343994,
|
|
"num_tokens": 44524310.0,
|
|
"reward": -8.717179245110174e-08,
|
|
"reward_std": 0.9933235943317413,
|
|
"rewards/rnd_reward/mean": -8.717178818784533e-08,
|
|
"rewards/rnd_reward/std": 0.9933236181735993,
|
|
"step": 2280,
|
|
"step_time": 10.662292190524749
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06750000100582838,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.0,
|
|
"completions/mean_length": 84.15750160217286,
|
|
"completions/mean_terminated_length": 62.54970436096191,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.019434627890587,
|
|
"epoch": 1.4977109221713538,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0537996292114258,
|
|
"kl": 0.10465592611581087,
|
|
"learning_rate": 1.2573577501635055e-07,
|
|
"loss": 0.15651319026947022,
|
|
"num_tokens": 44686211.0,
|
|
"reward": -7.629394449537585e-08,
|
|
"reward_std": 0.9937636077404022,
|
|
"rewards/rnd_reward/mean": -7.629394520591859e-08,
|
|
"rewards/rnd_reward/std": 0.9937636256217957,
|
|
"step": 2290,
|
|
"step_time": 10.87905588184949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07375000063329935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 337.0,
|
|
"completions/mean_length": 91.2612518310547,
|
|
"completions/mean_terminated_length": 67.97848358154297,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7947313100099564,
|
|
"epoch": 1.5042511445389142,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9707646369934082,
|
|
"kl": 0.0921105288900435,
|
|
"learning_rate": 1.2410071942446043e-07,
|
|
"loss": 0.037941274046897885,
|
|
"num_tokens": 44854255.0,
|
|
"reward": -3.1441446779467697e-08,
|
|
"reward_std": 0.9936448276042938,
|
|
"rewards/rnd_reward/mean": -3.144145281908095e-08,
|
|
"rewards/rnd_reward/std": 0.9936448395252228,
|
|
"step": 2300,
|
|
"step_time": 10.841944112465717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05875000171363354,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 353.4,
|
|
"completions/mean_length": 82.65125198364258,
|
|
"completions/mean_terminated_length": 63.85377426147461,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.9727181673049925,
|
|
"epoch": 1.5107913669064748,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.044342279434204,
|
|
"kl": 0.10308242114260793,
|
|
"learning_rate": 1.224656638325703e-07,
|
|
"loss": 0.08985200524330139,
|
|
"num_tokens": 45021761.0,
|
|
"reward": -2.3201108554360416e-07,
|
|
"reward_std": 0.9937818229198456,
|
|
"rewards/rnd_reward/mean": -2.3201108447779007e-07,
|
|
"rewards/rnd_reward/std": 0.9937818348407745,
|
|
"step": 2310,
|
|
"step_time": 12.239735689153894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07500000195577741,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 327.8,
|
|
"completions/mean_length": 90.46125106811523,
|
|
"completions/mean_terminated_length": 66.89172592163087,
|
|
"completions/min_length": 2.6,
|
|
"completions/min_terminated_length": 2.6,
|
|
"entropy": 2.8864229738712313,
|
|
"epoch": 1.5173315892740353,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1288195848464966,
|
|
"kl": 0.08688412019982934,
|
|
"learning_rate": 1.2083060824068016e-07,
|
|
"loss": 0.029066413640975952,
|
|
"num_tokens": 45186095.0,
|
|
"reward": 5.736946953049937e-08,
|
|
"reward_std": 0.99327272772789,
|
|
"rewards/rnd_reward/mean": 5.736947450429852e-08,
|
|
"rewards/rnd_reward/std": 0.9932727217674255,
|
|
"step": 2320,
|
|
"step_time": 10.734421823220327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07000000132247805,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 339.1,
|
|
"completions/mean_length": 92.6050018310547,
|
|
"completions/mean_terminated_length": 70.5366439819336,
|
|
"completions/min_length": 1.9,
|
|
"completions/min_terminated_length": 1.9,
|
|
"entropy": 2.7233670204877853,
|
|
"epoch": 1.523871811641596,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.24017333984375,
|
|
"kl": 0.08950559962540865,
|
|
"learning_rate": 1.1919555264879005e-07,
|
|
"loss": 0.07465057373046875,
|
|
"num_tokens": 45354024.0,
|
|
"reward": -2.0265579721012726e-07,
|
|
"reward_std": 0.993443387746811,
|
|
"rewards/rnd_reward/mean": -2.026557901046999e-07,
|
|
"rewards/rnd_reward/std": 0.9934433996677399,
|
|
"step": 2330,
|
|
"step_time": 10.924078989424743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06375000141561031,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.3,
|
|
"completions/mean_length": 85.09000129699707,
|
|
"completions/mean_terminated_length": 64.81442031860351,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.803052818775177,
|
|
"epoch": 1.5304120340091563,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.400153636932373,
|
|
"kl": 0.09231640296056867,
|
|
"learning_rate": 1.1756049705689992e-07,
|
|
"loss": 0.126143217086792,
|
|
"num_tokens": 45510991.0,
|
|
"reward": -1.0177493525720394e-07,
|
|
"reward_std": 0.9933503866195679,
|
|
"rewards/rnd_reward/mean": -1.0177492921759068e-07,
|
|
"rewards/rnd_reward/std": 0.9933504104614258,
|
|
"step": 2340,
|
|
"step_time": 10.554431772092357
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08000000081956386,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 340.3,
|
|
"completions/mean_length": 88.79000091552734,
|
|
"completions/mean_terminated_length": 62.90113410949707,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7890906423330306,
|
|
"epoch": 1.5369522563767168,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1739593744277954,
|
|
"kl": 0.08973503690212965,
|
|
"learning_rate": 1.1592544146500981e-07,
|
|
"loss": 0.016262634098529814,
|
|
"num_tokens": 45673063.0,
|
|
"reward": -2.1778047649689823e-07,
|
|
"reward_std": 0.9933681070804596,
|
|
"rewards/rnd_reward/mean": -2.177804736547273e-07,
|
|
"rewards/rnd_reward/std": 0.9933681011199951,
|
|
"step": 2350,
|
|
"step_time": 10.583909913594834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09750000089406967,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.9,
|
|
"completions/mean_length": 96.30625228881836,
|
|
"completions/mean_terminated_length": 65.25630912780761,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7480629831552505,
|
|
"epoch": 1.5434924787442772,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.265565037727356,
|
|
"kl": 0.08257783968001604,
|
|
"learning_rate": 1.1429038587311968e-07,
|
|
"loss": 0.09627790451049804,
|
|
"num_tokens": 45842103.0,
|
|
"reward": -8.836388873945111e-08,
|
|
"reward_std": 0.9934152483940124,
|
|
"rewards/rnd_reward/mean": -8.836388589728018e-08,
|
|
"rewards/rnd_reward/std": 0.993415230512619,
|
|
"step": 2360,
|
|
"step_time": 10.685579545726068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08375000134110451,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 333.4,
|
|
"completions/mean_length": 91.16250152587891,
|
|
"completions/mean_terminated_length": 64.41967430114747,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.663706639409065,
|
|
"epoch": 1.5500327011118378,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.5513228178024292,
|
|
"kl": 0.08946019792929291,
|
|
"learning_rate": 1.1265533028122955e-07,
|
|
"loss": 0.03821712434291839,
|
|
"num_tokens": 46015743.0,
|
|
"reward": -2.2679567930339316e-07,
|
|
"reward_std": 0.993412846326828,
|
|
"rewards/rnd_reward/mean": -2.2679567575067948e-07,
|
|
"rewards/rnd_reward/std": 0.9934128284454345,
|
|
"step": 2370,
|
|
"step_time": 11.590211745630949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06125000081956387,
|
|
"completions/max_length": 383.4,
|
|
"completions/max_terminated_length": 346.7,
|
|
"completions/mean_length": 89.28625183105468,
|
|
"completions/mean_terminated_length": 69.96004066467285,
|
|
"completions/min_length": 2.7,
|
|
"completions/min_terminated_length": 2.7,
|
|
"entropy": 3.0011739432811737,
|
|
"epoch": 1.5565729234793984,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1020563840866089,
|
|
"kl": 0.09448451111093163,
|
|
"learning_rate": 1.1102027468933943e-07,
|
|
"loss": 0.13594624996185303,
|
|
"num_tokens": 46183567.0,
|
|
"reward": 6.988644907224284e-08,
|
|
"reward_std": 0.9929902851581573,
|
|
"rewards/rnd_reward/mean": 6.98864462300719e-08,
|
|
"rewards/rnd_reward/std": 0.9929902911186218,
|
|
"step": 2380,
|
|
"step_time": 11.415522536402568
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0750000024214387,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.0,
|
|
"completions/mean_length": 90.28625106811523,
|
|
"completions/mean_terminated_length": 66.44939155578614,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.8806687116622927,
|
|
"epoch": 1.563113145846959,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0451889038085938,
|
|
"kl": 0.09008068647235631,
|
|
"learning_rate": 1.0938521909744931e-07,
|
|
"loss": 0.047099918127059937,
|
|
"num_tokens": 46343361.0,
|
|
"reward": -5.662440694109705e-09,
|
|
"reward_std": 0.9929801285266876,
|
|
"rewards/rnd_reward/mean": -5.6624399835669695e-09,
|
|
"rewards/rnd_reward/std": 0.9929801285266876,
|
|
"step": 2390,
|
|
"step_time": 10.55025691683404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000234693289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 333.8,
|
|
"completions/mean_length": 92.99375228881836,
|
|
"completions/mean_terminated_length": 66.80701446533203,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.8602111876010894,
|
|
"epoch": 1.5696533682145193,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2736488580703735,
|
|
"kl": 0.09589690109714866,
|
|
"learning_rate": 1.0775016350555919e-07,
|
|
"loss": 0.09055531024932861,
|
|
"num_tokens": 46510141.0,
|
|
"reward": -1.1608004477636768e-07,
|
|
"reward_std": 0.9930357456207275,
|
|
"rewards/rnd_reward/mean": -1.1608004477636768e-07,
|
|
"rewards/rnd_reward/std": 0.9930357277393341,
|
|
"step": 2400,
|
|
"step_time": 10.752315263729542
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000150874257,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.3,
|
|
"completions/mean_length": 99.20625152587891,
|
|
"completions/mean_terminated_length": 75.86554489135742,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.8277246713638307,
|
|
"epoch": 1.5761935905820796,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0789973735809326,
|
|
"kl": 0.08739803759381175,
|
|
"learning_rate": 1.0611510791366907e-07,
|
|
"loss": 0.06712510585784912,
|
|
"num_tokens": 46679921.0,
|
|
"reward": -9.715557247602646e-08,
|
|
"reward_std": 0.9933283686637878,
|
|
"rewards/rnd_reward/mean": -9.715556821277005e-08,
|
|
"rewards/rnd_reward/std": 0.9933283627033234,
|
|
"step": 2410,
|
|
"step_time": 10.51516141500324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000186264514,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 335.9,
|
|
"completions/mean_length": 99.28125076293945,
|
|
"completions/mean_terminated_length": 72.07235450744629,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6956777185201646,
|
|
"epoch": 1.5827338129496402,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.215997338294983,
|
|
"kl": 0.08504411997273564,
|
|
"learning_rate": 1.0448005232177893e-07,
|
|
"loss": 0.06522908210754394,
|
|
"num_tokens": 46848921.0,
|
|
"reward": -4.142523053474179e-08,
|
|
"reward_std": 0.9928943634033203,
|
|
"rewards/rnd_reward/mean": -4.142522982419905e-08,
|
|
"rewards/rnd_reward/std": 0.9928943693637848,
|
|
"step": 2420,
|
|
"step_time": 10.540731458202936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.06875000121071935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.5,
|
|
"completions/mean_length": 85.0125015258789,
|
|
"completions/mean_terminated_length": 62.73664093017578,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 3.0628849148750303,
|
|
"epoch": 1.5892740353172008,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.336714267730713,
|
|
"kl": 0.10201989021152258,
|
|
"learning_rate": 1.0284499672988881e-07,
|
|
"loss": 0.05233617424964905,
|
|
"num_tokens": 47004126.0,
|
|
"reward": -1.521408613314179e-07,
|
|
"reward_std": 0.9932492971420288,
|
|
"rewards/rnd_reward/mean": -1.5214085848924697e-07,
|
|
"rewards/rnd_reward/std": 0.9932492971420288,
|
|
"step": 2430,
|
|
"step_time": 10.492817461281083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07500000074505805,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 332.8,
|
|
"completions/mean_length": 86.80125122070312,
|
|
"completions/mean_terminated_length": 62.760490036010744,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.799528333544731,
|
|
"epoch": 1.5958142576847614,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4625661373138428,
|
|
"kl": 0.10000391742214561,
|
|
"learning_rate": 1.0120994113799869e-07,
|
|
"loss": 0.031586483120918274,
|
|
"num_tokens": 47168777.0,
|
|
"reward": -7.808208586368437e-08,
|
|
"reward_std": 0.9931057214736938,
|
|
"rewards/rnd_reward/mean": -7.80820865742271e-08,
|
|
"rewards/rnd_reward/std": 0.9931057274341584,
|
|
"step": 2440,
|
|
"step_time": 10.696252883039415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000163912774,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 334.2,
|
|
"completions/mean_length": 88.19125213623047,
|
|
"completions/mean_terminated_length": 62.81879196166992,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.742514914274216,
|
|
"epoch": 1.6023544800523217,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1427501440048218,
|
|
"kl": 0.09996775221079587,
|
|
"learning_rate": 9.957488554610855e-08,
|
|
"loss": 0.041766366362571715,
|
|
"num_tokens": 47331300.0,
|
|
"reward": 5.722046019229765e-08,
|
|
"reward_std": 0.9931129157543183,
|
|
"rewards/rnd_reward/mean": 5.7220459481754915e-08,
|
|
"rewards/rnd_reward/std": 0.9931129038333892,
|
|
"step": 2450,
|
|
"step_time": 10.62438986178022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07874999996274709,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 347.3,
|
|
"completions/mean_length": 90.64625091552735,
|
|
"completions/mean_terminated_length": 65.4418701171875,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.8206116437911986,
|
|
"epoch": 1.6088947024198823,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2679322957992554,
|
|
"kl": 0.0997009389102459,
|
|
"learning_rate": 9.793982995421845e-08,
|
|
"loss": 0.08149058222770691,
|
|
"num_tokens": 47497532.0,
|
|
"reward": 1.2382865293147917e-07,
|
|
"reward_std": 0.9925880253314971,
|
|
"rewards/rnd_reward/mean": 1.238286500893082e-07,
|
|
"rewards/rnd_reward/std": 0.9925880193710327,
|
|
"step": 2460,
|
|
"step_time": 10.703694239328616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08750000111758709,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 341.8,
|
|
"completions/mean_length": 95.64250106811524,
|
|
"completions/mean_terminated_length": 67.96361427307129,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.671072342991829,
|
|
"epoch": 1.6154349247874427,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.008203387260437,
|
|
"kl": 0.09101189225912094,
|
|
"learning_rate": 9.630477436232831e-08,
|
|
"loss": 0.02300444096326828,
|
|
"num_tokens": 47673241.0,
|
|
"reward": -1.6734004162355376e-07,
|
|
"reward_std": 0.9930606245994568,
|
|
"rewards/rnd_reward/mean": -1.6734004304463923e-07,
|
|
"rewards/rnd_reward/std": 0.9930606305599212,
|
|
"step": 2470,
|
|
"step_time": 11.121605953085236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000104308128,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 342.3,
|
|
"completions/mean_length": 91.83125076293945,
|
|
"completions/mean_terminated_length": 63.43943824768066,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.891758808493614,
|
|
"epoch": 1.6219751471550032,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0772875547409058,
|
|
"kl": 0.09280837709084153,
|
|
"learning_rate": 9.466971877043819e-08,
|
|
"loss": 0.032535853981971743,
|
|
"num_tokens": 47836976.0,
|
|
"reward": -1.448392879410676e-07,
|
|
"reward_std": 0.9926415860652924,
|
|
"rewards/rnd_reward/mean": -1.4483928936215307e-07,
|
|
"rewards/rnd_reward/std": 0.9926415801048278,
|
|
"step": 2480,
|
|
"step_time": 10.584856062568724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000201165676,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 343.0,
|
|
"completions/mean_length": 91.06500015258788,
|
|
"completions/mean_terminated_length": 65.96636085510254,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.72680883705616,
|
|
"epoch": 1.6285153695225638,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3315449953079224,
|
|
"kl": 0.09668447291478514,
|
|
"learning_rate": 9.303466317854807e-08,
|
|
"loss": 0.1025187611579895,
|
|
"num_tokens": 48003343.0,
|
|
"reward": -8.106231996762858e-08,
|
|
"reward_std": 0.9925062835216523,
|
|
"rewards/rnd_reward/mean": -8.10623164149149e-08,
|
|
"rewards/rnd_reward/std": 0.9925062835216523,
|
|
"step": 2490,
|
|
"step_time": 10.71222500174772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07000000141561032,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 318.3,
|
|
"completions/mean_length": 85.66875228881835,
|
|
"completions/mean_terminated_length": 63.251688385009764,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.662171345949173,
|
|
"epoch": 1.6350555918901244,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3252993822097778,
|
|
"kl": 0.09045366756618023,
|
|
"learning_rate": 9.139960758665795e-08,
|
|
"loss": 0.0439204216003418,
|
|
"num_tokens": 48168473.0,
|
|
"reward": -3.5464765701931354e-08,
|
|
"reward_std": 0.9928713321685791,
|
|
"rewards/rnd_reward/mean": -3.5464758596503997e-08,
|
|
"rewards/rnd_reward/std": 0.9928713619709015,
|
|
"step": 2500,
|
|
"step_time": 10.784380649239756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.8,
|
|
"completions/mean_length": 96.40625228881837,
|
|
"completions/mean_terminated_length": 68.52096328735351,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.7209132075309754,
|
|
"epoch": 1.6415958142576847,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0920087099075317,
|
|
"kl": 0.08929237592965364,
|
|
"learning_rate": 8.976455199476782e-08,
|
|
"loss": 0.07013883590698242,
|
|
"num_tokens": 48335753.0,
|
|
"reward": -1.1816621103832858e-07,
|
|
"reward_std": 0.9925974249839783,
|
|
"rewards/rnd_reward/mean": -1.1816620961724312e-07,
|
|
"rewards/rnd_reward/std": 0.9925974488258362,
|
|
"step": 2510,
|
|
"step_time": 10.549463000241667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000067055225,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 335.7,
|
|
"completions/mean_length": 94.45250015258789,
|
|
"completions/mean_terminated_length": 66.36481285095215,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6327116012573244,
|
|
"epoch": 1.648136036625245,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.945324182510376,
|
|
"kl": 0.08747830325737596,
|
|
"learning_rate": 8.812949640287769e-08,
|
|
"loss": 0.13749951124191284,
|
|
"num_tokens": 48500460.0,
|
|
"reward": -7.227063427706071e-08,
|
|
"reward_std": 0.9929073750972748,
|
|
"rewards/rnd_reward/mean": -7.227063250070387e-08,
|
|
"rewards/rnd_reward/std": 0.9929073631763459,
|
|
"step": 2520,
|
|
"step_time": 10.486474628048018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000104308129,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.3,
|
|
"completions/mean_length": 92.80625228881836,
|
|
"completions/mean_terminated_length": 68.86259307861329,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.5589567482471467,
|
|
"epoch": 1.6546762589928057,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.173561692237854,
|
|
"kl": 0.08464234415441751,
|
|
"learning_rate": 8.649444081098757e-08,
|
|
"loss": 0.07577463388442993,
|
|
"num_tokens": 48664250.0,
|
|
"reward": -1.436471933402572e-07,
|
|
"reward_std": 0.9925457060337066,
|
|
"rewards/rnd_reward/mean": -1.436471933402572e-07,
|
|
"rewards/rnd_reward/std": 0.9925457000732422,
|
|
"step": 2530,
|
|
"step_time": 10.735207921243273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0800000000745058,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.6,
|
|
"completions/mean_length": 93.55000152587891,
|
|
"completions/mean_terminated_length": 68.25583038330078,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6400635212659838,
|
|
"epoch": 1.6612164813603663,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0104870796203613,
|
|
"kl": 0.08781723519787192,
|
|
"learning_rate": 8.485938521909745e-08,
|
|
"loss": 0.03668081760406494,
|
|
"num_tokens": 48831920.0,
|
|
"reward": -4.6491619976052335e-08,
|
|
"reward_std": 0.9925249576568603,
|
|
"rewards/rnd_reward/mean": -4.6491622818223274e-08,
|
|
"rewards/rnd_reward/std": 0.9925249457359314,
|
|
"step": 2540,
|
|
"step_time": 10.869395459746011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10750000104308129,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 332.1,
|
|
"completions/mean_length": 100.57750244140625,
|
|
"completions/mean_terminated_length": 66.49176979064941,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.7832937359809877,
|
|
"epoch": 1.6677567037279268,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0521481037139893,
|
|
"kl": 0.08043702868744732,
|
|
"learning_rate": 8.322432962720732e-08,
|
|
"loss": 0.05087646245956421,
|
|
"num_tokens": 49003242.0,
|
|
"reward": -6.422400495864622e-08,
|
|
"reward_std": 0.9929387688636779,
|
|
"rewards/rnd_reward/mean": -6.422400566918895e-08,
|
|
"rewards/rnd_reward/std": 0.9929387867450714,
|
|
"step": 2550,
|
|
"step_time": 10.56416127383709
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000104308128,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 337.2,
|
|
"completions/mean_length": 98.30000152587891,
|
|
"completions/mean_terminated_length": 70.51053009033203,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.768531382083893,
|
|
"epoch": 1.6742969260954872,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9573103785514832,
|
|
"kl": 0.08502876814454793,
|
|
"learning_rate": 8.158927403531719e-08,
|
|
"loss": 0.0772208571434021,
|
|
"num_tokens": 49174652.0,
|
|
"reward": -1.469254531372144e-07,
|
|
"reward_std": 0.9924641728401185,
|
|
"rewards/rnd_reward/mean": -1.4692545029504346e-07,
|
|
"rewards/rnd_reward/std": 0.9924641907215118,
|
|
"step": 2560,
|
|
"step_time": 10.583414485724642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.7,
|
|
"completions/mean_length": 96.5462516784668,
|
|
"completions/mean_terminated_length": 69.8155361175537,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.8294411927461622,
|
|
"epoch": 1.6808371484630478,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9854137301445007,
|
|
"kl": 0.09185233255848288,
|
|
"learning_rate": 7.995421844342708e-08,
|
|
"loss": 0.1261582612991333,
|
|
"num_tokens": 49354119.0,
|
|
"reward": 8.67247596403331e-08,
|
|
"reward_std": 0.9922890484333038,
|
|
"rewards/rnd_reward/mean": 8.67247596403331e-08,
|
|
"rewards/rnd_reward/std": 0.9922890365123749,
|
|
"step": 2570,
|
|
"step_time": 11.603049716376699
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000052154064,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.7,
|
|
"completions/mean_length": 92.55000152587891,
|
|
"completions/mean_terminated_length": 67.6265712738037,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7690848499536513,
|
|
"epoch": 1.6873773708306081,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1284137964248657,
|
|
"kl": 0.09411861123517155,
|
|
"learning_rate": 7.831916285153695e-08,
|
|
"loss": 0.13598878383636476,
|
|
"num_tokens": 49524444.0,
|
|
"reward": 6.571412010103473e-08,
|
|
"reward_std": 0.9924658596515655,
|
|
"rewards/rnd_reward/mean": 6.571412010103473e-08,
|
|
"rewards/rnd_reward/std": 0.9924658596515655,
|
|
"step": 2580,
|
|
"step_time": 11.369293246162124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09000000208616257,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.5,
|
|
"completions/mean_length": 100.7300018310547,
|
|
"completions/mean_terminated_length": 72.78359603881836,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.656350103020668,
|
|
"epoch": 1.6939175931981687,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1325582265853882,
|
|
"kl": 0.08164091287180782,
|
|
"learning_rate": 7.668410725964682e-08,
|
|
"loss": 0.09759321212768554,
|
|
"num_tokens": 49702873.0,
|
|
"reward": -8.478760804564445e-08,
|
|
"reward_std": 0.9928931176662446,
|
|
"rewards/rnd_reward/mean": -8.478760946672991e-08,
|
|
"rewards/rnd_reward/std": 0.9928931057453155,
|
|
"step": 2590,
|
|
"step_time": 10.856351627549156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07000000141561032,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.6,
|
|
"completions/mean_length": 91.03375091552735,
|
|
"completions/mean_terminated_length": 69.03646697998047,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.887327644228935,
|
|
"epoch": 1.7004578155657293,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3039114475250244,
|
|
"kl": 0.0926243552006781,
|
|
"learning_rate": 7.50490516677567e-08,
|
|
"loss": 0.0819834053516388,
|
|
"num_tokens": 49876500.0,
|
|
"reward": 1.3411066479562806e-09,
|
|
"reward_std": 0.9926319658756256,
|
|
"rewards/rnd_reward/mean": 1.3411066479562806e-09,
|
|
"rewards/rnd_reward/std": 0.9926319539546966,
|
|
"step": 2600,
|
|
"step_time": 11.05162076470442
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08125000149011612,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 354.7,
|
|
"completions/mean_length": 89.97250061035156,
|
|
"completions/mean_terminated_length": 63.9521728515625,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.807651326060295,
|
|
"epoch": 1.7069980379332899,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0977989435195923,
|
|
"kl": 0.08759319856762886,
|
|
"learning_rate": 7.341399607586658e-08,
|
|
"loss": 0.13040846586227417,
|
|
"num_tokens": 50044628.0,
|
|
"reward": -6.228685363041109e-08,
|
|
"reward_std": 0.9921288013458252,
|
|
"rewards/rnd_reward/mean": -6.228685291986836e-08,
|
|
"rewards/rnd_reward/std": 0.9921288132667542,
|
|
"step": 2610,
|
|
"step_time": 11.296212270553223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08375000227242708,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.6,
|
|
"completions/mean_length": 91.39750061035156,
|
|
"completions/mean_terminated_length": 64.60456695556641,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.771796461939812,
|
|
"epoch": 1.7135382603008502,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.101976990699768,
|
|
"kl": 0.09053975995630026,
|
|
"learning_rate": 7.177894048397645e-08,
|
|
"loss": 0.07479513287544251,
|
|
"num_tokens": 50204246.0,
|
|
"reward": -1.3120472530658844e-07,
|
|
"reward_std": 0.9925267994403839,
|
|
"rewards/rnd_reward/mean": -1.3120472530658844e-07,
|
|
"rewards/rnd_reward/std": 0.9925268173217774,
|
|
"step": 2620,
|
|
"step_time": 10.44964153978508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375000074505806,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 325.7,
|
|
"completions/mean_length": 95.06375122070312,
|
|
"completions/mean_terminated_length": 65.27039031982422,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6531850188970565,
|
|
"epoch": 1.7200784826684106,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.112268090248108,
|
|
"kl": 0.08289336860179901,
|
|
"learning_rate": 7.014388489208632e-08,
|
|
"loss": 0.08899658918380737,
|
|
"num_tokens": 50373747.0,
|
|
"reward": -2.539157884839938e-07,
|
|
"reward_std": 0.9920955657958984,
|
|
"rewards/rnd_reward/mean": -2.539157849312801e-07,
|
|
"rewards/rnd_reward/std": 0.9920955717563629,
|
|
"step": 2630,
|
|
"step_time": 10.710641134064645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000201165676,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 357.8,
|
|
"completions/mean_length": 93.92000198364258,
|
|
"completions/mean_terminated_length": 69.12318840026856,
|
|
"completions/min_length": 2.7,
|
|
"completions/min_terminated_length": 2.7,
|
|
"entropy": 2.5591167956590652,
|
|
"epoch": 1.7266187050359711,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0098210573196411,
|
|
"kl": 0.08759762533009052,
|
|
"learning_rate": 6.85088293001962e-08,
|
|
"loss": 0.11666052341461182,
|
|
"num_tokens": 50542968.0,
|
|
"reward": -2.8014184039193423e-08,
|
|
"reward_std": 0.9922697126865387,
|
|
"rewards/rnd_reward/mean": -2.8014186881364366e-08,
|
|
"rewards/rnd_reward/std": 0.9922697246074677,
|
|
"step": 2640,
|
|
"step_time": 10.810134377889336
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09000000171363354,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 327.5,
|
|
"completions/mean_length": 90.80375213623047,
|
|
"completions/mean_terminated_length": 61.77821655273438,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.779536947607994,
|
|
"epoch": 1.7331589274035317,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1324180364608765,
|
|
"kl": 0.08631603391841054,
|
|
"learning_rate": 6.687377370830608e-08,
|
|
"loss": 0.09856376647949219,
|
|
"num_tokens": 50707726.0,
|
|
"reward": -5.237758227139011e-08,
|
|
"reward_std": 0.992433100938797,
|
|
"rewards/rnd_reward/mean": -5.237758085030464e-08,
|
|
"rewards/rnd_reward/std": 0.9924331188201905,
|
|
"step": 2650,
|
|
"step_time": 10.956776090711354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375000186264515,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 334.8,
|
|
"completions/mean_length": 97.68875045776367,
|
|
"completions/mean_terminated_length": 68.05043869018554,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.7804177552461624,
|
|
"epoch": 1.7396991497710923,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0591152906417847,
|
|
"kl": 0.09146134741604328,
|
|
"learning_rate": 6.523871811641596e-08,
|
|
"loss": 0.04577620327472687,
|
|
"num_tokens": 50878792.0,
|
|
"reward": -7.227059839465255e-09,
|
|
"reward_std": 0.9924111425876617,
|
|
"rewards/rnd_reward/mean": -7.227060550007991e-09,
|
|
"rewards/rnd_reward/std": 0.9924111247062684,
|
|
"step": 2660,
|
|
"step_time": 10.708624366554432
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000108033419,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 330.3,
|
|
"completions/mean_length": 95.15875205993652,
|
|
"completions/mean_terminated_length": 68.52624740600587,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.626550641655922,
|
|
"epoch": 1.7462393721386529,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0052024126052856,
|
|
"kl": 0.09057780019938946,
|
|
"learning_rate": 6.360366252452582e-08,
|
|
"loss": 0.03838871717453003,
|
|
"num_tokens": 51047709.0,
|
|
"reward": -1.60261992654398e-07,
|
|
"reward_std": 0.9925138711929321,
|
|
"rewards/rnd_reward/mean": -1.6026198892404864e-07,
|
|
"rewards/rnd_reward/std": 0.9925138831138611,
|
|
"step": 2670,
|
|
"step_time": 10.687237256928347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07000000113621355,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.8,
|
|
"completions/mean_length": 91.47625198364258,
|
|
"completions/mean_terminated_length": 69.65823402404786,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6517731666564943,
|
|
"epoch": 1.7527795945062132,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.085845947265625,
|
|
"kl": 0.08733204193413258,
|
|
"learning_rate": 6.19686069326357e-08,
|
|
"loss": 0.126353120803833,
|
|
"num_tokens": 51221270.0,
|
|
"reward": -1.2561679199052378e-07,
|
|
"reward_std": 0.9924490749835968,
|
|
"rewards/rnd_reward/mean": -1.2561678275346821e-07,
|
|
"rewards/rnd_reward/std": 0.9924490511417389,
|
|
"step": 2680,
|
|
"step_time": 11.201374626043252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000150874257,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 339.3,
|
|
"completions/mean_length": 97.07750091552734,
|
|
"completions/mean_terminated_length": 71.1569839477539,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.534248262643814,
|
|
"epoch": 1.7593198168737736,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1865549087524414,
|
|
"kl": 0.07945491755381226,
|
|
"learning_rate": 6.033355134074558e-08,
|
|
"loss": 0.023079755902290344,
|
|
"num_tokens": 51391307.0,
|
|
"reward": 1.2665985660298417e-08,
|
|
"reward_std": 0.9924781501293183,
|
|
"rewards/rnd_reward/mean": 1.2665986370841154e-08,
|
|
"rewards/rnd_reward/std": 0.9924781501293183,
|
|
"step": 2690,
|
|
"step_time": 10.600427453825251
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10375000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.4,
|
|
"completions/mean_length": 102.50500183105468,
|
|
"completions/mean_terminated_length": 69.94977340698242,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.5867824822664263,
|
|
"epoch": 1.7658600392413342,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9612109661102295,
|
|
"kl": 0.08055433174595236,
|
|
"learning_rate": 5.869849574885546e-08,
|
|
"loss": 0.09532687664031983,
|
|
"num_tokens": 51566461.0,
|
|
"reward": -4.351139040181806e-08,
|
|
"reward_std": 0.9921834826469421,
|
|
"rewards/rnd_reward/mean": -4.351139040181806e-08,
|
|
"rewards/rnd_reward/std": 0.9921834707260132,
|
|
"step": 2700,
|
|
"step_time": 10.596421872614883
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09625000208616256,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.4,
|
|
"completions/mean_length": 103.48625259399414,
|
|
"completions/mean_terminated_length": 73.66226654052734,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.574134039878845,
|
|
"epoch": 1.7724002616088947,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8851132392883301,
|
|
"kl": 0.07895892215892672,
|
|
"learning_rate": 5.706344015696534e-08,
|
|
"loss": 0.03208762109279632,
|
|
"num_tokens": 51739420.0,
|
|
"reward": 1.4901161127234275e-08,
|
|
"reward_std": 0.9922367691993713,
|
|
"rewards/rnd_reward/mean": 1.4901161482505642e-08,
|
|
"rewards/rnd_reward/std": 0.992236751317978,
|
|
"step": 2710,
|
|
"step_time": 10.52075885424856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07750000096857548,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.7,
|
|
"completions/mean_length": 98.13000183105468,
|
|
"completions/mean_terminated_length": 74.1987533569336,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.7050154358148575,
|
|
"epoch": 1.7789404839764553,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2265753746032715,
|
|
"kl": 0.08297120407223701,
|
|
"learning_rate": 5.542838456507521e-08,
|
|
"loss": 0.07181986570358276,
|
|
"num_tokens": 51906604.0,
|
|
"reward": 2.1010632700324548e-08,
|
|
"reward_std": 0.9923449456691742,
|
|
"rewards/rnd_reward/mean": 2.101064406900832e-08,
|
|
"rewards/rnd_reward/std": 0.9923449575901031,
|
|
"step": 2720,
|
|
"step_time": 10.726289883395657
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10125000029802322,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.8,
|
|
"completions/mean_length": 97.52250137329102,
|
|
"completions/mean_terminated_length": 65.36152725219726,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.5555239796638487,
|
|
"epoch": 1.7854807063440157,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.344201922416687,
|
|
"kl": 0.07681396808475256,
|
|
"learning_rate": 5.379332897318509e-08,
|
|
"loss": 0.09209707379341125,
|
|
"num_tokens": 52078602.0,
|
|
"reward": 7.450558392463336e-10,
|
|
"reward_std": 0.991978645324707,
|
|
"rewards/rnd_reward/mean": 7.450586814172766e-10,
|
|
"rewards/rnd_reward/std": 0.9919786393642426,
|
|
"step": 2730,
|
|
"step_time": 10.693091615103185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09500000141561031,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.1,
|
|
"completions/mean_length": 99.98125228881835,
|
|
"completions/mean_terminated_length": 70.13334350585937,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.684102329611778,
|
|
"epoch": 1.792020928711576,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1477556228637695,
|
|
"kl": 0.08274123426526785,
|
|
"learning_rate": 5.2158273381294966e-08,
|
|
"loss": 0.025458133220672606,
|
|
"num_tokens": 52245797.0,
|
|
"reward": 3.457069563950199e-08,
|
|
"reward_std": 0.9920007467269898,
|
|
"rewards/rnd_reward/mean": 3.457069741585883e-08,
|
|
"rewards/rnd_reward/std": 0.9920007586479187,
|
|
"step": 2740,
|
|
"step_time": 10.481961290817708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07750000078231097,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 339.5,
|
|
"completions/mean_length": 91.90375061035157,
|
|
"completions/mean_terminated_length": 67.26799354553222,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.7344156444072723,
|
|
"epoch": 1.7985611510791366,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1605439186096191,
|
|
"kl": 0.08739866856485605,
|
|
"learning_rate": 5.052321778940484e-08,
|
|
"loss": 0.0825818121433258,
|
|
"num_tokens": 52417430.0,
|
|
"reward": -5.275011112360062e-08,
|
|
"reward_std": 0.9921901881694793,
|
|
"rewards/rnd_reward/mean": -5.275011254468609e-08,
|
|
"rewards/rnd_reward/std": 0.992190170288086,
|
|
"step": 2750,
|
|
"step_time": 11.562560918554663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000126659871,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 355.7,
|
|
"completions/mean_length": 95.38625106811523,
|
|
"completions/mean_terminated_length": 68.69163131713867,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.508667880296707,
|
|
"epoch": 1.8051013734466972,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1222717761993408,
|
|
"kl": 0.08437787899747491,
|
|
"learning_rate": 4.8888162197514716e-08,
|
|
"loss": 0.08072326779365539,
|
|
"num_tokens": 52587249.0,
|
|
"reward": -3.2708050046181825e-08,
|
|
"reward_std": 0.9921342968940735,
|
|
"rewards/rnd_reward/mean": -3.270804720401088e-08,
|
|
"rewards/rnd_reward/std": 0.9921342968940735,
|
|
"step": 2760,
|
|
"step_time": 10.81694066869095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09125000275671483,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 342.4,
|
|
"completions/mean_length": 100.94500198364258,
|
|
"completions/mean_terminated_length": 72.59214401245117,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.762883481383324,
|
|
"epoch": 1.8116415958142578,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3192421197891235,
|
|
"kl": 0.08027938418090344,
|
|
"learning_rate": 4.725310660562459e-08,
|
|
"loss": 0.10753175020217895,
|
|
"num_tokens": 52769960.0,
|
|
"reward": 5.5879356897214906e-08,
|
|
"reward_std": 0.9917623281478882,
|
|
"rewards/rnd_reward/mean": 5.587935405504396e-08,
|
|
"rewards/rnd_reward/std": 0.9917623460292816,
|
|
"step": 2770,
|
|
"step_time": 11.582827594131231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0787500012665987,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.5,
|
|
"completions/mean_length": 95.13875122070313,
|
|
"completions/mean_terminated_length": 70.49329643249511,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.6798209667205812,
|
|
"epoch": 1.8181818181818183,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1080927848815918,
|
|
"kl": 0.08364078365266323,
|
|
"learning_rate": 4.5618051013734466e-08,
|
|
"loss": 0.04997023940086365,
|
|
"num_tokens": 52940211.0,
|
|
"reward": -9.596347978479969e-08,
|
|
"reward_std": 0.9920522153377533,
|
|
"rewards/rnd_reward/mean": -9.596347978479969e-08,
|
|
"rewards/rnd_reward/std": 0.9920521914958954,
|
|
"step": 2780,
|
|
"step_time": 10.742116263648494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000044703483,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 326.3,
|
|
"completions/mean_length": 93.41375198364258,
|
|
"completions/mean_terminated_length": 65.94217224121094,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.7876433610916136,
|
|
"epoch": 1.8247220405493787,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2520111799240112,
|
|
"kl": 0.08399344785138965,
|
|
"learning_rate": 4.398299542184434e-08,
|
|
"loss": 0.11648783683776856,
|
|
"num_tokens": 53106412.0,
|
|
"reward": -1.3023615004215116e-07,
|
|
"reward_std": 0.9917174458503724,
|
|
"rewards/rnd_reward/mean": -1.3023615146323665e-07,
|
|
"rewards/rnd_reward/std": 0.9917174398899078,
|
|
"step": 2790,
|
|
"step_time": 10.729628002620302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000029802323,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 350.1,
|
|
"completions/mean_length": 95.42125091552734,
|
|
"completions/mean_terminated_length": 67.35246047973632,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.751756873726845,
|
|
"epoch": 1.831262262916939,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2754117250442505,
|
|
"kl": 0.08136667888611555,
|
|
"learning_rate": 4.2347939829954215e-08,
|
|
"loss": 0.10360376834869385,
|
|
"num_tokens": 53273129.0,
|
|
"reward": -8.925795818015558e-08,
|
|
"reward_std": 0.992232620716095,
|
|
"rewards/rnd_reward/mean": -8.92579546274419e-08,
|
|
"rewards/rnd_reward/std": 0.9922326266765594,
|
|
"step": 2800,
|
|
"step_time": 10.579858994716778
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000178813935,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 349.3,
|
|
"completions/mean_length": 101.0425018310547,
|
|
"completions/mean_terminated_length": 75.79266548156738,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.514050582051277,
|
|
"epoch": 1.8378024852844996,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.292152762413025,
|
|
"kl": 0.07886797543615102,
|
|
"learning_rate": 4.0712884238064094e-08,
|
|
"loss": 0.07606931924819946,
|
|
"num_tokens": 53448303.0,
|
|
"reward": -2.5182961227443456e-08,
|
|
"reward_std": 0.9920957505702972,
|
|
"rewards/rnd_reward/mean": -2.5182961227443456e-08,
|
|
"rewards/rnd_reward/std": 0.9920957386493683,
|
|
"step": 2810,
|
|
"step_time": 11.144761186954565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000126659871,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 348.3,
|
|
"completions/mean_length": 99.48500213623046,
|
|
"completions/mean_terminated_length": 73.15060997009277,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.75241442322731,
|
|
"epoch": 1.8443427076520602,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0998656749725342,
|
|
"kl": 0.07965973112732172,
|
|
"learning_rate": 3.9077828646173965e-08,
|
|
"loss": 0.08108996152877808,
|
|
"num_tokens": 53620201.0,
|
|
"reward": 1.832842806415158e-08,
|
|
"reward_std": 0.9918050169944763,
|
|
"rewards/rnd_reward/mean": 1.832843197213663e-08,
|
|
"rewards/rnd_reward/std": 0.9918050229549408,
|
|
"step": 2820,
|
|
"step_time": 11.469240892818197
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000119209289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 346.9,
|
|
"completions/mean_length": 97.51125106811523,
|
|
"completions/mean_terminated_length": 70.54906845092773,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.549181741476059,
|
|
"epoch": 1.8508829300196208,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0469340085983276,
|
|
"kl": 0.0786884649656713,
|
|
"learning_rate": 3.7442773054283844e-08,
|
|
"loss": 0.05131007432937622,
|
|
"num_tokens": 53801605.0,
|
|
"reward": -5.766749477942312e-08,
|
|
"reward_std": 0.9918078780174255,
|
|
"rewards/rnd_reward/mean": -5.766749477942312e-08,
|
|
"rewards/rnd_reward/std": 0.9918078899383544,
|
|
"step": 2830,
|
|
"step_time": 11.47003021549899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07375000119209289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.9,
|
|
"completions/mean_length": 91.74375228881836,
|
|
"completions/mean_terminated_length": 68.46907119750976,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.741053342819214,
|
|
"epoch": 1.8574231523871811,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0931239128112793,
|
|
"kl": 0.0858668725937605,
|
|
"learning_rate": 3.580771746239372e-08,
|
|
"loss": 0.01960263103246689,
|
|
"num_tokens": 53967620.0,
|
|
"reward": -8.866190732703672e-08,
|
|
"reward_std": 0.9914042472839355,
|
|
"rewards/rnd_reward/mean": -8.866190661649398e-08,
|
|
"rewards/rnd_reward/std": 0.9914042532444001,
|
|
"step": 2840,
|
|
"step_time": 11.055267255497165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07875000052154064,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 94.5212516784668,
|
|
"completions/mean_terminated_length": 69.70959434509277,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.616148290038109,
|
|
"epoch": 1.8639633747547415,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1507196426391602,
|
|
"kl": 0.07959493407979608,
|
|
"learning_rate": 3.41726618705036e-08,
|
|
"loss": 0.09600520730018616,
|
|
"num_tokens": 54134747.0,
|
|
"reward": -1.1116266360744475e-07,
|
|
"reward_std": 0.9913456678390503,
|
|
"rewards/rnd_reward/mean": -1.1116266360744475e-07,
|
|
"rewards/rnd_reward/std": 0.9913456499576568,
|
|
"step": 2850,
|
|
"step_time": 10.70933379684575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08000000175088644,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.4,
|
|
"completions/mean_length": 92.95874938964843,
|
|
"completions/mean_terminated_length": 67.40721321105957,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.561130478978157,
|
|
"epoch": 1.870503597122302,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9997308254241943,
|
|
"kl": 0.08147254046052695,
|
|
"learning_rate": 3.253760627861347e-08,
|
|
"loss": 0.09602594375610352,
|
|
"num_tokens": 54297779.0,
|
|
"reward": -1.2561679305633788e-07,
|
|
"reward_std": 0.9915251851081848,
|
|
"rewards/rnd_reward/mean": -1.256167838192823e-07,
|
|
"rewards/rnd_reward/std": 0.9915251791477203,
|
|
"step": 2860,
|
|
"step_time": 10.617276117694564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07625000048428773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 324.1,
|
|
"completions/mean_length": 94.39250106811524,
|
|
"completions/mean_terminated_length": 70.46714973449707,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.5941164553165437,
|
|
"epoch": 1.8770438194898627,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0238882303237915,
|
|
"kl": 0.08611528370529413,
|
|
"learning_rate": 3.090255068672335e-08,
|
|
"loss": 0.1147921085357666,
|
|
"num_tokens": 54467228.0,
|
|
"reward": -1.1593103739215849e-07,
|
|
"reward_std": 0.9916765570640564,
|
|
"rewards/rnd_reward/mean": -1.159310317078166e-07,
|
|
"rewards/rnd_reward/std": 0.9916765511035919,
|
|
"step": 2870,
|
|
"step_time": 11.231190301850438
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375000149011611,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.1,
|
|
"completions/mean_length": 96.7587516784668,
|
|
"completions/mean_terminated_length": 67.1300651550293,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 2.5091202437877653,
|
|
"epoch": 1.8835840418574232,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9646898508071899,
|
|
"kl": 0.08239567633718252,
|
|
"learning_rate": 2.9267495094833225e-08,
|
|
"loss": 0.110014808177948,
|
|
"num_tokens": 54638260.0,
|
|
"reward": -1.0073185103820493e-07,
|
|
"reward_std": 0.9914380252361298,
|
|
"rewards/rnd_reward/mean": -1.0073185103820493e-07,
|
|
"rewards/rnd_reward/std": 0.9914380311965942,
|
|
"step": 2880,
|
|
"step_time": 10.927705601369961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1087500024586916,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 336.5,
|
|
"completions/mean_length": 104.54500198364258,
|
|
"completions/mean_terminated_length": 70.47102432250976,
|
|
"completions/min_length": 2.3,
|
|
"completions/min_terminated_length": 2.3,
|
|
"entropy": 2.605569988489151,
|
|
"epoch": 1.8901242642249838,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9842224717140198,
|
|
"kl": 0.07593777822330594,
|
|
"learning_rate": 2.7632439502943097e-08,
|
|
"loss": 0.09672788381576539,
|
|
"num_tokens": 54823046.0,
|
|
"reward": 4.0531157274870114e-08,
|
|
"reward_std": 0.9914953529834747,
|
|
"rewards/rnd_reward/mean": 4.053116011704105e-08,
|
|
"rewards/rnd_reward/std": 0.9914953589439393,
|
|
"step": 2890,
|
|
"step_time": 12.156373349227943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0912500012665987,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 362.0,
|
|
"completions/mean_length": 101.96250228881836,
|
|
"completions/mean_terminated_length": 73.72331771850585,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.55685573220253,
|
|
"epoch": 1.8966644865925442,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0599300861358643,
|
|
"kl": 0.07744721919298173,
|
|
"learning_rate": 2.5997383911052975e-08,
|
|
"loss": 0.07260736227035522,
|
|
"num_tokens": 54993921.0,
|
|
"reward": -7.45057977535879e-09,
|
|
"reward_std": 0.9911764979362487,
|
|
"rewards/rnd_reward/mean": -7.450580485901526e-09,
|
|
"rewards/rnd_reward/std": 0.9911765158176422,
|
|
"step": 2900,
|
|
"step_time": 10.672014983766712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000160187482,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 352.5,
|
|
"completions/mean_length": 97.90375137329102,
|
|
"completions/mean_terminated_length": 70.14430389404296,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.7193702697753905,
|
|
"epoch": 1.9032047089601045,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4999836683273315,
|
|
"kl": 0.08308390835300088,
|
|
"learning_rate": 2.436232831916285e-08,
|
|
"loss": 0.07153651714324952,
|
|
"num_tokens": 55168059.0,
|
|
"reward": -1.7181038884928058e-07,
|
|
"reward_std": 0.991384220123291,
|
|
"rewards/rnd_reward/mean": -1.7181038458602416e-07,
|
|
"rewards/rnd_reward/std": 0.9913842082023621,
|
|
"step": 2910,
|
|
"step_time": 11.116872487938963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09875000230967998,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.8,
|
|
"completions/mean_length": 100.51500244140625,
|
|
"completions/mean_terminated_length": 69.35774002075195,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.603737047314644,
|
|
"epoch": 1.909744931327665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0500236749649048,
|
|
"kl": 0.08141210982576012,
|
|
"learning_rate": 2.2727272727272725e-08,
|
|
"loss": 0.02487606555223465,
|
|
"num_tokens": 55350021.0,
|
|
"reward": -4.3809413696749286e-08,
|
|
"reward_std": 0.9915971457958221,
|
|
"rewards/rnd_reward/mean": -4.3809414407292026e-08,
|
|
"rewards/rnd_reward/std": 0.9915971517562866,
|
|
"step": 2920,
|
|
"step_time": 11.766088206716812
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000163912773,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 345.9,
|
|
"completions/mean_length": 96.75125122070312,
|
|
"completions/mean_terminated_length": 70.08793716430664,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6659322917461394,
|
|
"epoch": 1.9162851536952257,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1256399154663086,
|
|
"kl": 0.08670818526297808,
|
|
"learning_rate": 2.1092217135382604e-08,
|
|
"loss": 0.09936992526054382,
|
|
"num_tokens": 55522567.0,
|
|
"reward": -4.485250073571478e-08,
|
|
"reward_std": 0.9913349151611328,
|
|
"rewards/rnd_reward/mean": -4.485249220920195e-08,
|
|
"rewards/rnd_reward/std": 0.9913349032402039,
|
|
"step": 2930,
|
|
"step_time": 11.034189373115078
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10750000178813934,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 319.1,
|
|
"completions/mean_length": 99.32625198364258,
|
|
"completions/mean_terminated_length": 65.10275955200196,
|
|
"completions/min_length": 2.1,
|
|
"completions/min_terminated_length": 2.1,
|
|
"entropy": 2.6324418902397158,
|
|
"epoch": 1.9228253760627863,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9896228313446045,
|
|
"kl": 0.08653121832758189,
|
|
"learning_rate": 1.9457161543492475e-08,
|
|
"loss": 0.0832364022731781,
|
|
"num_tokens": 55692093.0,
|
|
"reward": -1.2382865079985095e-07,
|
|
"reward_std": 0.9916486382484436,
|
|
"rewards/rnd_reward/mean": -1.238286500893082e-07,
|
|
"rewards/rnd_reward/std": 0.9916486382484436,
|
|
"step": 2940,
|
|
"step_time": 11.120865160599351
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08250000048428774,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 360.6,
|
|
"completions/mean_length": 100.11875228881836,
|
|
"completions/mean_terminated_length": 74.31617698669433,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.6869781494140623,
|
|
"epoch": 1.9293655984303466,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.051353931427002,
|
|
"kl": 0.0818368756212294,
|
|
"learning_rate": 1.7822105951602354e-08,
|
|
"loss": 0.11521185636520385,
|
|
"num_tokens": 55868653.0,
|
|
"reward": -9.492039936276342e-08,
|
|
"reward_std": 0.9913798630237579,
|
|
"rewards/rnd_reward/mean": -9.492039865222068e-08,
|
|
"rewards/rnd_reward/std": 0.9913798570632935,
|
|
"step": 2950,
|
|
"step_time": 11.270131665398367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08875000141561032,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 351.1,
|
|
"completions/mean_length": 97.49125289916992,
|
|
"completions/mean_terminated_length": 69.66969985961914,
|
|
"completions/min_length": 2.6,
|
|
"completions/min_terminated_length": 2.6,
|
|
"entropy": 2.753763663768768,
|
|
"epoch": 1.9359058207979072,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.8812336325645447,
|
|
"kl": 0.09095769878476859,
|
|
"learning_rate": 1.618705035971223e-08,
|
|
"loss": 0.043346887826919554,
|
|
"num_tokens": 56043276.0,
|
|
"reward": -6.645918113612481e-08,
|
|
"reward_std": 0.9914931774139404,
|
|
"rewards/rnd_reward/mean": -6.645917722813977e-08,
|
|
"rewards/rnd_reward/std": 0.991493159532547,
|
|
"step": 2960,
|
|
"step_time": 11.296948407590389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.07750000078231097,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 363.9,
|
|
"completions/mean_length": 89.08500251770019,
|
|
"completions/mean_terminated_length": 64.43022918701172,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.546912333369255,
|
|
"epoch": 1.9424460431654675,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2673183679580688,
|
|
"kl": 0.09239747198298573,
|
|
"learning_rate": 1.4551994767822105e-08,
|
|
"loss": 0.05778201222419739,
|
|
"num_tokens": 56208799.0,
|
|
"reward": -1.1771917272085374e-07,
|
|
"reward_std": 0.9914692223072052,
|
|
"rewards/rnd_reward/mean": -1.1771917272085374e-07,
|
|
"rewards/rnd_reward/std": 0.9914692461490631,
|
|
"step": 2970,
|
|
"step_time": 10.944724940578453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08625000119209289,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 344.3,
|
|
"completions/mean_length": 94.3025016784668,
|
|
"completions/mean_terminated_length": 67.02933235168457,
|
|
"completions/min_length": 2.2,
|
|
"completions/min_terminated_length": 2.2,
|
|
"entropy": 2.585839459300041,
|
|
"epoch": 1.9489862655330281,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1477481126785278,
|
|
"kl": 0.08655477333813906,
|
|
"learning_rate": 1.291693917593198e-08,
|
|
"loss": 0.04644548296928406,
|
|
"num_tokens": 56386666.0,
|
|
"reward": -1.7434358667856031e-07,
|
|
"reward_std": 0.9915457606315613,
|
|
"rewards/rnd_reward/mean": -1.7434358596801757e-07,
|
|
"rewards/rnd_reward/std": 0.9915457487106323,
|
|
"step": 2980,
|
|
"step_time": 12.27661042441614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08500000275671482,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 332.0,
|
|
"completions/mean_length": 92.81125259399414,
|
|
"completions/mean_terminated_length": 65.69409408569337,
|
|
"completions/min_length": 2.5,
|
|
"completions/min_terminated_length": 2.5,
|
|
"entropy": 2.7046052366495132,
|
|
"epoch": 1.9555264879005887,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1466176509857178,
|
|
"kl": 0.08695422541350126,
|
|
"learning_rate": 1.1281883584041857e-08,
|
|
"loss": 0.10825624465942382,
|
|
"num_tokens": 56555065.0,
|
|
"reward": -2.5480985499370944e-08,
|
|
"reward_std": 0.9915042757987976,
|
|
"rewards/rnd_reward/mean": -2.5480984078285475e-08,
|
|
"rewards/rnd_reward/std": 0.9915042877197265,
|
|
"step": 2990,
|
|
"step_time": 11.024760208884254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09250000044703484,
|
|
"completions/max_length": 384.0,
|
|
"completions/max_terminated_length": 358.3,
|
|
"completions/mean_length": 92.64375152587891,
|
|
"completions/mean_terminated_length": 62.92685241699219,
|
|
"completions/min_length": 2.4,
|
|
"completions/min_terminated_length": 2.4,
|
|
"entropy": 2.619092810153961,
|
|
"epoch": 1.9620667102681493,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9762617349624634,
|
|
"kl": 0.09417410604655743,
|
|
"learning_rate": 9.646827992151732e-09,
|
|
"loss": 0.11771528720855713,
|
|
"num_tokens": 56719745.0,
|
|
"reward": -4.336237964253087e-08,
|
|
"reward_std": 0.9909454762935639,
|
|
"rewards/rnd_reward/mean": -4.336238106361634e-08,
|
|
"rewards/rnd_reward/std": 0.9909454941749573,
|
|
"step": 3000,
|
|
"step_time": 10.862733572837897
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 3058,
|
|
"num_input_tokens_seen": 56719745,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 1000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 20,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|