10295 lines
358 KiB
JSON
10295 lines
358 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.9587628865979383,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 1900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.8,
|
||
|
|
"completions/max_terminated_length": 24.8,
|
||
|
|
"completions/mean_length": 15.05,
|
||
|
|
"completions/mean_terminated_length": 15.05,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 1.0464817345142365,
|
||
|
|
"epoch": 0.005154639175257732,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.0309278350515463e-08,
|
||
|
|
"loss": 0.14594812393188478,
|
||
|
|
"num_tokens": 1329.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 5,
|
||
|
|
"step_time": 0.7463470116257668
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.6,
|
||
|
|
"completions/max_terminated_length": 25.6,
|
||
|
|
"completions/mean_length": 14.95,
|
||
|
|
"completions/mean_terminated_length": 14.95,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.8166205495595932,
|
||
|
|
"epoch": 0.010309278350515464,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.3195876288659794e-08,
|
||
|
|
"loss": 0.0769703209400177,
|
||
|
|
"num_tokens": 2696.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 10,
|
||
|
|
"step_time": 0.6073689106851816
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.4,
|
||
|
|
"completions/max_terminated_length": 15.4,
|
||
|
|
"completions/mean_length": 12.2,
|
||
|
|
"completions/mean_terminated_length": 12.2,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.9185687482357026,
|
||
|
|
"epoch": 0.015463917525773196,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 13.9375,
|
||
|
|
"learning_rate": 3.608247422680412e-08,
|
||
|
|
"loss": -0.0014619171619415284,
|
||
|
|
"num_tokens": 3944.0,
|
||
|
|
"reward": -0.6100000023841858,
|
||
|
|
"reward_std": 0.15754843950271608,
|
||
|
|
"rewards/reward_fn/mean": -0.6100000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.15754843950271608,
|
||
|
|
"step": 15,
|
||
|
|
"step_time": 0.449109182972461
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.4,
|
||
|
|
"completions/max_terminated_length": 15.4,
|
||
|
|
"completions/mean_length": 12.0,
|
||
|
|
"completions/mean_terminated_length": 12.0,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.6480183869600296,
|
||
|
|
"epoch": 0.020618556701030927,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 13.875,
|
||
|
|
"learning_rate": 4.896907216494845e-08,
|
||
|
|
"loss": 0.11619714498519898,
|
||
|
|
"num_tokens": 5188.0,
|
||
|
|
"reward": -0.6175000011920929,
|
||
|
|
"reward_std": 0.36337884664535525,
|
||
|
|
"rewards/reward_fn/mean": -0.6175000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.36337885856628416,
|
||
|
|
"step": 20,
|
||
|
|
"step_time": 0.452333062607795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.2,
|
||
|
|
"completions/max_terminated_length": 15.2,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.9010713428258896,
|
||
|
|
"epoch": 0.02577319587628866,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 34.75,
|
||
|
|
"learning_rate": 6.185567010309278e-08,
|
||
|
|
"loss": 0.044125625491142274,
|
||
|
|
"num_tokens": 6388.0,
|
||
|
|
"reward": -0.2300000011920929,
|
||
|
|
"reward_std": 0.43011106848716735,
|
||
|
|
"rewards/reward_fn/mean": -0.2300000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.43011106848716735,
|
||
|
|
"step": 25,
|
||
|
|
"step_time": 0.44755604499951007
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.4,
|
||
|
|
"completions/max_terminated_length": 11.4,
|
||
|
|
"completions/mean_length": 9.0,
|
||
|
|
"completions/mean_terminated_length": 9.0,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.8676799893379211,
|
||
|
|
"epoch": 0.030927835051546393,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.75,
|
||
|
|
"learning_rate": 7.474226804123711e-08,
|
||
|
|
"loss": 0.1554401397705078,
|
||
|
|
"num_tokens": 7568.0,
|
||
|
|
"reward": -0.602999997138977,
|
||
|
|
"reward_std": 0.2935142517089844,
|
||
|
|
"rewards/reward_fn/mean": -0.602999997138977,
|
||
|
|
"rewards/reward_fn/std": 0.29351426362991334,
|
||
|
|
"step": 30,
|
||
|
|
"step_time": 0.39083454087376596
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.4,
|
||
|
|
"completions/max_terminated_length": 27.4,
|
||
|
|
"completions/mean_length": 14.8,
|
||
|
|
"completions/mean_terminated_length": 14.8,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.8659799456596374,
|
||
|
|
"epoch": 0.03608247422680412,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 24.5,
|
||
|
|
"learning_rate": 8.762886597938143e-08,
|
||
|
|
"loss": 0.1145447850227356,
|
||
|
|
"num_tokens": 8864.0,
|
||
|
|
"reward": -0.75,
|
||
|
|
"reward_std": 0.2154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.75,
|
||
|
|
"rewards/reward_fn/std": 0.2154700517654419,
|
||
|
|
"step": 35,
|
||
|
|
"step_time": 0.7107938446104527
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.2,
|
||
|
|
"completions/max_terminated_length": 19.2,
|
||
|
|
"completions/mean_length": 13.3,
|
||
|
|
"completions/mean_terminated_length": 13.3,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.8932873725891113,
|
||
|
|
"epoch": 0.041237113402061855,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 22.125,
|
||
|
|
"learning_rate": 1.0051546391752578e-07,
|
||
|
|
"loss": 0.11950116157531739,
|
||
|
|
"num_tokens": 10174.0,
|
||
|
|
"reward": -0.24749999344348908,
|
||
|
|
"reward_std": 0.3299496531486511,
|
||
|
|
"rewards/reward_fn/mean": -0.24749999344348908,
|
||
|
|
"rewards/reward_fn/std": 0.32994966506958007,
|
||
|
|
"step": 40,
|
||
|
|
"step_time": 0.5169275973923504
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 11.5,
|
||
|
|
"completions/mean_terminated_length": 11.5,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.8041223675012589,
|
||
|
|
"epoch": 0.04639175257731959,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.5,
|
||
|
|
"learning_rate": 1.134020618556701e-07,
|
||
|
|
"loss": 0.1752654194831848,
|
||
|
|
"num_tokens": 11420.0,
|
||
|
|
"reward": -0.5862500011920929,
|
||
|
|
"reward_std": 0.37825352549552915,
|
||
|
|
"rewards/reward_fn/mean": -0.5862500011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3782535195350647,
|
||
|
|
"step": 45,
|
||
|
|
"step_time": 0.45717077497392894
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.4,
|
||
|
|
"completions/max_terminated_length": 15.4,
|
||
|
|
"completions/mean_length": 9.8,
|
||
|
|
"completions/mean_terminated_length": 9.8,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.6957661122083664,
|
||
|
|
"epoch": 0.05154639175257732,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.2628865979381443e-07,
|
||
|
|
"loss": 0.1835160255432129,
|
||
|
|
"num_tokens": 12616.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.2809401035308838,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.2809401035308838,
|
||
|
|
"step": 50,
|
||
|
|
"step_time": 0.45142662366852165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.6,
|
||
|
|
"completions/max_terminated_length": 23.6,
|
||
|
|
"completions/mean_length": 11.7,
|
||
|
|
"completions/mean_terminated_length": 11.7,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.9917734563350677,
|
||
|
|
"epoch": 0.05670103092783505,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.3917525773195876e-07,
|
||
|
|
"loss": 0.14120221138000488,
|
||
|
|
"num_tokens": 13862.0,
|
||
|
|
"reward": -0.43666666746139526,
|
||
|
|
"reward_std": 0.3740913510322571,
|
||
|
|
"rewards/reward_fn/mean": -0.43666666746139526,
|
||
|
|
"rewards/reward_fn/std": 0.3740913510322571,
|
||
|
|
"step": 55,
|
||
|
|
"step_time": 0.6059751262888312
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.4,
|
||
|
|
"completions/max_terminated_length": 24.4,
|
||
|
|
"completions/mean_length": 17.15,
|
||
|
|
"completions/mean_terminated_length": 17.15,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.8903216406702995,
|
||
|
|
"epoch": 0.061855670103092786,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 5.6875,
|
||
|
|
"learning_rate": 1.5206185567010307e-07,
|
||
|
|
"loss": 0.07498554587364196,
|
||
|
|
"num_tokens": 15237.0,
|
||
|
|
"reward": -0.6100000023841858,
|
||
|
|
"reward_std": 0.2532050788402557,
|
||
|
|
"rewards/reward_fn/mean": -0.6100000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2532050788402557,
|
||
|
|
"step": 60,
|
||
|
|
"step_time": 0.6272518557496369
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 34.4,
|
||
|
|
"completions/max_terminated_length": 34.4,
|
||
|
|
"completions/mean_length": 16.65,
|
||
|
|
"completions/mean_terminated_length": 16.65,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.894698689877987,
|
||
|
|
"epoch": 0.06701030927835051,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.0,
|
||
|
|
"learning_rate": 1.649484536082474e-07,
|
||
|
|
"loss": 0.3684789419174194,
|
||
|
|
"num_tokens": 16570.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 65,
|
||
|
|
"step_time": 0.7892749737016856
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.2,
|
||
|
|
"completions/max_terminated_length": 12.2,
|
||
|
|
"completions/mean_length": 9.45,
|
||
|
|
"completions/mean_terminated_length": 9.45,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9464951276779174,
|
||
|
|
"epoch": 0.07216494845360824,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 15.3125,
|
||
|
|
"learning_rate": 1.7783505154639176e-07,
|
||
|
|
"loss": 0.05676301717758179,
|
||
|
|
"num_tokens": 17791.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 70,
|
||
|
|
"step_time": 0.45811321027576923
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.4,
|
||
|
|
"completions/max_terminated_length": 27.4,
|
||
|
|
"completions/mean_length": 17.65,
|
||
|
|
"completions/mean_terminated_length": 17.65,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 1.323281478881836,
|
||
|
|
"epoch": 0.07731958762886598,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.9072164948453607e-07,
|
||
|
|
"loss": 0.08197730779647827,
|
||
|
|
"num_tokens": 19168.0,
|
||
|
|
"reward": -0.4675000011920929,
|
||
|
|
"reward_std": 0.16499999761581421,
|
||
|
|
"rewards/reward_fn/mean": -0.4675000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.16499999761581421,
|
||
|
|
"step": 75,
|
||
|
|
"step_time": 0.6762679857201874
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.8,
|
||
|
|
"completions/max_terminated_length": 12.8,
|
||
|
|
"completions/mean_length": 9.3,
|
||
|
|
"completions/mean_terminated_length": 9.3,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.6954251408576966,
|
||
|
|
"epoch": 0.08247422680412371,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.375,
|
||
|
|
"learning_rate": 2.036082474226804e-07,
|
||
|
|
"loss": 0.1398395538330078,
|
||
|
|
"num_tokens": 20382.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.2,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.2,
|
||
|
|
"step": 80,
|
||
|
|
"step_time": 0.5250002114102245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.2,
|
||
|
|
"completions/max_terminated_length": 20.2,
|
||
|
|
"completions/mean_length": 13.85,
|
||
|
|
"completions/mean_terminated_length": 13.85,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 1.0841705203056335,
|
||
|
|
"epoch": 0.08762886597938144,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 23.5,
|
||
|
|
"learning_rate": 2.1649484536082473e-07,
|
||
|
|
"loss": 0.06400776505470276,
|
||
|
|
"num_tokens": 21667.0,
|
||
|
|
"reward": -0.5949999988079071,
|
||
|
|
"reward_std": 0.3001110672950745,
|
||
|
|
"rewards/reward_fn/mean": -0.5949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.3001110672950745,
|
||
|
|
"step": 85,
|
||
|
|
"step_time": 0.5649234197102487
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 51.2,
|
||
|
|
"completions/max_terminated_length": 51.2,
|
||
|
|
"completions/mean_length": 20.95,
|
||
|
|
"completions/mean_terminated_length": 20.95,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.9970528572797775,
|
||
|
|
"epoch": 0.09278350515463918,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.4375,
|
||
|
|
"learning_rate": 2.2938144329896904e-07,
|
||
|
|
"loss": 0.30556085109710696,
|
||
|
|
"num_tokens": 23110.0,
|
||
|
|
"reward": -0.5733333349227905,
|
||
|
|
"reward_std": 0.2945683240890503,
|
||
|
|
"rewards/reward_fn/mean": -0.5733333349227905,
|
||
|
|
"rewards/reward_fn/std": 0.2945683240890503,
|
||
|
|
"step": 90,
|
||
|
|
"step_time": 1.0468046895228327
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.0,
|
||
|
|
"completions/max_terminated_length": 24.0,
|
||
|
|
"completions/mean_length": 10.55,
|
||
|
|
"completions/mean_terminated_length": 10.55,
|
||
|
|
"completions/min_length": 5.0,
|
||
|
|
"completions/min_terminated_length": 5.0,
|
||
|
|
"entropy": 0.6387317299842834,
|
||
|
|
"epoch": 0.0979381443298969,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.4226804123711337e-07,
|
||
|
|
"loss": 0.18993494510650635,
|
||
|
|
"num_tokens": 24341.0,
|
||
|
|
"reward": -0.3,
|
||
|
|
"reward_std": 0.1154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.3,
|
||
|
|
"rewards/reward_fn/std": 0.1154700517654419,
|
||
|
|
"step": 95,
|
||
|
|
"step_time": 0.6111652076244354
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 101.8,
|
||
|
|
"completions/max_terminated_length": 55.8,
|
||
|
|
"completions/mean_length": 37.95,
|
||
|
|
"completions/mean_terminated_length": 26.15,
|
||
|
|
"completions/min_length": 12.4,
|
||
|
|
"completions/min_terminated_length": 12.4,
|
||
|
|
"entropy": 1.1228386163711548,
|
||
|
|
"epoch": 0.10309278350515463,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.6875,
|
||
|
|
"learning_rate": 2.551546391752577e-07,
|
||
|
|
"loss": 0.16065646409988404,
|
||
|
|
"num_tokens": 26128.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 100,
|
||
|
|
"step_time": 1.852026186324656
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.0,
|
||
|
|
"completions/max_terminated_length": 27.0,
|
||
|
|
"completions/mean_length": 16.9,
|
||
|
|
"completions/mean_terminated_length": 16.9,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.9643263280391693,
|
||
|
|
"epoch": 0.10824742268041238,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.6804123711340204e-07,
|
||
|
|
"loss": 0.07265600562095642,
|
||
|
|
"num_tokens": 27494.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 105,
|
||
|
|
"step_time": 0.6847881611436606
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.2,
|
||
|
|
"completions/max_terminated_length": 19.2,
|
||
|
|
"completions/mean_length": 13.1,
|
||
|
|
"completions/mean_terminated_length": 13.1,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.5918209165334701,
|
||
|
|
"epoch": 0.1134020618556701,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 8.75,
|
||
|
|
"learning_rate": 2.809278350515464e-07,
|
||
|
|
"loss": 0.14053068161010743,
|
||
|
|
"num_tokens": 28792.0,
|
||
|
|
"reward": -0.725,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.725,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 110,
|
||
|
|
"step_time": 0.5405681782402099
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.8,
|
||
|
|
"completions/max_terminated_length": 20.8,
|
||
|
|
"completions/mean_length": 12.4,
|
||
|
|
"completions/mean_terminated_length": 12.4,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0056458711624146,
|
||
|
|
"epoch": 0.11855670103092783,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.625,
|
||
|
|
"learning_rate": 2.938144329896907e-07,
|
||
|
|
"loss": 0.23783767223358154,
|
||
|
|
"num_tokens": 30064.0,
|
||
|
|
"reward": -0.4850000023841858,
|
||
|
|
"reward_std": 0.4591261982917786,
|
||
|
|
"rewards/reward_fn/mean": -0.4850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.4591261982917786,
|
||
|
|
"step": 115,
|
||
|
|
"step_time": 0.5577491442672908
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.4,
|
||
|
|
"completions/max_terminated_length": 13.4,
|
||
|
|
"completions/mean_length": 10.5,
|
||
|
|
"completions/mean_terminated_length": 10.5,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.6822600662708282,
|
||
|
|
"epoch": 0.12371134020618557,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 12.75,
|
||
|
|
"learning_rate": 3.0670103092783506e-07,
|
||
|
|
"loss": 0.11178064346313477,
|
||
|
|
"num_tokens": 31290.0,
|
||
|
|
"reward": -0.3266666650772095,
|
||
|
|
"reward_std": 0.4108335912227631,
|
||
|
|
"rewards/reward_fn/mean": -0.3266666650772095,
|
||
|
|
"rewards/reward_fn/std": 0.4108336091041565,
|
||
|
|
"step": 120,
|
||
|
|
"step_time": 0.5232813514769077
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.6,
|
||
|
|
"completions/max_terminated_length": 23.6,
|
||
|
|
"completions/mean_length": 14.2,
|
||
|
|
"completions/mean_terminated_length": 14.2,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.8330166220664978,
|
||
|
|
"epoch": 0.12886597938144329,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.5,
|
||
|
|
"learning_rate": 3.195876288659793e-07,
|
||
|
|
"loss": 0.236272931098938,
|
||
|
|
"num_tokens": 32594.0,
|
||
|
|
"reward": -0.725,
|
||
|
|
"reward_std": 0.25,
|
||
|
|
"rewards/reward_fn/mean": -0.725,
|
||
|
|
"rewards/reward_fn/std": 0.25,
|
||
|
|
"step": 125,
|
||
|
|
"step_time": 0.5966341249644757
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 13.4,
|
||
|
|
"completions/mean_terminated_length": 13.4,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 1.0293640315532684,
|
||
|
|
"epoch": 0.13402061855670103,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.5625,
|
||
|
|
"learning_rate": 3.324742268041237e-07,
|
||
|
|
"loss": 0.03524231314659119,
|
||
|
|
"num_tokens": 33878.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.2,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.2,
|
||
|
|
"step": 130,
|
||
|
|
"step_time": 0.5198140442371368
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 12.4,
|
||
|
|
"completions/mean_terminated_length": 12.4,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.8266386479139328,
|
||
|
|
"epoch": 0.13917525773195877,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 15.625,
|
||
|
|
"learning_rate": 3.4536082474226804e-07,
|
||
|
|
"loss": 0.019967517256736754,
|
||
|
|
"num_tokens": 35158.0,
|
||
|
|
"reward": -0.6100000023841858,
|
||
|
|
"reward_std": 0.315283465385437,
|
||
|
|
"rewards/reward_fn/mean": -0.6100000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.315283465385437,
|
||
|
|
"step": 135,
|
||
|
|
"step_time": 0.5171223360113799
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.6,
|
||
|
|
"completions/max_terminated_length": 19.6,
|
||
|
|
"completions/mean_length": 14.5,
|
||
|
|
"completions/mean_terminated_length": 14.5,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.8061805367469788,
|
||
|
|
"epoch": 0.14432989690721648,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 13.5625,
|
||
|
|
"learning_rate": 3.5824742268041234e-07,
|
||
|
|
"loss": 0.05468950271606445,
|
||
|
|
"num_tokens": 36480.0,
|
||
|
|
"reward": -0.4533333331346512,
|
||
|
|
"reward_std": 0.3666622221469879,
|
||
|
|
"rewards/reward_fn/mean": -0.4533333331346512,
|
||
|
|
"rewards/reward_fn/std": 0.36666223406791687,
|
||
|
|
"step": 140,
|
||
|
|
"step_time": 0.5422550996765494
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 10.05,
|
||
|
|
"completions/mean_terminated_length": 10.05,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.6229674249887467,
|
||
|
|
"epoch": 0.14948453608247422,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 20.5,
|
||
|
|
"learning_rate": 3.711340206185567e-07,
|
||
|
|
"loss": 0.09494972825050355,
|
||
|
|
"num_tokens": 37697.0,
|
||
|
|
"reward": -0.5850000023841858,
|
||
|
|
"reward_std": 0.23048137426376342,
|
||
|
|
"rewards/reward_fn/mean": -0.5850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.23048137426376342,
|
||
|
|
"step": 145,
|
||
|
|
"step_time": 0.4210976084694266
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.4,
|
||
|
|
"completions/max_terminated_length": 17.4,
|
||
|
|
"completions/mean_length": 11.35,
|
||
|
|
"completions/mean_terminated_length": 11.35,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.8502504467964173,
|
||
|
|
"epoch": 0.15463917525773196,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.0,
|
||
|
|
"learning_rate": 3.8402061855670106e-07,
|
||
|
|
"loss": 0.16331512928009034,
|
||
|
|
"num_tokens": 38932.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.25,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.25,
|
||
|
|
"step": 150,
|
||
|
|
"step_time": 0.5016824883408845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.8,
|
||
|
|
"completions/max_terminated_length": 17.8,
|
||
|
|
"completions/mean_length": 11.7,
|
||
|
|
"completions/mean_terminated_length": 11.7,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8068040907382965,
|
||
|
|
"epoch": 0.15979381443298968,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.969072164948453e-07,
|
||
|
|
"loss": 0.046058404445648196,
|
||
|
|
"num_tokens": 40186.0,
|
||
|
|
"reward": -0.5083333313465118,
|
||
|
|
"reward_std": 0.22301849126815795,
|
||
|
|
"rewards/reward_fn/mean": -0.5083333313465118,
|
||
|
|
"rewards/reward_fn/std": 0.22301849126815795,
|
||
|
|
"step": 155,
|
||
|
|
"step_time": 0.6020563900470733
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.2,
|
||
|
|
"completions/max_terminated_length": 25.2,
|
||
|
|
"completions/mean_length": 13.3,
|
||
|
|
"completions/mean_terminated_length": 13.3,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.9016699820756913,
|
||
|
|
"epoch": 0.16494845360824742,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.1875,
|
||
|
|
"learning_rate": 4.097938144329897e-07,
|
||
|
|
"loss": 0.1976318597793579,
|
||
|
|
"num_tokens": 41464.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.20773502588272094,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.20773502588272094,
|
||
|
|
"step": 160,
|
||
|
|
"step_time": 0.6755657859146595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.4,
|
||
|
|
"completions/max_terminated_length": 15.4,
|
||
|
|
"completions/mean_length": 11.25,
|
||
|
|
"completions/mean_terminated_length": 11.25,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.7308270364999772,
|
||
|
|
"epoch": 0.17010309278350516,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.4375,
|
||
|
|
"learning_rate": 4.22680412371134e-07,
|
||
|
|
"loss": 0.10954289436340332,
|
||
|
|
"num_tokens": 42701.0,
|
||
|
|
"reward": -0.725,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.725,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 165,
|
||
|
|
"step_time": 0.49519333997741344
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.8,
|
||
|
|
"completions/max_terminated_length": 21.8,
|
||
|
|
"completions/mean_length": 12.55,
|
||
|
|
"completions/mean_terminated_length": 12.55,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.746524891257286,
|
||
|
|
"epoch": 0.17525773195876287,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 11.125,
|
||
|
|
"learning_rate": 4.3556701030927834e-07,
|
||
|
|
"loss": 0.14766517877578736,
|
||
|
|
"num_tokens": 43944.0,
|
||
|
|
"reward": -0.75,
|
||
|
|
"reward_std": 0.25773502588272096,
|
||
|
|
"rewards/reward_fn/mean": -0.75,
|
||
|
|
"rewards/reward_fn/std": 0.25773502588272096,
|
||
|
|
"step": 170,
|
||
|
|
"step_time": 0.6326937489211559
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.2,
|
||
|
|
"completions/max_terminated_length": 27.2,
|
||
|
|
"completions/mean_length": 16.4,
|
||
|
|
"completions/mean_terminated_length": 16.4,
|
||
|
|
"completions/min_length": 10.2,
|
||
|
|
"completions/min_terminated_length": 10.2,
|
||
|
|
"entropy": 0.5598460659384727,
|
||
|
|
"epoch": 0.18041237113402062,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 7.34375,
|
||
|
|
"learning_rate": 4.484536082474227e-07,
|
||
|
|
"loss": 0.05735980272293091,
|
||
|
|
"num_tokens": 45308.0,
|
||
|
|
"reward": -0.7,
|
||
|
|
"reward_std": 0.2732050776481628,
|
||
|
|
"rewards/reward_fn/mean": -0.7,
|
||
|
|
"rewards/reward_fn/std": 0.2732050776481628,
|
||
|
|
"step": 175,
|
||
|
|
"step_time": 0.7125312600284814
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.0,
|
||
|
|
"completions/max_terminated_length": 11.0,
|
||
|
|
"completions/mean_length": 8.25,
|
||
|
|
"completions/mean_terminated_length": 8.25,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.5958581089973449,
|
||
|
|
"epoch": 0.18556701030927836,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.625,
|
||
|
|
"learning_rate": 4.61340206185567e-07,
|
||
|
|
"loss": 0.11873382329940796,
|
||
|
|
"num_tokens": 46469.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 180,
|
||
|
|
"step_time": 0.4346679733134806
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.6,
|
||
|
|
"completions/max_terminated_length": 23.6,
|
||
|
|
"completions/mean_length": 13.75,
|
||
|
|
"completions/mean_terminated_length": 13.75,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.8451726168394089,
|
||
|
|
"epoch": 0.19072164948453607,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 4.742268041237113e-07,
|
||
|
|
"loss": 0.058754390478134154,
|
||
|
|
"num_tokens": 47752.0,
|
||
|
|
"reward": -0.5240000009536743,
|
||
|
|
"reward_std": 0.387283456325531,
|
||
|
|
"rewards/reward_fn/mean": -0.5240000009536743,
|
||
|
|
"rewards/reward_fn/std": 0.3872834801673889,
|
||
|
|
"step": 185,
|
||
|
|
"step_time": 0.6394024872221052
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.6,
|
||
|
|
"completions/max_terminated_length": 20.6,
|
||
|
|
"completions/mean_length": 13.35,
|
||
|
|
"completions/mean_terminated_length": 13.35,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8434844225645065,
|
||
|
|
"epoch": 0.1958762886597938,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 34.75,
|
||
|
|
"learning_rate": 4.871134020618556e-07,
|
||
|
|
"loss": 0.23021700382232665,
|
||
|
|
"num_tokens": 49043.0,
|
||
|
|
"reward": -0.5949999988079071,
|
||
|
|
"reward_std": 0.30011107325553893,
|
||
|
|
"rewards/reward_fn/mean": -0.5949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.30011107325553893,
|
||
|
|
"step": 190,
|
||
|
|
"step_time": 0.5879216003231704
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.2,
|
||
|
|
"completions/max_terminated_length": 14.2,
|
||
|
|
"completions/mean_length": 11.25,
|
||
|
|
"completions/mean_terminated_length": 11.25,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.900922417640686,
|
||
|
|
"epoch": 0.20103092783505155,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 5e-07,
|
||
|
|
"loss": 0.017963612079620363,
|
||
|
|
"num_tokens": 50296.0,
|
||
|
|
"reward": -0.4800000011920929,
|
||
|
|
"reward_std": 0.2878461003303528,
|
||
|
|
"rewards/reward_fn/mean": -0.4800000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2878461003303528,
|
||
|
|
"step": 195,
|
||
|
|
"step_time": 0.47086463402956724
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 32.8,
|
||
|
|
"completions/max_terminated_length": 32.8,
|
||
|
|
"completions/mean_length": 15.8,
|
||
|
|
"completions/mean_terminated_length": 15.8,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.8438230156898499,
|
||
|
|
"epoch": 0.20618556701030927,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.999898828460382e-07,
|
||
|
|
"loss": 0.05334873795509339,
|
||
|
|
"num_tokens": 51648.0,
|
||
|
|
"reward": -0.4050000011920929,
|
||
|
|
"reward_std": 0.43887237906455995,
|
||
|
|
"rewards/reward_fn/mean": -0.4050000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.43887237906455995,
|
||
|
|
"step": 200,
|
||
|
|
"step_time": 0.820699253398925
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.4,
|
||
|
|
"completions/max_terminated_length": 20.4,
|
||
|
|
"completions/mean_length": 13.0,
|
||
|
|
"completions/mean_terminated_length": 13.0,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.703792467713356,
|
||
|
|
"epoch": 0.211340206185567,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.75,
|
||
|
|
"learning_rate": 4.999595322030074e-07,
|
||
|
|
"loss": 0.05575525760650635,
|
||
|
|
"num_tokens": 52944.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 205,
|
||
|
|
"step_time": 0.599746462609619
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 12.1,
|
||
|
|
"completions/mean_terminated_length": 12.1,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.696284830570221,
|
||
|
|
"epoch": 0.21649484536082475,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.8125,
|
||
|
|
"learning_rate": 4.999089505274044e-07,
|
||
|
|
"loss": 0.17524465322494506,
|
||
|
|
"num_tokens": 54234.0,
|
||
|
|
"reward": -0.55,
|
||
|
|
"reward_std": 0.5196152329444885,
|
||
|
|
"rewards/reward_fn/mean": -0.55,
|
||
|
|
"rewards/reward_fn/std": 0.5196152329444885,
|
||
|
|
"step": 210,
|
||
|
|
"step_time": 0.5528348212130367
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 15.9,
|
||
|
|
"completions/mean_terminated_length": 15.9,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.6460077106952667,
|
||
|
|
"epoch": 0.22164948453608246,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.125,
|
||
|
|
"learning_rate": 4.998381419131701e-07,
|
||
|
|
"loss": -0.008495497703552245,
|
||
|
|
"num_tokens": 55584.0,
|
||
|
|
"reward": -0.7425000011920929,
|
||
|
|
"reward_std": 0.3133788466453552,
|
||
|
|
"rewards/reward_fn/mean": -0.7425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.31337885856628417,
|
||
|
|
"step": 215,
|
||
|
|
"step_time": 0.5730495001189411
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 59.4,
|
||
|
|
"completions/max_terminated_length": 59.4,
|
||
|
|
"completions/mean_length": 24.85,
|
||
|
|
"completions/mean_terminated_length": 24.85,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.6805280774831772,
|
||
|
|
"epoch": 0.2268041237113402,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 9.6875,
|
||
|
|
"learning_rate": 4.997471120913578e-07,
|
||
|
|
"loss": 0.09315776824951172,
|
||
|
|
"num_tokens": 57113.0,
|
||
|
|
"reward": -0.5025000005960465,
|
||
|
|
"reward_std": 0.29997222423553466,
|
||
|
|
"rewards/reward_fn/mean": -0.5025000005960465,
|
||
|
|
"rewards/reward_fn/std": 0.2999722123146057,
|
||
|
|
"step": 220,
|
||
|
|
"step_time": 1.246410696581006
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.0,
|
||
|
|
"completions/max_terminated_length": 21.0,
|
||
|
|
"completions/mean_length": 13.35,
|
||
|
|
"completions/mean_terminated_length": 13.35,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.5927552968263626,
|
||
|
|
"epoch": 0.23195876288659795,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 14.9375,
|
||
|
|
"learning_rate": 4.996358684296693e-07,
|
||
|
|
"loss": 0.1518040895462036,
|
||
|
|
"num_tokens": 58416.0,
|
||
|
|
"reward": -0.23333333134651185,
|
||
|
|
"reward_std": 0.5039585947990417,
|
||
|
|
"rewards/reward_fn/mean": -0.23333333134651185,
|
||
|
|
"rewards/reward_fn/std": 0.5039585947990417,
|
||
|
|
"step": 225,
|
||
|
|
"step_time": 0.6047702097333968
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.4,
|
||
|
|
"completions/max_terminated_length": 13.4,
|
||
|
|
"completions/mean_length": 8.75,
|
||
|
|
"completions/mean_terminated_length": 8.75,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.7510967075824737,
|
||
|
|
"epoch": 0.23711340206185566,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 4.995044199318583e-07,
|
||
|
|
"loss": 0.1716856002807617,
|
||
|
|
"num_tokens": 59591.0,
|
||
|
|
"reward": -0.40666666328907014,
|
||
|
|
"reward_std": 0.3867302894592285,
|
||
|
|
"rewards/reward_fn/mean": -0.40666666328907014,
|
||
|
|
"rewards/reward_fn/std": 0.386730295419693,
|
||
|
|
"step": 230,
|
||
|
|
"step_time": 0.4744772708974779
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.0,
|
||
|
|
"completions/max_terminated_length": 10.0,
|
||
|
|
"completions/mean_length": 7.55,
|
||
|
|
"completions/mean_terminated_length": 7.55,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.6909678518772125,
|
||
|
|
"epoch": 0.2422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 40.75,
|
||
|
|
"learning_rate": 4.993527772370027e-07,
|
||
|
|
"loss": 0.10667741298675537,
|
||
|
|
"num_tokens": 60754.0,
|
||
|
|
"reward": -0.4350000023841858,
|
||
|
|
"reward_std": 0.23279056549072266,
|
||
|
|
"rewards/reward_fn/mean": -0.4350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.23279056549072266,
|
||
|
|
"step": 235,
|
||
|
|
"step_time": 0.40216352241113784
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.4,
|
||
|
|
"completions/max_terminated_length": 16.4,
|
||
|
|
"completions/mean_length": 10.7,
|
||
|
|
"completions/mean_terminated_length": 10.7,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.7458656638860702,
|
||
|
|
"epoch": 0.24742268041237114,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.125,
|
||
|
|
"learning_rate": 4.991809526186423e-07,
|
||
|
|
"loss": -0.0752179503440857,
|
||
|
|
"num_tokens": 62004.0,
|
||
|
|
"reward": -0.40208333283662795,
|
||
|
|
"reward_std": 0.4155647039413452,
|
||
|
|
"rewards/reward_fn/mean": -0.40208333283662795,
|
||
|
|
"rewards/reward_fn/std": 0.4155647039413452,
|
||
|
|
"step": 240,
|
||
|
|
"step_time": 0.5043572745285928
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 9.65,
|
||
|
|
"completions/mean_terminated_length": 9.65,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.7298850357532501,
|
||
|
|
"epoch": 0.25257731958762886,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.989889599837861e-07,
|
||
|
|
"loss": 0.137645161151886,
|
||
|
|
"num_tokens": 63209.0,
|
||
|
|
"reward": -0.5,
|
||
|
|
"reward_std": 0.25773502588272096,
|
||
|
|
"rewards/reward_fn/mean": -0.5,
|
||
|
|
"rewards/reward_fn/std": 0.25773502588272096,
|
||
|
|
"step": 245,
|
||
|
|
"step_time": 0.5419826946221292
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.6,
|
||
|
|
"completions/max_terminated_length": 13.6,
|
||
|
|
"completions/mean_length": 8.1,
|
||
|
|
"completions/mean_terminated_length": 8.1,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 0.6799002349376678,
|
||
|
|
"epoch": 0.25773195876288657,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.375,
|
||
|
|
"learning_rate": 4.987768148717863e-07,
|
||
|
|
"loss": 0.14280350208282472,
|
||
|
|
"num_tokens": 64371.0,
|
||
|
|
"reward": -0.4350000023841858,
|
||
|
|
"reward_std": 0.5459514260292053,
|
||
|
|
"rewards/reward_fn/mean": -0.4350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.5459514260292053,
|
||
|
|
"step": 250,
|
||
|
|
"step_time": 0.46002390459179876
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.4,
|
||
|
|
"completions/max_terminated_length": 17.4,
|
||
|
|
"completions/mean_length": 12.05,
|
||
|
|
"completions/mean_terminated_length": 12.05,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.6803009271621704,
|
||
|
|
"epoch": 0.26288659793814434,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.98544534453081e-07,
|
||
|
|
"loss": 0.14994003772735595,
|
||
|
|
"num_tokens": 65652.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 255,
|
||
|
|
"step_time": 0.5426725285127759
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.8,
|
||
|
|
"completions/max_terminated_length": 14.8,
|
||
|
|
"completions/mean_length": 13.1,
|
||
|
|
"completions/mean_terminated_length": 13.1,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.6927774518728256,
|
||
|
|
"epoch": 0.26804123711340205,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.982921375278046e-07,
|
||
|
|
"loss": 0.05132720470428467,
|
||
|
|
"num_tokens": 66962.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.30773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.30773502588272095,
|
||
|
|
"step": 260,
|
||
|
|
"step_time": 0.49349935110658405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.4,
|
||
|
|
"completions/max_terminated_length": 24.4,
|
||
|
|
"completions/mean_length": 14.55,
|
||
|
|
"completions/mean_terminated_length": 14.55,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.6729653716087342,
|
||
|
|
"epoch": 0.27319587628865977,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 11.5625,
|
||
|
|
"learning_rate": 4.98019644524265e-07,
|
||
|
|
"loss": 0.1055484414100647,
|
||
|
|
"num_tokens": 68277.0,
|
||
|
|
"reward": -0.7,
|
||
|
|
"reward_std": 0.2,
|
||
|
|
"rewards/reward_fn/mean": -0.7,
|
||
|
|
"rewards/reward_fn/std": 0.2,
|
||
|
|
"step": 265,
|
||
|
|
"step_time": 0.6585672344081104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.4,
|
||
|
|
"completions/max_terminated_length": 17.4,
|
||
|
|
"completions/mean_length": 11.15,
|
||
|
|
"completions/mean_terminated_length": 11.15,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 1.0371261090040207,
|
||
|
|
"epoch": 0.27835051546391754,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.875,
|
||
|
|
"learning_rate": 4.97727077497292e-07,
|
||
|
|
"loss": 0.13463952541351318,
|
||
|
|
"num_tokens": 69496.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.20773502588272094,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.20773502588272094,
|
||
|
|
"step": 270,
|
||
|
|
"step_time": 0.5468919139355422
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.0,
|
||
|
|
"completions/max_terminated_length": 15.0,
|
||
|
|
"completions/mean_length": 9.3,
|
||
|
|
"completions/mean_terminated_length": 9.3,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.7886780798435211,
|
||
|
|
"epoch": 0.28350515463917525,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 4.974144601264506e-07,
|
||
|
|
"loss": 0.21641788482666016,
|
||
|
|
"num_tokens": 70690.0,
|
||
|
|
"reward": -0.41000000238418577,
|
||
|
|
"reward_std": 0.4559956431388855,
|
||
|
|
"rewards/reward_fn/mean": -0.41000000238418577,
|
||
|
|
"rewards/reward_fn/std": 0.4559956431388855,
|
||
|
|
"step": 275,
|
||
|
|
"step_time": 0.5083554971963167
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 11.9,
|
||
|
|
"completions/mean_terminated_length": 11.9,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.9502978086471557,
|
||
|
|
"epoch": 0.28865979381443296,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.970818177141255e-07,
|
||
|
|
"loss": 0.03687924444675446,
|
||
|
|
"num_tokens": 71972.0,
|
||
|
|
"reward": -0.4033333331346512,
|
||
|
|
"reward_std": 0.3089271903038025,
|
||
|
|
"rewards/reward_fn/mean": -0.4033333331346512,
|
||
|
|
"rewards/reward_fn/std": 0.30892720222473147,
|
||
|
|
"step": 280,
|
||
|
|
"step_time": 0.5869207398965954
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.0,
|
||
|
|
"completions/max_terminated_length": 24.0,
|
||
|
|
"completions/mean_length": 14.7,
|
||
|
|
"completions/mean_terminated_length": 14.7,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 1.0687203228473663,
|
||
|
|
"epoch": 0.29381443298969073,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 19.0,
|
||
|
|
"learning_rate": 4.967291771834726e-07,
|
||
|
|
"loss": 0.006063053011894226,
|
||
|
|
"num_tokens": 73274.0,
|
||
|
|
"reward": -0.5199999988079071,
|
||
|
|
"reward_std": 0.4233161449432373,
|
||
|
|
"rewards/reward_fn/mean": -0.5199999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.4233161449432373,
|
||
|
|
"step": 285,
|
||
|
|
"step_time": 0.6561609494499863
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 8.55,
|
||
|
|
"completions/mean_terminated_length": 8.55,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.5886041432619095,
|
||
|
|
"epoch": 0.29896907216494845,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.5,
|
||
|
|
"learning_rate": 4.963565670762404e-07,
|
||
|
|
"loss": 0.20434038639068602,
|
||
|
|
"num_tokens": 74445.0,
|
||
|
|
"reward": -0.3962500035762787,
|
||
|
|
"reward_std": 0.48144287467002866,
|
||
|
|
"rewards/reward_fn/mean": -0.3962500035762787,
|
||
|
|
"rewards/reward_fn/std": 0.48144287467002866,
|
||
|
|
"step": 290,
|
||
|
|
"step_time": 0.4511735229752958
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.8,
|
||
|
|
"completions/max_terminated_length": 13.8,
|
||
|
|
"completions/mean_length": 10.25,
|
||
|
|
"completions/mean_terminated_length": 10.25,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8686697453260421,
|
||
|
|
"epoch": 0.30412371134020616,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.959640175504593e-07,
|
||
|
|
"loss": 0.0394548237323761,
|
||
|
|
"num_tokens": 75694.0,
|
||
|
|
"reward": -0.4675000011920929,
|
||
|
|
"reward_std": 0.16499999761581421,
|
||
|
|
"rewards/reward_fn/mean": -0.4675000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.16499999761581421,
|
||
|
|
"step": 295,
|
||
|
|
"step_time": 0.474055802822113
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.0,
|
||
|
|
"completions/max_terminated_length": 15.0,
|
||
|
|
"completions/mean_length": 9.85,
|
||
|
|
"completions/mean_terminated_length": 9.85,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.7599256873130799,
|
||
|
|
"epoch": 0.30927835051546393,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.25,
|
||
|
|
"learning_rate": 4.955515603780013e-07,
|
||
|
|
"loss": 0.16931414604187012,
|
||
|
|
"num_tokens": 76915.0,
|
||
|
|
"reward": -0.49083333611488345,
|
||
|
|
"reward_std": 0.35166221857070923,
|
||
|
|
"rewards/reward_fn/mean": -0.49083333611488345,
|
||
|
|
"rewards/reward_fn/std": 0.35166221857070923,
|
||
|
|
"step": 300,
|
||
|
|
"step_time": 0.4883337885141373
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 29.6,
|
||
|
|
"completions/max_terminated_length": 29.6,
|
||
|
|
"completions/mean_length": 14.55,
|
||
|
|
"completions/mean_terminated_length": 14.55,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.8846446335315704,
|
||
|
|
"epoch": 0.31443298969072164,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 8.375,
|
||
|
|
"learning_rate": 4.951192289420081e-07,
|
||
|
|
"loss": 0.11423228979110718,
|
||
|
|
"num_tokens": 78226.0,
|
||
|
|
"reward": -0.5699999988079071,
|
||
|
|
"reward_std": 0.192376047372818,
|
||
|
|
"rewards/reward_fn/mean": -0.5699999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.192376047372818,
|
||
|
|
"step": 305,
|
||
|
|
"step_time": 0.7256460329517722
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 35.4,
|
||
|
|
"completions/max_terminated_length": 35.4,
|
||
|
|
"completions/mean_length": 17.4,
|
||
|
|
"completions/mean_terminated_length": 17.4,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 1.0954357445240022,
|
||
|
|
"epoch": 0.31958762886597936,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.946670582341894e-07,
|
||
|
|
"loss": 0.2573416709899902,
|
||
|
|
"num_tokens": 79606.0,
|
||
|
|
"reward": -0.675,
|
||
|
|
"reward_std": 0.20773502588272094,
|
||
|
|
"rewards/reward_fn/mean": -0.675,
|
||
|
|
"rewards/reward_fn/std": 0.20773502588272094,
|
||
|
|
"step": 310,
|
||
|
|
"step_time": 0.8166594183072448
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 28.4,
|
||
|
|
"completions/max_terminated_length": 28.4,
|
||
|
|
"completions/mean_length": 14.6,
|
||
|
|
"completions/mean_terminated_length": 14.6,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.6663665235042572,
|
||
|
|
"epoch": 0.3247422680412371,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 10.75,
|
||
|
|
"learning_rate": 4.941950848519903e-07,
|
||
|
|
"loss": 0.12805309295654296,
|
||
|
|
"num_tokens": 80894.0,
|
||
|
|
"reward": -0.41166666746139524,
|
||
|
|
"reward_std": 0.21860073804855346,
|
||
|
|
"rewards/reward_fn/mean": -0.41166666746139524,
|
||
|
|
"rewards/reward_fn/std": 0.21860073804855346,
|
||
|
|
"step": 315,
|
||
|
|
"step_time": 0.691332861687988
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 44.8,
|
||
|
|
"completions/max_terminated_length": 44.8,
|
||
|
|
"completions/mean_length": 18.1,
|
||
|
|
"completions/mean_terminated_length": 18.1,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 1.0570011407136917,
|
||
|
|
"epoch": 0.32989690721649484,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 17.5,
|
||
|
|
"learning_rate": 4.937033469956302e-07,
|
||
|
|
"loss": 0.2395000457763672,
|
||
|
|
"num_tokens": 82264.0,
|
||
|
|
"reward": -0.4905555546283722,
|
||
|
|
"reward_std": 0.21888889074325563,
|
||
|
|
"rewards/reward_fn/mean": -0.4905555546283722,
|
||
|
|
"rewards/reward_fn/std": 0.21888889074325563,
|
||
|
|
"step": 320,
|
||
|
|
"step_time": 1.0275981751270593
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.2,
|
||
|
|
"completions/max_terminated_length": 20.2,
|
||
|
|
"completions/mean_length": 12.15,
|
||
|
|
"completions/mean_terminated_length": 12.15,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.7988920092582703,
|
||
|
|
"epoch": 0.33505154639175255,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 4.931918844650094e-07,
|
||
|
|
"loss": 0.04101354777812958,
|
||
|
|
"num_tokens": 83523.0,
|
||
|
|
"reward": -0.4850000023841858,
|
||
|
|
"reward_std": 0.10754843950271606,
|
||
|
|
"rewards/reward_fn/mean": -0.4850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.10754843950271606,
|
||
|
|
"step": 325,
|
||
|
|
"step_time": 0.5553480639122427
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 34.6,
|
||
|
|
"completions/max_terminated_length": 34.6,
|
||
|
|
"completions/mean_length": 15.25,
|
||
|
|
"completions/mean_terminated_length": 15.25,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.7929889589548111,
|
||
|
|
"epoch": 0.3402061855670103,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 24.75,
|
||
|
|
"learning_rate": 4.926607386564898e-07,
|
||
|
|
"loss": 0.14523402452468873,
|
||
|
|
"num_tokens": 84840.0,
|
||
|
|
"reward": -0.6350000023841857,
|
||
|
|
"reward_std": 0.32301849126815796,
|
||
|
|
"rewards/reward_fn/mean": -0.6350000023841857,
|
||
|
|
"rewards/reward_fn/std": 0.32301849126815796,
|
||
|
|
"step": 330,
|
||
|
|
"step_time": 0.7962054027244448
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 9.6,
|
||
|
|
"completions/max_terminated_length": 9.6,
|
||
|
|
"completions/mean_length": 7.5,
|
||
|
|
"completions/mean_terminated_length": 7.5,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.48006263077259065,
|
||
|
|
"epoch": 0.34536082474226804,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.921099525595423e-07,
|
||
|
|
"loss": 0.10299931764602661,
|
||
|
|
"num_tokens": 85986.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 335,
|
||
|
|
"step_time": 0.38521298291161654
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.8,
|
||
|
|
"completions/max_terminated_length": 17.8,
|
||
|
|
"completions/mean_length": 10.75,
|
||
|
|
"completions/mean_terminated_length": 10.75,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9134504497051239,
|
||
|
|
"epoch": 0.35051546391752575,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.915395707532691e-07,
|
||
|
|
"loss": 0.017949485778808595,
|
||
|
|
"num_tokens": 87201.0,
|
||
|
|
"reward": -0.55,
|
||
|
|
"reward_std": 0.1,
|
||
|
|
"rewards/reward_fn/mean": -0.55,
|
||
|
|
"rewards/reward_fn/std": 0.1,
|
||
|
|
"step": 340,
|
||
|
|
"step_time": 0.5092797569930554
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.2,
|
||
|
|
"completions/max_terminated_length": 13.2,
|
||
|
|
"completions/mean_length": 10.25,
|
||
|
|
"completions/mean_terminated_length": 10.25,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9801588356494904,
|
||
|
|
"epoch": 0.3556701030927835,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 24.125,
|
||
|
|
"learning_rate": 4.909496394027944e-07,
|
||
|
|
"loss": 0.12110986709594726,
|
||
|
|
"num_tokens": 88402.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 345,
|
||
|
|
"step_time": 0.44093081960454583
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 91.8,
|
||
|
|
"completions/max_terminated_length": 91.8,
|
||
|
|
"completions/mean_length": 32.55,
|
||
|
|
"completions/mean_terminated_length": 32.55,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.510153776407242,
|
||
|
|
"epoch": 0.36082474226804123,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 7.4375,
|
||
|
|
"learning_rate": 4.903402062555289e-07,
|
||
|
|
"loss": -0.05523960590362549,
|
||
|
|
"num_tokens": 90061.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.16547005176544188,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.16547005176544188,
|
||
|
|
"step": 350,
|
||
|
|
"step_time": 1.6976578251458705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.0,
|
||
|
|
"completions/max_terminated_length": 25.0,
|
||
|
|
"completions/mean_length": 11.8,
|
||
|
|
"completions/mean_terminated_length": 11.8,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.9649739861488342,
|
||
|
|
"epoch": 0.36597938144329895,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.5,
|
||
|
|
"learning_rate": 4.897113206373042e-07,
|
||
|
|
"loss": 0.3782911539077759,
|
||
|
|
"num_tokens": 91317.0,
|
||
|
|
"reward": -0.425,
|
||
|
|
"reward_std": 0.32320507764816286,
|
||
|
|
"rewards/reward_fn/mean": -0.425,
|
||
|
|
"rewards/reward_fn/std": 0.32320507764816286,
|
||
|
|
"step": 355,
|
||
|
|
"step_time": 0.6405022434890271
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 9.95,
|
||
|
|
"completions/mean_terminated_length": 9.95,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.9136595696210861,
|
||
|
|
"epoch": 0.3711340206185567,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 22.125,
|
||
|
|
"learning_rate": 4.890630334483814e-07,
|
||
|
|
"loss": 0.13049967288970948,
|
||
|
|
"num_tokens": 92528.0,
|
||
|
|
"reward": -0.4025000035762787,
|
||
|
|
"reward_std": 0.5875245332717896,
|
||
|
|
"rewards/reward_fn/mean": -0.4025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.5875245332717896,
|
||
|
|
"step": 360,
|
||
|
|
"step_time": 0.5148123029619456
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 11.0,
|
||
|
|
"completions/mean_terminated_length": 11.0,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 1.0597147643566132,
|
||
|
|
"epoch": 0.37628865979381443,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 4.883953971593308e-07,
|
||
|
|
"loss": 0.4041938304901123,
|
||
|
|
"num_tokens": 93772.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.25,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.25,
|
||
|
|
"step": 365,
|
||
|
|
"step_time": 0.6115478255786002
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 52.8,
|
||
|
|
"completions/max_terminated_length": 52.8,
|
||
|
|
"completions/mean_length": 20.05,
|
||
|
|
"completions/mean_terminated_length": 20.05,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.7085862278938293,
|
||
|
|
"epoch": 0.38144329896907214,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 4.877084658067855e-07,
|
||
|
|
"loss": 0.40793662071228026,
|
||
|
|
"num_tokens": 95209.0,
|
||
|
|
"reward": -0.6233333319425582,
|
||
|
|
"reward_std": 0.3110683560371399,
|
||
|
|
"rewards/reward_fn/mean": -0.6233333319425582,
|
||
|
|
"rewards/reward_fn/std": 0.31106836795806886,
|
||
|
|
"step": 370,
|
||
|
|
"step_time": 1.0920773649588227
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.2,
|
||
|
|
"completions/max_terminated_length": 17.2,
|
||
|
|
"completions/mean_length": 11.65,
|
||
|
|
"completions/mean_terminated_length": 11.65,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.7929205089807511,
|
||
|
|
"epoch": 0.3865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 42.75,
|
||
|
|
"learning_rate": 4.870022949890676e-07,
|
||
|
|
"loss": 0.1025731086730957,
|
||
|
|
"num_tokens": 96430.0,
|
||
|
|
"reward": -0.3235000044107437,
|
||
|
|
"reward_std": 0.4177010774612427,
|
||
|
|
"rewards/reward_fn/mean": -0.3235000044107437,
|
||
|
|
"rewards/reward_fn/std": 0.41770108938217165,
|
||
|
|
"step": 375,
|
||
|
|
"step_time": 0.5476652001962066
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 9.85,
|
||
|
|
"completions/mean_terminated_length": 9.85,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8785172998905182,
|
||
|
|
"epoch": 0.3917525773195876,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 4.862769418616881e-07,
|
||
|
|
"loss": 0.01562568247318268,
|
||
|
|
"num_tokens": 97623.0,
|
||
|
|
"reward": -0.5175000011920929,
|
||
|
|
"reward_std": 0.42273502349853515,
|
||
|
|
"rewards/reward_fn/mean": -0.5175000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.42273502349853515,
|
||
|
|
"step": 380,
|
||
|
|
"step_time": 0.5373218949884176
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 43.0,
|
||
|
|
"completions/max_terminated_length": 43.0,
|
||
|
|
"completions/mean_length": 17.65,
|
||
|
|
"completions/mean_terminated_length": 17.65,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.6945732861757279,
|
||
|
|
"epoch": 0.39690721649484534,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 4.855324651327211e-07,
|
||
|
|
"loss": 0.32841565608978274,
|
||
|
|
"num_tokens": 98984.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.5768915295600892,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.5768915295600892,
|
||
|
|
"step": 385,
|
||
|
|
"step_time": 1.0561855778098106
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.2,
|
||
|
|
"completions/max_terminated_length": 14.2,
|
||
|
|
"completions/mean_length": 7.8,
|
||
|
|
"completions/mean_terminated_length": 7.8,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 1.1521853357553482,
|
||
|
|
"epoch": 0.4020618556701031,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 42.75,
|
||
|
|
"learning_rate": 4.847689250580522e-07,
|
||
|
|
"loss": 0.1270517587661743,
|
||
|
|
"num_tokens": 100152.0,
|
||
|
|
"reward": -0.45,
|
||
|
|
"reward_std": 0.35615527629852295,
|
||
|
|
"rewards/reward_fn/mean": -0.45,
|
||
|
|
"rewards/reward_fn/std": 0.35615527629852295,
|
||
|
|
"step": 390,
|
||
|
|
"step_time": 0.5240602082572877
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.8,
|
||
|
|
"completions/max_terminated_length": 27.8,
|
||
|
|
"completions/mean_length": 15.45,
|
||
|
|
"completions/mean_terminated_length": 15.45,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.6790560841560364,
|
||
|
|
"epoch": 0.4072164948453608,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.125,
|
||
|
|
"learning_rate": 4.839863834365013e-07,
|
||
|
|
"loss": 0.03930776119232178,
|
||
|
|
"num_tokens": 101485.0,
|
||
|
|
"reward": -0.75,
|
||
|
|
"reward_std": 0.2154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.75,
|
||
|
|
"rewards/reward_fn/std": 0.2154700517654419,
|
||
|
|
"step": 395,
|
||
|
|
"step_time": 0.769484898634255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.4,
|
||
|
|
"completions/max_terminated_length": 17.4,
|
||
|
|
"completions/mean_length": 11.0,
|
||
|
|
"completions/mean_terminated_length": 11.0,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.970324519276619,
|
||
|
|
"epoch": 0.41237113402061853,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.831849036048209e-07,
|
||
|
|
"loss": 0.12541403770446777,
|
||
|
|
"num_tokens": 102733.0,
|
||
|
|
"reward": -0.4425000011920929,
|
||
|
|
"reward_std": 0.2863911747932434,
|
||
|
|
"rewards/reward_fn/mean": -0.4425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2863911747932434,
|
||
|
|
"step": 400,
|
||
|
|
"step_time": 0.7362870733253658
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 10.0,
|
||
|
|
"completions/mean_terminated_length": 10.0,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.7727616012096405,
|
||
|
|
"epoch": 0.4175257731958763,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.823645504325699e-07,
|
||
|
|
"loss": 0.09246260523796082,
|
||
|
|
"num_tokens": 103969.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.32320507764816286,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.32320507764816286,
|
||
|
|
"step": 405,
|
||
|
|
"step_time": 0.5494077072478831
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.2,
|
||
|
|
"completions/max_terminated_length": 26.2,
|
||
|
|
"completions/mean_length": 15.35,
|
||
|
|
"completions/mean_terminated_length": 15.35,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.9095003753900528,
|
||
|
|
"epoch": 0.422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 11.875,
|
||
|
|
"learning_rate": 4.815253903168629e-07,
|
||
|
|
"loss": 0.04959979057312012,
|
||
|
|
"num_tokens": 105292.0,
|
||
|
|
"reward": -0.48850000500679014,
|
||
|
|
"reward_std": 0.338838791847229,
|
||
|
|
"rewards/reward_fn/mean": -0.48850000500679014,
|
||
|
|
"rewards/reward_fn/std": 0.338838791847229,
|
||
|
|
"step": 410,
|
||
|
|
"step_time": 0.7390698280185461
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.0,
|
||
|
|
"completions/max_terminated_length": 17.0,
|
||
|
|
"completions/mean_length": 12.45,
|
||
|
|
"completions/mean_terminated_length": 12.45,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.7868008881807327,
|
||
|
|
"epoch": 0.42783505154639173,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 24.625,
|
||
|
|
"learning_rate": 4.806674911769968e-07,
|
||
|
|
"loss": 0.1086654782295227,
|
||
|
|
"num_tokens": 106557.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.2300000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2300000011920929,
|
||
|
|
"step": 415,
|
||
|
|
"step_time": 0.5790117777884006
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 8.6,
|
||
|
|
"completions/max_terminated_length": 8.6,
|
||
|
|
"completions/mean_length": 7.15,
|
||
|
|
"completions/mean_terminated_length": 7.15,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 0.7389998614788056,
|
||
|
|
"epoch": 0.4329896907216495,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 13.375,
|
||
|
|
"learning_rate": 4.79790922448953e-07,
|
||
|
|
"loss": 0.06091185808181763,
|
||
|
|
"num_tokens": 107692.0,
|
||
|
|
"reward": -0.4850000023841858,
|
||
|
|
"reward_std": 0.1300000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.4850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.1300000011920929,
|
||
|
|
"step": 420,
|
||
|
|
"step_time": 0.43482481176033616
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.8,
|
||
|
|
"completions/max_terminated_length": 14.8,
|
||
|
|
"completions/mean_length": 10.5,
|
||
|
|
"completions/mean_terminated_length": 10.5,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.9143629461526871,
|
||
|
|
"epoch": 0.4381443298969072,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 4.788957550797777e-07,
|
||
|
|
"loss": -0.00275689959526062,
|
||
|
|
"num_tokens": 108958.0,
|
||
|
|
"reward": -0.08333334028720855,
|
||
|
|
"reward_std": 0.412871390581131,
|
||
|
|
"rewards/reward_fn/mean": -0.08333334028720855,
|
||
|
|
"rewards/reward_fn/std": 0.412871390581131,
|
||
|
|
"step": 425,
|
||
|
|
"step_time": 0.5315653963014484
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.6,
|
||
|
|
"completions/max_terminated_length": 10.6,
|
||
|
|
"completions/mean_length": 8.7,
|
||
|
|
"completions/mean_terminated_length": 8.7,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.6365309089422226,
|
||
|
|
"epoch": 0.44329896907216493,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.779820615218398e-07,
|
||
|
|
"loss": 0.04290743470191956,
|
||
|
|
"num_tokens": 110160.0,
|
||
|
|
"reward": -0.5366666674613952,
|
||
|
|
"reward_std": 0.2586212992668152,
|
||
|
|
"rewards/reward_fn/mean": -0.5366666674613952,
|
||
|
|
"rewards/reward_fn/std": 0.2586212992668152,
|
||
|
|
"step": 430,
|
||
|
|
"step_time": 0.5582886083982885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.2,
|
||
|
|
"completions/max_terminated_length": 15.2,
|
||
|
|
"completions/mean_length": 11.95,
|
||
|
|
"completions/mean_terminated_length": 11.95,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.6503553047776223,
|
||
|
|
"epoch": 0.4484536082474227,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 9.5625,
|
||
|
|
"learning_rate": 4.770499157269663e-07,
|
||
|
|
"loss": -0.07173663973808289,
|
||
|
|
"num_tokens": 111435.0,
|
||
|
|
"reward": -0.3712500035762787,
|
||
|
|
"reward_std": 0.32023502588272096,
|
||
|
|
"rewards/reward_fn/mean": -0.3712500035762787,
|
||
|
|
"rewards/reward_fn/std": 0.32023502588272096,
|
||
|
|
"step": 435,
|
||
|
|
"step_time": 0.5316761738620699
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.7,
|
||
|
|
"completions/mean_terminated_length": 10.7,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.6362611055374146,
|
||
|
|
"epoch": 0.4536082474226804,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.7609939314045755e-07,
|
||
|
|
"loss": 0.08669210076332093,
|
||
|
|
"num_tokens": 112669.0,
|
||
|
|
"reward": -0.5,
|
||
|
|
"reward_std": 0.2732050776481628,
|
||
|
|
"rewards/reward_fn/mean": -0.5,
|
||
|
|
"rewards/reward_fn/std": 0.2732050776481628,
|
||
|
|
"step": 440,
|
||
|
|
"step_time": 0.5227897326461971
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.0,
|
||
|
|
"completions/max_terminated_length": 16.0,
|
||
|
|
"completions/mean_length": 10.25,
|
||
|
|
"completions/mean_terminated_length": 10.25,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.5979740798473359,
|
||
|
|
"epoch": 0.4587628865979381,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 12.75,
|
||
|
|
"learning_rate": 4.7513057069498026e-07,
|
||
|
|
"loss": 0.1255246639251709,
|
||
|
|
"num_tokens": 113866.0,
|
||
|
|
"reward": -0.39499999582767487,
|
||
|
|
"reward_std": 0.30333333015441893,
|
||
|
|
"rewards/reward_fn/mean": -0.39499999582767487,
|
||
|
|
"rewards/reward_fn/std": 0.30333333015441893,
|
||
|
|
"step": 445,
|
||
|
|
"step_time": 0.5488045696169138
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.6,
|
||
|
|
"completions/max_terminated_length": 10.6,
|
||
|
|
"completions/mean_length": 9.0,
|
||
|
|
"completions/mean_terminated_length": 9.0,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.7348838299512863,
|
||
|
|
"epoch": 0.4639175257731959,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 22.75,
|
||
|
|
"learning_rate": 4.741435268043411e-07,
|
||
|
|
"loss": 0.10615365505218506,
|
||
|
|
"num_tokens": 115050.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.3470319628715515,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.3470319747924805,
|
||
|
|
"step": 450,
|
||
|
|
"step_time": 0.45176689634099604
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 44.2,
|
||
|
|
"completions/max_terminated_length": 44.2,
|
||
|
|
"completions/mean_length": 18.3,
|
||
|
|
"completions/mean_terminated_length": 18.3,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.7693002432584762,
|
||
|
|
"epoch": 0.4690721649484536,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 4.7313834135714036e-07,
|
||
|
|
"loss": 0.41129140853881835,
|
||
|
|
"num_tokens": 116436.0,
|
||
|
|
"reward": -0.55,
|
||
|
|
"reward_std": 0.3154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.55,
|
||
|
|
"rewards/reward_fn/std": 0.3154700517654419,
|
||
|
|
"step": 455,
|
||
|
|
"step_time": 1.0780604547820984
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 11.65,
|
||
|
|
"completions/mean_terminated_length": 11.65,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.7763857960700988,
|
||
|
|
"epoch": 0.4742268041237113,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 19.75,
|
||
|
|
"learning_rate": 4.721150957103053e-07,
|
||
|
|
"loss": 0.03669398725032806,
|
||
|
|
"num_tokens": 117713.0,
|
||
|
|
"reward": -0.5924999952316284,
|
||
|
|
"reward_std": 0.2727350234985352,
|
||
|
|
"rewards/reward_fn/mean": -0.5924999952316284,
|
||
|
|
"rewards/reward_fn/std": 0.2727350234985352,
|
||
|
|
"step": 460,
|
||
|
|
"step_time": 0.5201641391962767
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.6,
|
||
|
|
"completions/max_terminated_length": 13.6,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.8593772113323211,
|
||
|
|
"epoch": 0.4793814432989691,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.7107387268250586e-07,
|
||
|
|
"loss": 0.06270080804824829,
|
||
|
|
"num_tokens": 118905.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 465,
|
||
|
|
"step_time": 0.527864100690931
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.0,
|
||
|
|
"completions/max_terminated_length": 18.0,
|
||
|
|
"completions/mean_length": 13.5,
|
||
|
|
"completions/mean_terminated_length": 13.5,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 1.0802289962768554,
|
||
|
|
"epoch": 0.4845360824742268,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 31.625,
|
||
|
|
"learning_rate": 4.700147565474517e-07,
|
||
|
|
"loss": -0.023255640268325807,
|
||
|
|
"num_tokens": 120207.0,
|
||
|
|
"reward": -0.28666666746139524,
|
||
|
|
"reward_std": 0.2598089694976807,
|
||
|
|
"rewards/reward_fn/mean": -0.28666666746139524,
|
||
|
|
"rewards/reward_fn/std": 0.2598089694976807,
|
||
|
|
"step": 470,
|
||
|
|
"step_time": 0.5613622773438692
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 56.4,
|
||
|
|
"completions/max_terminated_length": 56.4,
|
||
|
|
"completions/mean_length": 23.7,
|
||
|
|
"completions/mean_terminated_length": 23.7,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.9632985845208168,
|
||
|
|
"epoch": 0.4896907216494845,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 28.875,
|
||
|
|
"learning_rate": 4.689378330270707e-07,
|
||
|
|
"loss": 0.08240939378738403,
|
||
|
|
"num_tokens": 121725.0,
|
||
|
|
"reward": -0.3850000023841858,
|
||
|
|
"reward_std": 0.3727463483810425,
|
||
|
|
"rewards/reward_fn/mean": -0.3850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.3727463483810425,
|
||
|
|
"step": 475,
|
||
|
|
"step_time": 1.2693270226940512
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 28.8,
|
||
|
|
"completions/max_terminated_length": 28.8,
|
||
|
|
"completions/mean_length": 16.3,
|
||
|
|
"completions/mean_terminated_length": 16.3,
|
||
|
|
"completions/min_length": 10.8,
|
||
|
|
"completions/min_terminated_length": 10.8,
|
||
|
|
"entropy": 0.5772907376289368,
|
||
|
|
"epoch": 0.4948453608247423,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.678431892845713e-07,
|
||
|
|
"loss": -0.026522517204284668,
|
||
|
|
"num_tokens": 123051.0,
|
||
|
|
"reward": -0.502999997138977,
|
||
|
|
"reward_std": 0.27458140850067136,
|
||
|
|
"rewards/reward_fn/mean": -0.502999997138977,
|
||
|
|
"rewards/reward_fn/std": 0.27458140850067136,
|
||
|
|
"step": 480,
|
||
|
|
"step_time": 0.721957803145051
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 46.2,
|
||
|
|
"completions/max_terminated_length": 46.2,
|
||
|
|
"completions/mean_length": 21.55,
|
||
|
|
"completions/mean_terminated_length": 21.55,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.7705437213182449,
|
||
|
|
"epoch": 0.5,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.1875,
|
||
|
|
"learning_rate": 4.6673091391738785e-07,
|
||
|
|
"loss": 0.1539371132850647,
|
||
|
|
"num_tokens": 124502.0,
|
||
|
|
"reward": -0.575555557012558,
|
||
|
|
"reward_std": 0.3588018655776978,
|
||
|
|
"rewards/reward_fn/mean": -0.575555557012558,
|
||
|
|
"rewards/reward_fn/std": 0.3588018655776978,
|
||
|
|
"step": 485,
|
||
|
|
"step_time": 1.0199918281286955
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 50.4,
|
||
|
|
"completions/max_terminated_length": 50.4,
|
||
|
|
"completions/mean_length": 20.05,
|
||
|
|
"completions/mean_terminated_length": 20.05,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.837895593047142,
|
||
|
|
"epoch": 0.5051546391752577,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 12.3125,
|
||
|
|
"learning_rate": 4.656010969500094e-07,
|
||
|
|
"loss": 0.3263136863708496,
|
||
|
|
"num_tokens": 125931.0,
|
||
|
|
"reward": -0.4266666680574417,
|
||
|
|
"reward_std": 0.46954628825187683,
|
||
|
|
"rewards/reward_fn/mean": -0.4266666680574417,
|
||
|
|
"rewards/reward_fn/std": 0.4695463001728058,
|
||
|
|
"step": 490,
|
||
|
|
"step_time": 1.08096160069108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.6,
|
||
|
|
"completions/max_terminated_length": 21.6,
|
||
|
|
"completions/mean_length": 15.5,
|
||
|
|
"completions/mean_terminated_length": 15.5,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.680565445125103,
|
||
|
|
"epoch": 0.5103092783505154,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 23.25,
|
||
|
|
"learning_rate": 4.6445382982669354e-07,
|
||
|
|
"loss": 0.13636120557785034,
|
||
|
|
"num_tokens": 127289.0,
|
||
|
|
"reward": -0.5175000011920929,
|
||
|
|
"reward_std": 0.23985110521316527,
|
||
|
|
"rewards/reward_fn/mean": -0.5175000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.23985110521316527,
|
||
|
|
"step": 495,
|
||
|
|
"step_time": 0.5912062424235046
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.2,
|
||
|
|
"completions/max_terminated_length": 27.2,
|
||
|
|
"completions/mean_length": 12.95,
|
||
|
|
"completions/mean_terminated_length": 12.95,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.105911335349083,
|
||
|
|
"epoch": 0.5154639175257731,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.6328920540406536e-07,
|
||
|
|
"loss": 0.17237958908081055,
|
||
|
|
"num_tokens": 128560.0,
|
||
|
|
"reward": -0.5600000023841858,
|
||
|
|
"reward_std": 0.21528346538543702,
|
||
|
|
"rewards/reward_fn/mean": -0.5600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.21528346538543702,
|
||
|
|
"step": 500,
|
||
|
|
"step_time": 0.6783841417171061
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 12.6,
|
||
|
|
"completions/mean_terminated_length": 12.6,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.8051907598972321,
|
||
|
|
"epoch": 0.520618556701031,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.5,
|
||
|
|
"learning_rate": 4.621073179436015e-07,
|
||
|
|
"loss": 0.10203053951263427,
|
||
|
|
"num_tokens": 129836.0,
|
||
|
|
"reward": -0.35850000083446504,
|
||
|
|
"reward_std": 0.6104013562202454,
|
||
|
|
"rewards/reward_fn/mean": -0.35850000083446504,
|
||
|
|
"rewards/reward_fn/std": 0.6104013681411743,
|
||
|
|
"step": 505,
|
||
|
|
"step_time": 0.5657726871781051
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.8,
|
||
|
|
"completions/max_terminated_length": 20.8,
|
||
|
|
"completions/mean_length": 11.55,
|
||
|
|
"completions/mean_terminated_length": 11.55,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.7712950989603996,
|
||
|
|
"epoch": 0.5257731958762887,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 8.5625,
|
||
|
|
"learning_rate": 4.609082631040011e-07,
|
||
|
|
"loss": 0.1616443634033203,
|
||
|
|
"num_tokens": 131107.0,
|
||
|
|
"reward": -0.4925000011920929,
|
||
|
|
"reward_std": 0.44758613109588624,
|
||
|
|
"rewards/reward_fn/mean": -0.4925000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.44758613109588624,
|
||
|
|
"step": 510,
|
||
|
|
"step_time": 0.6361785892397165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.2,
|
||
|
|
"completions/max_terminated_length": 27.2,
|
||
|
|
"completions/mean_length": 16.35,
|
||
|
|
"completions/mean_terminated_length": 16.35,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 1.0122096598148347,
|
||
|
|
"epoch": 0.5309278350515464,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 20.75,
|
||
|
|
"learning_rate": 4.5969213793344373e-07,
|
||
|
|
"loss": 0.028887468576431274,
|
||
|
|
"num_tokens": 132482.0,
|
||
|
|
"reward": -0.38416666984558107,
|
||
|
|
"reward_std": 0.3881505250930786,
|
||
|
|
"rewards/reward_fn/mean": -0.38416666984558107,
|
||
|
|
"rewards/reward_fn/std": 0.3881505370140076,
|
||
|
|
"step": 515,
|
||
|
|
"step_time": 0.755103122163564
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 31.0,
|
||
|
|
"completions/max_terminated_length": 31.0,
|
||
|
|
"completions/mean_length": 17.75,
|
||
|
|
"completions/mean_terminated_length": 17.75,
|
||
|
|
"completions/min_length": 10.4,
|
||
|
|
"completions/min_terminated_length": 10.4,
|
||
|
|
"entropy": 0.8129868268966675,
|
||
|
|
"epoch": 0.5360824742268041,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 23.25,
|
||
|
|
"learning_rate": 4.5845904086173405e-07,
|
||
|
|
"loss": 0.20167853832244872,
|
||
|
|
"num_tokens": 133845.0,
|
||
|
|
"reward": -0.725,
|
||
|
|
"reward_std": 0.22320507764816283,
|
||
|
|
"rewards/reward_fn/mean": -0.725,
|
||
|
|
"rewards/reward_fn/std": 0.22320507764816283,
|
||
|
|
"step": 520,
|
||
|
|
"step_time": 0.9605252320878208
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.8,
|
||
|
|
"completions/max_terminated_length": 17.8,
|
||
|
|
"completions/mean_length": 12.3,
|
||
|
|
"completions/mean_terminated_length": 12.3,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.8394870400428772,
|
||
|
|
"epoch": 0.5412371134020618,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 12.4375,
|
||
|
|
"learning_rate": 4.572090716923353e-07,
|
||
|
|
"loss": 0.1577498197555542,
|
||
|
|
"num_tokens": 135115.0,
|
||
|
|
"reward": -0.3600000023841858,
|
||
|
|
"reward_std": 0.41528346538543703,
|
||
|
|
"rewards/reward_fn/mean": -0.3600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.41528346538543703,
|
||
|
|
"step": 525,
|
||
|
|
"step_time": 0.5848557983525098
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 31.6,
|
||
|
|
"completions/max_terminated_length": 31.6,
|
||
|
|
"completions/mean_length": 15.45,
|
||
|
|
"completions/mean_terminated_length": 15.45,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 1.1088833928108215,
|
||
|
|
"epoch": 0.5463917525773195,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 26.375,
|
||
|
|
"learning_rate": 4.5594233159429185e-07,
|
||
|
|
"loss": 0.06881465315818787,
|
||
|
|
"num_tokens": 136436.0,
|
||
|
|
"reward": -0.4225000023841858,
|
||
|
|
"reward_std": 0.13499999642372132,
|
||
|
|
"rewards/reward_fn/mean": -0.4225000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.13499999642372132,
|
||
|
|
"step": 530,
|
||
|
|
"step_time": 0.840742127224803
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 40.2,
|
||
|
|
"completions/max_terminated_length": 40.2,
|
||
|
|
"completions/mean_length": 18.7,
|
||
|
|
"completions/mean_terminated_length": 18.7,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.5888572871685028,
|
||
|
|
"epoch": 0.5515463917525774,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.375,
|
||
|
|
"learning_rate": 4.5465892309404065e-07,
|
||
|
|
"loss": 0.19101574420928955,
|
||
|
|
"num_tokens": 137866.0,
|
||
|
|
"reward": -0.4560000032186508,
|
||
|
|
"reward_std": 0.26554843187332156,
|
||
|
|
"rewards/reward_fn/mean": -0.4560000032186508,
|
||
|
|
"rewards/reward_fn/std": 0.26554843187332156,
|
||
|
|
"step": 535,
|
||
|
|
"step_time": 0.9919451894238591
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 11.0,
|
||
|
|
"completions/mean_terminated_length": 11.0,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.7863050103187561,
|
||
|
|
"epoch": 0.5567010309278351,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.5335895006711257e-07,
|
||
|
|
"loss": 0.058580482006073,
|
||
|
|
"num_tokens": 139102.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 540,
|
||
|
|
"step_time": 0.5204128349199891
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 32.2,
|
||
|
|
"completions/max_terminated_length": 32.2,
|
||
|
|
"completions/mean_length": 14.75,
|
||
|
|
"completions/mean_terminated_length": 14.75,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.8185445249080658,
|
||
|
|
"epoch": 0.5618556701030928,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 13.375,
|
||
|
|
"learning_rate": 4.520425177297259e-07,
|
||
|
|
"loss": 0.25057742595672605,
|
||
|
|
"num_tokens": 140433.0,
|
||
|
|
"reward": -0.3316666603088379,
|
||
|
|
"reward_std": 0.12333332747220993,
|
||
|
|
"rewards/reward_fn/mean": -0.3316666603088379,
|
||
|
|
"rewards/reward_fn/std": 0.12333333045244217,
|
||
|
|
"step": 545,
|
||
|
|
"step_time": 0.8382557584904134
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.6,
|
||
|
|
"completions/max_terminated_length": 13.6,
|
||
|
|
"completions/mean_length": 11.05,
|
||
|
|
"completions/mean_terminated_length": 11.05,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.7716027140617371,
|
||
|
|
"epoch": 0.5670103092783505,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.507097326302696e-07,
|
||
|
|
"loss": 0.040840917825698854,
|
||
|
|
"num_tokens": 141698.0,
|
||
|
|
"reward": -0.4600000023841858,
|
||
|
|
"reward_std": 0.2957271337509155,
|
||
|
|
"rewards/reward_fn/mean": -0.4600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.29572715759277346,
|
||
|
|
"step": 550,
|
||
|
|
"step_time": 0.5102354141883552
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.8,
|
||
|
|
"completions/max_terminated_length": 19.8,
|
||
|
|
"completions/mean_length": 12.75,
|
||
|
|
"completions/mean_terminated_length": 12.75,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.938120248913765,
|
||
|
|
"epoch": 0.5721649484536082,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 29.125,
|
||
|
|
"learning_rate": 4.4936070264068016e-07,
|
||
|
|
"loss": 0.204956579208374,
|
||
|
|
"num_tokens": 142965.0,
|
||
|
|
"reward": -0.3550000011920929,
|
||
|
|
"reward_std": 0.14041451513767242,
|
||
|
|
"rewards/reward_fn/mean": -0.3550000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.14041451513767242,
|
||
|
|
"step": 555,
|
||
|
|
"step_time": 0.6310633135959506
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.2,
|
||
|
|
"completions/max_terminated_length": 11.2,
|
||
|
|
"completions/mean_length": 9.2,
|
||
|
|
"completions/mean_terminated_length": 9.2,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.7356794983148575,
|
||
|
|
"epoch": 0.5773195876288659,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 16.0,
|
||
|
|
"learning_rate": 4.4799553694771043e-07,
|
||
|
|
"loss": 0.024954798817634582,
|
||
|
|
"num_tokens": 144169.0,
|
||
|
|
"reward": -0.2474999964237213,
|
||
|
|
"reward_std": 0.13833333253860475,
|
||
|
|
"rewards/reward_fn/mean": -0.2474999964237213,
|
||
|
|
"rewards/reward_fn/std": 0.13833333253860475,
|
||
|
|
"step": 560,
|
||
|
|
"step_time": 0.5636249963194132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.8,
|
||
|
|
"completions/max_terminated_length": 14.8,
|
||
|
|
"completions/mean_length": 10.1,
|
||
|
|
"completions/mean_terminated_length": 10.1,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.6894051700830459,
|
||
|
|
"epoch": 0.5824742268041238,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.375,
|
||
|
|
"learning_rate": 4.4661434604409234e-07,
|
||
|
|
"loss": 0.09334607720375061,
|
||
|
|
"num_tokens": 145399.0,
|
||
|
|
"reward": -0.3425000011920929,
|
||
|
|
"reward_std": 0.4114371001720428,
|
||
|
|
"rewards/reward_fn/mean": -0.3425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.4114371001720428,
|
||
|
|
"step": 565,
|
||
|
|
"step_time": 0.5348106060177088
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 48.4,
|
||
|
|
"completions/max_terminated_length": 48.4,
|
||
|
|
"completions/mean_length": 22.6,
|
||
|
|
"completions/mean_terminated_length": 22.6,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.6998355552554131,
|
||
|
|
"epoch": 0.5876288659793815,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 19.375,
|
||
|
|
"learning_rate": 4.45217241719594e-07,
|
||
|
|
"loss": 0.21809990406036378,
|
||
|
|
"num_tokens": 146891.0,
|
||
|
|
"reward": -0.5775000035762787,
|
||
|
|
"reward_std": 0.3775861322879791,
|
||
|
|
"rewards/reward_fn/mean": -0.5775000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.3775861322879791,
|
||
|
|
"step": 570,
|
||
|
|
"step_time": 1.1578531011939048
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.0,
|
||
|
|
"completions/max_terminated_length": 13.0,
|
||
|
|
"completions/mean_length": 9.7,
|
||
|
|
"completions/mean_terminated_length": 9.7,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.7059618055820465,
|
||
|
|
"epoch": 0.5927835051546392,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.4380433705197193e-07,
|
||
|
|
"loss": 0.14664100408554076,
|
||
|
|
"num_tokens": 148093.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.2,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.2,
|
||
|
|
"step": 575,
|
||
|
|
"step_time": 0.49879590775817634
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.0,
|
||
|
|
"completions/max_terminated_length": 18.0,
|
||
|
|
"completions/mean_length": 10.2,
|
||
|
|
"completions/mean_terminated_length": 10.2,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 0.8498676598072052,
|
||
|
|
"epoch": 0.5979381443298969,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 25.75,
|
||
|
|
"learning_rate": 4.423757463978186e-07,
|
||
|
|
"loss": 0.1832141637802124,
|
||
|
|
"num_tokens": 149313.0,
|
||
|
|
"reward": -0.5200000047683716,
|
||
|
|
"reward_std": 0.23754844069480896,
|
||
|
|
"rewards/reward_fn/mean": -0.5200000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.23754844069480896,
|
||
|
|
"step": 580,
|
||
|
|
"step_time": 0.5867984751239419
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 53.8,
|
||
|
|
"completions/max_terminated_length": 53.8,
|
||
|
|
"completions/mean_length": 25.25,
|
||
|
|
"completions/mean_terminated_length": 25.25,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.9533327341079711,
|
||
|
|
"epoch": 0.6030927835051546,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.3125,
|
||
|
|
"learning_rate": 4.409315853833067e-07,
|
||
|
|
"loss": 0.22886526584625244,
|
||
|
|
"num_tokens": 150850.0,
|
||
|
|
"reward": -0.5675000011920929,
|
||
|
|
"reward_std": 0.32273502349853517,
|
||
|
|
"rewards/reward_fn/mean": -0.5675000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.32273502349853517,
|
||
|
|
"step": 585,
|
||
|
|
"step_time": 1.2400014178827405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.4,
|
||
|
|
"completions/max_terminated_length": 21.4,
|
||
|
|
"completions/mean_length": 13.65,
|
||
|
|
"completions/mean_terminated_length": 13.65,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 1.055218482017517,
|
||
|
|
"epoch": 0.6082474226804123,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.125,
|
||
|
|
"learning_rate": 4.394719708948309e-07,
|
||
|
|
"loss": 0.11721107959747315,
|
||
|
|
"num_tokens": 152179.0,
|
||
|
|
"reward": -0.3637500017881393,
|
||
|
|
"reward_std": 0.5170861959457398,
|
||
|
|
"rewards/reward_fn/mean": -0.3637500017881393,
|
||
|
|
"rewards/reward_fn/std": 0.5170861840248108,
|
||
|
|
"step": 590,
|
||
|
|
"step_time": 0.6499694635160267
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 10.1,
|
||
|
|
"completions/mean_terminated_length": 10.1,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.8289062976837158,
|
||
|
|
"epoch": 0.6134020618556701,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.3799702106954725e-07,
|
||
|
|
"loss": 0.054711729288101196,
|
||
|
|
"num_tokens": 153401.0,
|
||
|
|
"reward": -0.47750000059604647,
|
||
|
|
"reward_std": 0.24997222423553467,
|
||
|
|
"rewards/reward_fn/mean": -0.47750000059604647,
|
||
|
|
"rewards/reward_fn/std": 0.2499722123146057,
|
||
|
|
"step": 595,
|
||
|
|
"step_time": 0.5453162741847336
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 61.6,
|
||
|
|
"completions/max_terminated_length": 61.6,
|
||
|
|
"completions/mean_length": 23.55,
|
||
|
|
"completions/mean_terminated_length": 23.55,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9436829313635826,
|
||
|
|
"epoch": 0.6185567010309279,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.365068552858115e-07,
|
||
|
|
"loss": 0.03441246449947357,
|
||
|
|
"num_tokens": 154896.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.05,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.05,
|
||
|
|
"step": 600,
|
||
|
|
"step_time": 1.3744830467738212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.0,
|
||
|
|
"completions/max_terminated_length": 19.0,
|
||
|
|
"completions/mean_length": 11.65,
|
||
|
|
"completions/mean_terminated_length": 11.65,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.8497404426336288,
|
||
|
|
"epoch": 0.6237113402061856,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.350015941535169e-07,
|
||
|
|
"loss": 0.1358797073364258,
|
||
|
|
"num_tokens": 156149.0,
|
||
|
|
"reward": -0.17900000214576722,
|
||
|
|
"reward_std": 0.48975425958633423,
|
||
|
|
"rewards/reward_fn/mean": -0.17900000214576722,
|
||
|
|
"rewards/reward_fn/std": 0.48975429534912107,
|
||
|
|
"step": 605,
|
||
|
|
"step_time": 0.7100505581125617
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.0,
|
||
|
|
"completions/max_terminated_length": 19.0,
|
||
|
|
"completions/mean_length": 13.7,
|
||
|
|
"completions/mean_terminated_length": 13.7,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 1.0203462660312652,
|
||
|
|
"epoch": 0.6288659793814433,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.875,
|
||
|
|
"learning_rate": 4.334813595043325e-07,
|
||
|
|
"loss": 0.15752297639846802,
|
||
|
|
"num_tokens": 157443.0,
|
||
|
|
"reward": -0.4810000002384186,
|
||
|
|
"reward_std": 0.2585418462753296,
|
||
|
|
"rewards/reward_fn/mean": -0.4810000002384186,
|
||
|
|
"rewards/reward_fn/std": 0.2585418462753296,
|
||
|
|
"step": 610,
|
||
|
|
"step_time": 0.6184165499173104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.2,
|
||
|
|
"completions/max_terminated_length": 12.2,
|
||
|
|
"completions/mean_length": 8.6,
|
||
|
|
"completions/mean_terminated_length": 8.6,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.6573485657572746,
|
||
|
|
"epoch": 0.634020618556701,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 24.625,
|
||
|
|
"learning_rate": 4.3194627438184233e-07,
|
||
|
|
"loss": 0.09532037973403931,
|
||
|
|
"num_tokens": 158619.0,
|
||
|
|
"reward": -0.20666666626930236,
|
||
|
|
"reward_std": 0.4414422929286957,
|
||
|
|
"rewards/reward_fn/mean": -0.20666666626930236,
|
||
|
|
"rewards/reward_fn/std": 0.44144230484962466,
|
||
|
|
"step": 615,
|
||
|
|
"step_time": 0.4910893154330552
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.8,
|
||
|
|
"completions/max_terminated_length": 22.8,
|
||
|
|
"completions/mean_length": 14.35,
|
||
|
|
"completions/mean_terminated_length": 14.35,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8949583232402801,
|
||
|
|
"epoch": 0.6391752577319587,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 4.303964630315866e-07,
|
||
|
|
"loss": -0.06437566876411438,
|
||
|
|
"num_tokens": 159926.0,
|
||
|
|
"reward": -0.5116666674613952,
|
||
|
|
"reward_std": 0.21679993867874145,
|
||
|
|
"rewards/reward_fn/mean": -0.5116666674613952,
|
||
|
|
"rewards/reward_fn/std": 0.21679995059967042,
|
||
|
|
"step": 620,
|
||
|
|
"step_time": 0.6933138659223914
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 12.0,
|
||
|
|
"completions/mean_terminated_length": 12.0,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.2098346531391144,
|
||
|
|
"epoch": 0.6443298969072165,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 4.288320508910057e-07,
|
||
|
|
"loss": 0.07893999814987182,
|
||
|
|
"num_tokens": 161206.0,
|
||
|
|
"reward": -0.4925000011920929,
|
||
|
|
"reward_std": 0.36073286533355714,
|
||
|
|
"rewards/reward_fn/mean": -0.4925000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.36073286533355714,
|
||
|
|
"step": 625,
|
||
|
|
"step_time": 0.6323956601321697
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 10.2,
|
||
|
|
"completions/mean_terminated_length": 10.2,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.48069052398204803,
|
||
|
|
"epoch": 0.6494845360824743,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.272531645792876e-07,
|
||
|
|
"loss": 0.08115171790122985,
|
||
|
|
"num_tokens": 162410.0,
|
||
|
|
"reward": -0.31124999821186067,
|
||
|
|
"reward_std": 0.4536625385284424,
|
||
|
|
"rewards/reward_fn/mean": -0.31124999821186067,
|
||
|
|
"rewards/reward_fn/std": 0.4536625385284424,
|
||
|
|
"step": 630,
|
||
|
|
"step_time": 0.48424859074875715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.0,
|
||
|
|
"completions/max_terminated_length": 17.0,
|
||
|
|
"completions/mean_length": 10.35,
|
||
|
|
"completions/mean_terminated_length": 10.35,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8250537186861038,
|
||
|
|
"epoch": 0.654639175257732,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 14.125,
|
||
|
|
"learning_rate": 4.256599318871193e-07,
|
||
|
|
"loss": 0.17227303981781006,
|
||
|
|
"num_tokens": 163649.0,
|
||
|
|
"reward": -0.3962500035762787,
|
||
|
|
"reward_std": 0.3790696501731873,
|
||
|
|
"rewards/reward_fn/mean": -0.3962500035762787,
|
||
|
|
"rewards/reward_fn/std": 0.37906967401504515,
|
||
|
|
"step": 635,
|
||
|
|
"step_time": 0.5772591234184802
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.8,
|
||
|
|
"completions/max_terminated_length": 20.8,
|
||
|
|
"completions/mean_length": 14.5,
|
||
|
|
"completions/mean_terminated_length": 14.5,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.912872189283371,
|
||
|
|
"epoch": 0.6597938144329897,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.125,
|
||
|
|
"learning_rate": 4.240524817663445e-07,
|
||
|
|
"loss": 0.10593240261077881,
|
||
|
|
"num_tokens": 164959.0,
|
||
|
|
"reward": -0.5704999923706054,
|
||
|
|
"reward_std": 0.4167350232601166,
|
||
|
|
"rewards/reward_fn/mean": -0.5704999923706054,
|
||
|
|
"rewards/reward_fn/std": 0.4167350232601166,
|
||
|
|
"step": 640,
|
||
|
|
"step_time": 0.7264971459284425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.8,
|
||
|
|
"completions/max_terminated_length": 20.8,
|
||
|
|
"completions/mean_length": 15.1,
|
||
|
|
"completions/mean_terminated_length": 15.1,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.776880145072937,
|
||
|
|
"epoch": 0.6649484536082474,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 7.90625,
|
||
|
|
"learning_rate": 4.2243094431952607e-07,
|
||
|
|
"loss": 0.006071268022060395,
|
||
|
|
"num_tokens": 166273.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.2300000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2300000011920929,
|
||
|
|
"step": 645,
|
||
|
|
"step_time": 0.6461667789146304
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.6,
|
||
|
|
"completions/max_terminated_length": 19.6,
|
||
|
|
"completions/mean_length": 12.25,
|
||
|
|
"completions/mean_terminated_length": 12.25,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.8619225978851318,
|
||
|
|
"epoch": 0.6701030927835051,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 11.375,
|
||
|
|
"learning_rate": 4.2079545078941593e-07,
|
||
|
|
"loss": 0.007808300852775574,
|
||
|
|
"num_tokens": 167550.0,
|
||
|
|
"reward": -0.32999999523162843,
|
||
|
|
"reward_std": 0.07333333492279052,
|
||
|
|
"rewards/reward_fn/mean": -0.32999999523162843,
|
||
|
|
"rewards/reward_fn/std": 0.07333333492279052,
|
||
|
|
"step": 650,
|
||
|
|
"step_time": 0.6256520722061396
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.6,
|
||
|
|
"completions/max_terminated_length": 21.6,
|
||
|
|
"completions/mean_length": 12.75,
|
||
|
|
"completions/mean_terminated_length": 12.75,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.7383227556943893,
|
||
|
|
"epoch": 0.6752577319587629,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 11.75,
|
||
|
|
"learning_rate": 4.1914613354833297e-07,
|
||
|
|
"loss": 0.0942760944366455,
|
||
|
|
"num_tokens": 168825.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.16547005176544188,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.16547005176544188,
|
||
|
|
"step": 655,
|
||
|
|
"step_time": 0.6979141506366432
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.6,
|
||
|
|
"completions/max_terminated_length": 11.6,
|
||
|
|
"completions/mean_length": 8.5,
|
||
|
|
"completions/mean_terminated_length": 8.5,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.7141625940799713,
|
||
|
|
"epoch": 0.6804123711340206,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.1748312608744886e-07,
|
||
|
|
"loss": 0.08471410274505616,
|
||
|
|
"num_tokens": 170023.0,
|
||
|
|
"reward": -0.5,
|
||
|
|
"reward_std": 0.25773502588272096,
|
||
|
|
"rewards/reward_fn/mean": -0.5,
|
||
|
|
"rewards/reward_fn/std": 0.25773502588272096,
|
||
|
|
"step": 660,
|
||
|
|
"step_time": 0.4793179755099118
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 48.2,
|
||
|
|
"completions/max_terminated_length": 48.2,
|
||
|
|
"completions/mean_length": 17.95,
|
||
|
|
"completions/mean_terminated_length": 17.95,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.6713347196578979,
|
||
|
|
"epoch": 0.6855670103092784,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 6.625,
|
||
|
|
"learning_rate": 4.158065630059837e-07,
|
||
|
|
"loss": 0.25584611892700193,
|
||
|
|
"num_tokens": 171406.0,
|
||
|
|
"reward": -0.38000000417232516,
|
||
|
|
"reward_std": 0.418888658285141,
|
||
|
|
"rewards/reward_fn/mean": -0.38000000417232516,
|
||
|
|
"rewards/reward_fn/std": 0.418888658285141,
|
||
|
|
"step": 665,
|
||
|
|
"step_time": 1.1557064381428064
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 11.4,
|
||
|
|
"completions/mean_terminated_length": 11.4,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.8641545414924622,
|
||
|
|
"epoch": 0.6907216494845361,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 35.0,
|
||
|
|
"learning_rate": 4.1411658000031217e-07,
|
||
|
|
"loss": 0.11413931846618652,
|
||
|
|
"num_tokens": 172646.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.40320507884025575,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.40320507884025575,
|
||
|
|
"step": 670,
|
||
|
|
"step_time": 0.63343042479828
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.2,
|
||
|
|
"completions/max_terminated_length": 15.2,
|
||
|
|
"completions/mean_length": 10.15,
|
||
|
|
"completions/mean_terminated_length": 10.15,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8342425405979157,
|
||
|
|
"epoch": 0.6958762886597938,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 19.875,
|
||
|
|
"learning_rate": 4.124133138529803e-07,
|
||
|
|
"loss": 0.1467257022857666,
|
||
|
|
"num_tokens": 173877.0,
|
||
|
|
"reward": -0.38166667222976686,
|
||
|
|
"reward_std": 0.3997236371040344,
|
||
|
|
"rewards/reward_fn/mean": -0.38166667222976686,
|
||
|
|
"rewards/reward_fn/std": 0.3997236490249634,
|
||
|
|
"step": 675,
|
||
|
|
"step_time": 0.5436047066934406
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.0,
|
||
|
|
"completions/max_terminated_length": 17.0,
|
||
|
|
"completions/mean_length": 11.75,
|
||
|
|
"completions/mean_terminated_length": 11.75,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 1.0363103717565536,
|
||
|
|
"epoch": 0.7010309278350515,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 27.125,
|
||
|
|
"learning_rate": 4.106969024216348e-07,
|
||
|
|
"loss": 0.08919697999954224,
|
||
|
|
"num_tokens": 175136.0,
|
||
|
|
"reward": -0.27999999523162844,
|
||
|
|
"reward_std": 0.32804334759712217,
|
||
|
|
"rewards/reward_fn/mean": -0.27999999523162844,
|
||
|
|
"rewards/reward_fn/std": 0.32804334759712217,
|
||
|
|
"step": 680,
|
||
|
|
"step_time": 0.5779976899735629
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 32.4,
|
||
|
|
"completions/max_terminated_length": 32.4,
|
||
|
|
"completions/mean_length": 14.95,
|
||
|
|
"completions/mean_terminated_length": 14.95,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.9448502540588379,
|
||
|
|
"epoch": 0.7061855670103093,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 4.089674846278656e-07,
|
||
|
|
"loss": 0.04907660484313965,
|
||
|
|
"num_tokens": 176463.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.35,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.35,
|
||
|
|
"step": 685,
|
||
|
|
"step_time": 0.8561360297724605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.4,
|
||
|
|
"completions/max_terminated_length": 18.4,
|
||
|
|
"completions/mean_length": 11.05,
|
||
|
|
"completions/mean_terminated_length": 11.05,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.0519244968891144,
|
||
|
|
"epoch": 0.711340206185567,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 29.75,
|
||
|
|
"learning_rate": 4.072252004459611e-07,
|
||
|
|
"loss": 0.20497164726257325,
|
||
|
|
"num_tokens": 177712.0,
|
||
|
|
"reward": -0.39749999940395353,
|
||
|
|
"reward_std": 0.34408590793609617,
|
||
|
|
"rewards/reward_fn/mean": -0.39749999940395353,
|
||
|
|
"rewards/reward_fn/std": 0.34408590793609617,
|
||
|
|
"step": 690,
|
||
|
|
"step_time": 0.5954566929489374
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.8,
|
||
|
|
"completions/max_terminated_length": 18.8,
|
||
|
|
"completions/mean_length": 13.7,
|
||
|
|
"completions/mean_terminated_length": 13.7,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.8845273166894912,
|
||
|
|
"epoch": 0.7164948453608248,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.8125,
|
||
|
|
"learning_rate": 4.0547019089158004e-07,
|
||
|
|
"loss": 0.05732879638671875,
|
||
|
|
"num_tokens": 179006.0,
|
||
|
|
"reward": -0.5483333319425583,
|
||
|
|
"reward_std": 0.2610683560371399,
|
||
|
|
"rewards/reward_fn/mean": -0.5483333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.26106836795806887,
|
||
|
|
"step": 695,
|
||
|
|
"step_time": 0.6034244776703417
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.8,
|
||
|
|
"completions/max_terminated_length": 21.8,
|
||
|
|
"completions/mean_length": 12.25,
|
||
|
|
"completions/mean_terminated_length": 12.25,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.8579754889011383,
|
||
|
|
"epoch": 0.7216494845360825,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.3125,
|
||
|
|
"learning_rate": 4.037025980103372e-07,
|
||
|
|
"loss": 0.15266115665435792,
|
||
|
|
"num_tokens": 180275.0,
|
||
|
|
"reward": -0.5233333319425583,
|
||
|
|
"reward_std": 0.25333333015441895,
|
||
|
|
"rewards/reward_fn/mean": -0.5233333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.2533333420753479,
|
||
|
|
"step": 700,
|
||
|
|
"step_time": 0.657803092431277
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.0,
|
||
|
|
"completions/max_terminated_length": 15.0,
|
||
|
|
"completions/mean_length": 9.0,
|
||
|
|
"completions/mean_terminated_length": 9.0,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.6864415079355239,
|
||
|
|
"epoch": 0.7268041237113402,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 38.0,
|
||
|
|
"learning_rate": 4.019225648663072e-07,
|
||
|
|
"loss": 0.16153377294540405,
|
||
|
|
"num_tokens": 181455.0,
|
||
|
|
"reward": -0.29500000476837157,
|
||
|
|
"reward_std": 0.3967333257198334,
|
||
|
|
"rewards/reward_fn/mean": -0.29500000476837157,
|
||
|
|
"rewards/reward_fn/std": 0.3967333257198334,
|
||
|
|
"step": 705,
|
||
|
|
"step_time": 0.5445711076259613
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 8.95,
|
||
|
|
"completions/mean_terminated_length": 8.95,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8134978413581848,
|
||
|
|
"epoch": 0.7319587628865979,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 21.25,
|
||
|
|
"learning_rate": 4.0013023553044504e-07,
|
||
|
|
"loss": 0.09724601507186889,
|
||
|
|
"num_tokens": 182658.0,
|
||
|
|
"reward": -0.375,
|
||
|
|
"reward_std": 0.506155276298523,
|
||
|
|
"rewards/reward_fn/mean": -0.375,
|
||
|
|
"rewards/reward_fn/std": 0.506155276298523,
|
||
|
|
"step": 710,
|
||
|
|
"step_time": 0.5672263609245419
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.6,
|
||
|
|
"completions/max_terminated_length": 20.6,
|
||
|
|
"completions/mean_length": 12.95,
|
||
|
|
"completions/mean_terminated_length": 12.95,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 1.1158703505992889,
|
||
|
|
"epoch": 0.7371134020618557,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.983257550689255e-07,
|
||
|
|
"loss": 0.02380000948905945,
|
||
|
|
"num_tokens": 183929.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.05,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.05,
|
||
|
|
"step": 715,
|
||
|
|
"step_time": 0.6364484421908856
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.8,
|
||
|
|
"completions/max_terminated_length": 13.8,
|
||
|
|
"completions/mean_length": 8.25,
|
||
|
|
"completions/mean_terminated_length": 8.25,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.8417123228311538,
|
||
|
|
"epoch": 0.7422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.9650926953140173e-07,
|
||
|
|
"loss": 0.08947134613990784,
|
||
|
|
"num_tokens": 185134.0,
|
||
|
|
"reward": -0.5199999988079071,
|
||
|
|
"reward_std": 0.20784609913825988,
|
||
|
|
"rewards/reward_fn/mean": -0.5199999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.20784609913825988,
|
||
|
|
"step": 720,
|
||
|
|
"step_time": 0.51900653578341
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 9.6,
|
||
|
|
"completions/max_terminated_length": 9.6,
|
||
|
|
"completions/mean_length": 7.85,
|
||
|
|
"completions/mean_terminated_length": 7.85,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.868690624833107,
|
||
|
|
"epoch": 0.7474226804123711,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.946809259391846e-07,
|
||
|
|
"loss": 0.018198567628860473,
|
||
|
|
"num_tokens": 186303.0,
|
||
|
|
"reward": -0.20000000894069672,
|
||
|
|
"reward_std": 0.3599011540412903,
|
||
|
|
"rewards/reward_fn/mean": -0.20000000894069672,
|
||
|
|
"rewards/reward_fn/std": 0.3599011540412903,
|
||
|
|
"step": 725,
|
||
|
|
"step_time": 0.4509535185061395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 8.8,
|
||
|
|
"completions/mean_terminated_length": 8.8,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.7654918372631073,
|
||
|
|
"epoch": 0.7525773195876289,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 23.25,
|
||
|
|
"learning_rate": 3.9284087227334283e-07,
|
||
|
|
"loss": 0.19318230152130128,
|
||
|
|
"num_tokens": 187483.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.2154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.2154700517654419,
|
||
|
|
"step": 730,
|
||
|
|
"step_time": 0.48818655135110023
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.6,
|
||
|
|
"completions/max_terminated_length": 13.6,
|
||
|
|
"completions/mean_length": 8.6,
|
||
|
|
"completions/mean_terminated_length": 8.6,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.9335297107696533,
|
||
|
|
"epoch": 0.7577319587628866,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 30.375,
|
||
|
|
"learning_rate": 3.909892574627266e-07,
|
||
|
|
"loss": 0.11663612127304077,
|
||
|
|
"num_tokens": 188679.0,
|
||
|
|
"reward": -0.37750000357627866,
|
||
|
|
"reward_std": 0.2149999976158142,
|
||
|
|
"rewards/reward_fn/mean": -0.37750000357627866,
|
||
|
|
"rewards/reward_fn/std": 0.2149999976158142,
|
||
|
|
"step": 735,
|
||
|
|
"step_time": 0.5085009665228426
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 8.8,
|
||
|
|
"completions/max_terminated_length": 8.8,
|
||
|
|
"completions/mean_length": 6.35,
|
||
|
|
"completions/mean_terminated_length": 6.35,
|
||
|
|
"completions/min_length": 4.4,
|
||
|
|
"completions/min_terminated_length": 4.4,
|
||
|
|
"entropy": 0.6853881716728211,
|
||
|
|
"epoch": 0.7628865979381443,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 53.5,
|
||
|
|
"learning_rate": 3.891262313719127e-07,
|
||
|
|
"loss": 0.11004903316497802,
|
||
|
|
"num_tokens": 189822.0,
|
||
|
|
"reward": 0.03999999761581421,
|
||
|
|
"reward_std": 0.45320507884025574,
|
||
|
|
"rewards/reward_fn/mean": 0.03999999761581421,
|
||
|
|
"rewards/reward_fn/std": 0.45320507884025574,
|
||
|
|
"step": 740,
|
||
|
|
"step_time": 0.42834897292777896
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 37.6,
|
||
|
|
"completions/max_terminated_length": 37.6,
|
||
|
|
"completions/mean_length": 16.5,
|
||
|
|
"completions/mean_terminated_length": 16.5,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.8831792533397674,
|
||
|
|
"epoch": 0.7680412371134021,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.375,
|
||
|
|
"learning_rate": 3.872519447890755e-07,
|
||
|
|
"loss": 0.2623473644256592,
|
||
|
|
"num_tokens": 191184.0,
|
||
|
|
"reward": -0.4025000035762787,
|
||
|
|
"reward_std": 0.2649999976158142,
|
||
|
|
"rewards/reward_fn/mean": -0.4025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.2649999976158142,
|
||
|
|
"step": 745,
|
||
|
|
"step_time": 1.0432677429169417
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.4,
|
||
|
|
"completions/max_terminated_length": 20.4,
|
||
|
|
"completions/mean_length": 13.25,
|
||
|
|
"completions/mean_terminated_length": 13.25,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.7962756127119064,
|
||
|
|
"epoch": 0.7731958762886598,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 21.875,
|
||
|
|
"learning_rate": 3.8536654941378247e-07,
|
||
|
|
"loss": 0.1507752299308777,
|
||
|
|
"num_tokens": 192497.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.34826061725616453,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.34826061725616453,
|
||
|
|
"step": 750,
|
||
|
|
"step_time": 0.6385473405010998
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 48.6,
|
||
|
|
"completions/max_terminated_length": 48.6,
|
||
|
|
"completions/mean_length": 20.95,
|
||
|
|
"completions/mean_terminated_length": 20.95,
|
||
|
|
"completions/min_length": 10.4,
|
||
|
|
"completions/min_terminated_length": 10.4,
|
||
|
|
"entropy": 1.0322799324989318,
|
||
|
|
"epoch": 0.7783505154639175,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 48.0,
|
||
|
|
"learning_rate": 3.834701978447159e-07,
|
||
|
|
"loss": 0.3006690502166748,
|
||
|
|
"num_tokens": 193956.0,
|
||
|
|
"reward": -0.41600000858306885,
|
||
|
|
"reward_std": 0.48129103779792787,
|
||
|
|
"rewards/reward_fn/mean": -0.41600000858306885,
|
||
|
|
"rewards/reward_fn/std": 0.48129103779792787,
|
||
|
|
"step": 755,
|
||
|
|
"step_time": 1.164021307323128
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 46.8,
|
||
|
|
"completions/max_terminated_length": 46.8,
|
||
|
|
"completions/mean_length": 21.65,
|
||
|
|
"completions/mean_terminated_length": 21.65,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.7505376458168029,
|
||
|
|
"epoch": 0.7835051546391752,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 6.6875,
|
||
|
|
"learning_rate": 3.8156304356732203e-07,
|
||
|
|
"loss": 0.18233208656311034,
|
||
|
|
"num_tokens": 195397.0,
|
||
|
|
"reward": -0.6100000023841858,
|
||
|
|
"reward_std": 0.2800000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.6100000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2800000011920929,
|
||
|
|
"step": 760,
|
||
|
|
"step_time": 1.1370311999693512
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.6,
|
||
|
|
"completions/max_terminated_length": 24.6,
|
||
|
|
"completions/mean_length": 14.5,
|
||
|
|
"completions/mean_terminated_length": 14.5,
|
||
|
|
"completions/min_length": 9.6,
|
||
|
|
"completions/min_terminated_length": 9.6,
|
||
|
|
"entropy": 0.5627210974693299,
|
||
|
|
"epoch": 0.788659793814433,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 9.875,
|
||
|
|
"learning_rate": 3.796452409413887e-07,
|
||
|
|
"loss": -0.004269063472747803,
|
||
|
|
"num_tokens": 196715.0,
|
||
|
|
"reward": -0.1147499993443489,
|
||
|
|
"reward_std": 0.5147351384162903,
|
||
|
|
"rewards/reward_fn/mean": -0.1147499993443489,
|
||
|
|
"rewards/reward_fn/std": 0.5147351384162903,
|
||
|
|
"step": 765,
|
||
|
|
"step_time": 0.716858146712184
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 9.6,
|
||
|
|
"completions/max_terminated_length": 9.6,
|
||
|
|
"completions/mean_length": 6.95,
|
||
|
|
"completions/mean_terminated_length": 6.95,
|
||
|
|
"completions/min_length": 5.0,
|
||
|
|
"completions/min_terminated_length": 5.0,
|
||
|
|
"entropy": 1.1454240024089812,
|
||
|
|
"epoch": 0.7938144329896907,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 22.375,
|
||
|
|
"learning_rate": 3.7771694518855113e-07,
|
||
|
|
"loss": 0.15136839151382447,
|
||
|
|
"num_tokens": 197870.0,
|
||
|
|
"reward": -0.09916666746139527,
|
||
|
|
"reward_std": 0.47160890102386477,
|
||
|
|
"rewards/reward_fn/mean": -0.09916666746139527,
|
||
|
|
"rewards/reward_fn/std": 0.47160890102386477,
|
||
|
|
"step": 770,
|
||
|
|
"step_time": 0.4394523199647665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.4,
|
||
|
|
"completions/max_terminated_length": 21.4,
|
||
|
|
"completions/mean_length": 12.8,
|
||
|
|
"completions/mean_terminated_length": 12.8,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.9384787321090698,
|
||
|
|
"epoch": 0.7989690721649485,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 25.875,
|
||
|
|
"learning_rate": 3.7577831237972965e-07,
|
||
|
|
"loss": 0.1645453691482544,
|
||
|
|
"num_tokens": 199170.0,
|
||
|
|
"reward": -0.15049999952316284,
|
||
|
|
"reward_std": 0.3982495129108429,
|
||
|
|
"rewards/reward_fn/mean": -0.15049999952316284,
|
||
|
|
"rewards/reward_fn/std": 0.3982495129108429,
|
||
|
|
"step": 775,
|
||
|
|
"step_time": 0.6595026399940253
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 35.4,
|
||
|
|
"completions/max_terminated_length": 35.4,
|
||
|
|
"completions/mean_length": 17.7,
|
||
|
|
"completions/mean_terminated_length": 17.7,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.908982464671135,
|
||
|
|
"epoch": 0.8041237113402062,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.738294994224969e-07,
|
||
|
|
"loss": 0.16901147365570068,
|
||
|
|
"num_tokens": 200548.0,
|
||
|
|
"reward": -0.4850000023841858,
|
||
|
|
"reward_std": 0.24826061725616455,
|
||
|
|
"rewards/reward_fn/mean": -0.4850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.24826061725616455,
|
||
|
|
"step": 780,
|
||
|
|
"step_time": 0.9208143584430217
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 14.0,
|
||
|
|
"completions/mean_terminated_length": 14.0,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.608512119948864,
|
||
|
|
"epoch": 0.8092783505154639,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 12.375,
|
||
|
|
"learning_rate": 3.7187066404837885e-07,
|
||
|
|
"loss": 0.022144103050231935,
|
||
|
|
"num_tokens": 201868.0,
|
||
|
|
"reward": -0.3925000011920929,
|
||
|
|
"reward_std": 0.1863911747932434,
|
||
|
|
"rewards/reward_fn/mean": -0.3925000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.1863911747932434,
|
||
|
|
"step": 785,
|
||
|
|
"step_time": 0.6640401553362608
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.8,
|
||
|
|
"completions/max_terminated_length": 13.8,
|
||
|
|
"completions/mean_length": 9.5,
|
||
|
|
"completions/mean_terminated_length": 9.5,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.7681404441595078,
|
||
|
|
"epoch": 0.8144329896907216,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.69901964800088e-07,
|
||
|
|
"loss": 0.12942266464233398,
|
||
|
|
"num_tokens": 203070.0,
|
||
|
|
"reward": -0.4525000035762787,
|
||
|
|
"reward_std": 0.32273502349853517,
|
||
|
|
"rewards/reward_fn/mean": -0.4525000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.32273502349853517,
|
||
|
|
"step": 790,
|
||
|
|
"step_time": 0.5212488909251988
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.0,
|
||
|
|
"completions/max_terminated_length": 13.0,
|
||
|
|
"completions/mean_length": 10.65,
|
||
|
|
"completions/mean_terminated_length": 10.65,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8925413608551025,
|
||
|
|
"epoch": 0.8195876288659794,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.625,
|
||
|
|
"learning_rate": 3.6792356101869156e-07,
|
||
|
|
"loss": 0.12305893898010253,
|
||
|
|
"num_tokens": 204299.0,
|
||
|
|
"reward": -0.2700000047683716,
|
||
|
|
"reward_std": 0.45541470050811766,
|
||
|
|
"rewards/reward_fn/mean": -0.2700000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.45541470050811766,
|
||
|
|
"step": 795,
|
||
|
|
"step_time": 0.49430387681350113
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.4,
|
||
|
|
"completions/max_terminated_length": 11.4,
|
||
|
|
"completions/mean_length": 9.6,
|
||
|
|
"completions/mean_terminated_length": 9.6,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.6558867126703263,
|
||
|
|
"epoch": 0.8247422680412371,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.125,
|
||
|
|
"learning_rate": 3.659356128307147e-07,
|
||
|
|
"loss": 0.08286584615707397,
|
||
|
|
"num_tokens": 205515.0,
|
||
|
|
"reward": -0.2750000059604645,
|
||
|
|
"reward_std": 0.2779629558324814,
|
||
|
|
"rewards/reward_fn/mean": -0.2750000059604645,
|
||
|
|
"rewards/reward_fn/std": 0.2779629558324814,
|
||
|
|
"step": 800,
|
||
|
|
"step_time": 0.4695729834027588
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.0,
|
||
|
|
"completions/max_terminated_length": 13.0,
|
||
|
|
"completions/mean_length": 10.65,
|
||
|
|
"completions/mean_terminated_length": 10.65,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.6253852218389511,
|
||
|
|
"epoch": 0.8298969072164949,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.639382811351806e-07,
|
||
|
|
"loss": 0.06376473903656006,
|
||
|
|
"num_tokens": 206756.0,
|
||
|
|
"reward": -0.4025000035762787,
|
||
|
|
"reward_std": 0.29779056310653684,
|
||
|
|
"rewards/reward_fn/mean": -0.4025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.29779056310653684,
|
||
|
|
"step": 805,
|
||
|
|
"step_time": 0.5086913261562586
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 8.75,
|
||
|
|
"completions/mean_terminated_length": 8.75,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.1803167939186097,
|
||
|
|
"epoch": 0.8350515463917526,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 3.619317275905874e-07,
|
||
|
|
"loss": 0.040542510151863095,
|
||
|
|
"num_tokens": 207911.0,
|
||
|
|
"reward": -0.20699999630451202,
|
||
|
|
"reward_std": 0.3604273974895477,
|
||
|
|
"rewards/reward_fn/mean": -0.20699999630451202,
|
||
|
|
"rewards/reward_fn/std": 0.36042741537094114,
|
||
|
|
"step": 810,
|
||
|
|
"step_time": 0.4909832508303225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.0,
|
||
|
|
"completions/max_terminated_length": 16.0,
|
||
|
|
"completions/mean_length": 12.6,
|
||
|
|
"completions/mean_terminated_length": 12.6,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.9079984992742538,
|
||
|
|
"epoch": 0.8402061855670103,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.5991611460182426e-07,
|
||
|
|
"loss": 0.014759808778762817,
|
||
|
|
"num_tokens": 209179.0,
|
||
|
|
"reward": -0.4949999988079071,
|
||
|
|
"reward_std": 0.20011106729507447,
|
||
|
|
"rewards/reward_fn/mean": -0.4949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.20011106729507447,
|
||
|
|
"step": 815,
|
||
|
|
"step_time": 0.5565207180567086
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 28.0,
|
||
|
|
"completions/max_terminated_length": 28.0,
|
||
|
|
"completions/mean_length": 14.85,
|
||
|
|
"completions/mean_terminated_length": 14.85,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.9968389987945556,
|
||
|
|
"epoch": 0.845360824742268,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 13.1875,
|
||
|
|
"learning_rate": 3.578916053070267e-07,
|
||
|
|
"loss": 0.18342533111572265,
|
||
|
|
"num_tokens": 210516.0,
|
||
|
|
"reward": -0.725,
|
||
|
|
"reward_std": 0.16547005176544188,
|
||
|
|
"rewards/reward_fn/mean": -0.725,
|
||
|
|
"rewards/reward_fn/std": 0.16547005176544188,
|
||
|
|
"step": 820,
|
||
|
|
"step_time": 0.7959272848442197
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.8,
|
||
|
|
"completions/max_terminated_length": 26.8,
|
||
|
|
"completions/mean_length": 13.5,
|
||
|
|
"completions/mean_terminated_length": 13.5,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.8428839266300201,
|
||
|
|
"epoch": 0.8505154639175257,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.558583635643726e-07,
|
||
|
|
"loss": 0.12303779125213624,
|
||
|
|
"num_tokens": 211806.0,
|
||
|
|
"reward": -0.3550000011920929,
|
||
|
|
"reward_std": 0.3120825529098511,
|
||
|
|
"rewards/reward_fn/mean": -0.3550000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3120825529098511,
|
||
|
|
"step": 825,
|
||
|
|
"step_time": 0.8458523162640631
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 44.6,
|
||
|
|
"completions/max_terminated_length": 44.6,
|
||
|
|
"completions/mean_length": 19.8,
|
||
|
|
"completions/mean_terminated_length": 19.8,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.9209296464920044,
|
||
|
|
"epoch": 0.8556701030927835,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.9375,
|
||
|
|
"learning_rate": 3.5381655393881984e-07,
|
||
|
|
"loss": 0.16743425130844117,
|
||
|
|
"num_tokens": 213238.0,
|
||
|
|
"reward": -0.6175000011920929,
|
||
|
|
"reward_std": 0.2975861310958862,
|
||
|
|
"rewards/reward_fn/mean": -0.6175000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2975861310958862,
|
||
|
|
"step": 830,
|
||
|
|
"step_time": 1.088095944840461
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 66.0,
|
||
|
|
"completions/max_terminated_length": 18.4,
|
||
|
|
"completions/mean_length": 26.3,
|
||
|
|
"completions/mean_terminated_length": 14.216666793823242,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.6402861684560776,
|
||
|
|
"epoch": 0.8608247422680413,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.0625,
|
||
|
|
"learning_rate": 3.5176634168878723e-07,
|
||
|
|
"loss": 0.15643105506896973,
|
||
|
|
"num_tokens": 214812.0,
|
||
|
|
"reward": -0.56700000166893,
|
||
|
|
"reward_std": 0.31664379835128786,
|
||
|
|
"rewards/reward_fn/mean": -0.56700000166893,
|
||
|
|
"rewards/reward_fn/std": 0.31664379835128786,
|
||
|
|
"step": 835,
|
||
|
|
"step_time": 1.4822699318639934
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.25,
|
||
|
|
"completions/mean_terminated_length": 10.25,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.566778889298439,
|
||
|
|
"epoch": 0.865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 19.125,
|
||
|
|
"learning_rate": 3.497078927527787e-07,
|
||
|
|
"loss": 0.13858178853988648,
|
||
|
|
"num_tokens": 216065.0,
|
||
|
|
"reward": -0.5950000047683716,
|
||
|
|
"reward_std": 0.3880298137664795,
|
||
|
|
"rewards/reward_fn/mean": -0.5950000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.3880298137664795,
|
||
|
|
"step": 840,
|
||
|
|
"step_time": 0.5335512371733785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.0,
|
||
|
|
"completions/max_terminated_length": 12.0,
|
||
|
|
"completions/mean_length": 8.95,
|
||
|
|
"completions/mean_terminated_length": 8.95,
|
||
|
|
"completions/min_length": 4.8,
|
||
|
|
"completions/min_terminated_length": 4.8,
|
||
|
|
"entropy": 0.7644638597965241,
|
||
|
|
"epoch": 0.8711340206185567,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.476413737359527e-07,
|
||
|
|
"loss": 0.08275651931762695,
|
||
|
|
"num_tokens": 217268.0,
|
||
|
|
"reward": -0.3950000047683716,
|
||
|
|
"reward_std": 0.29033899307250977,
|
||
|
|
"rewards/reward_fn/mean": -0.3950000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.29033900499343873,
|
||
|
|
"step": 845,
|
||
|
|
"step_time": 0.48589533697813747
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 9.95,
|
||
|
|
"completions/mean_terminated_length": 9.95,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.178800243139267,
|
||
|
|
"epoch": 0.8762886597938144,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 19.375,
|
||
|
|
"learning_rate": 3.455669518966378e-07,
|
||
|
|
"loss": 0.06842224597930908,
|
||
|
|
"num_tokens": 218483.0,
|
||
|
|
"reward": -0.4350000023841858,
|
||
|
|
"reward_std": 0.3048511028289795,
|
||
|
|
"rewards/reward_fn/mean": -0.4350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.3048511028289795,
|
||
|
|
"step": 850,
|
||
|
|
"step_time": 0.5550074013881385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 32.4,
|
||
|
|
"completions/max_terminated_length": 32.4,
|
||
|
|
"completions/mean_length": 16.55,
|
||
|
|
"completions/mean_terminated_length": 16.55,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.0851195693016051,
|
||
|
|
"epoch": 0.8814432989690721,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.4348479513279484e-07,
|
||
|
|
"loss": 0.21348931789398193,
|
||
|
|
"num_tokens": 219874.0,
|
||
|
|
"reward": -0.5600000023841858,
|
||
|
|
"reward_std": 0.25754843950271605,
|
||
|
|
"rewards/reward_fn/mean": -0.5600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.25754843950271605,
|
||
|
|
"step": 855,
|
||
|
|
"step_time": 0.8693027082830668
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 58.0,
|
||
|
|
"completions/max_terminated_length": 58.0,
|
||
|
|
"completions/mean_length": 29.15,
|
||
|
|
"completions/mean_terminated_length": 29.15,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.9453159362077713,
|
||
|
|
"epoch": 0.8865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 12.75,
|
||
|
|
"learning_rate": 3.4139507196842837e-07,
|
||
|
|
"loss": 0.165838360786438,
|
||
|
|
"num_tokens": 221461.0,
|
||
|
|
"reward": -0.5233333319425583,
|
||
|
|
"reward_std": 0.2110683560371399,
|
||
|
|
"rewards/reward_fn/mean": -0.5233333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.21106836795806885,
|
||
|
|
"step": 860,
|
||
|
|
"step_time": 1.4117650521919132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.8,
|
||
|
|
"completions/max_terminated_length": 16.8,
|
||
|
|
"completions/mean_length": 10.7,
|
||
|
|
"completions/mean_terminated_length": 10.7,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 1.284914356470108,
|
||
|
|
"epoch": 0.8917525773195877,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 29.875,
|
||
|
|
"learning_rate": 3.3929795153994624e-07,
|
||
|
|
"loss": 0.030526399612426758,
|
||
|
|
"num_tokens": 222691.0,
|
||
|
|
"reward": -0.23999999165534974,
|
||
|
|
"reward_std": 0.40157958269119265,
|
||
|
|
"rewards/reward_fn/mean": -0.23999999165534974,
|
||
|
|
"rewards/reward_fn/std": 0.4015796065330505,
|
||
|
|
"step": 865,
|
||
|
|
"step_time": 0.5726813119836152
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.2,
|
||
|
|
"completions/max_terminated_length": 21.2,
|
||
|
|
"completions/mean_length": 11.15,
|
||
|
|
"completions/mean_terminated_length": 11.15,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 1.0081340759992599,
|
||
|
|
"epoch": 0.8969072164948454,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.3719360358247053e-07,
|
||
|
|
"loss": 0.07258962392807007,
|
||
|
|
"num_tokens": 223930.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 870,
|
||
|
|
"step_time": 0.6567351558245719
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.0,
|
||
|
|
"completions/max_terminated_length": 21.0,
|
||
|
|
"completions/mean_length": 13.0,
|
||
|
|
"completions/mean_terminated_length": 13.0,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9066625654697418,
|
||
|
|
"epoch": 0.9020618556701031,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.350821984160994e-07,
|
||
|
|
"loss": 0.12728797197341918,
|
||
|
|
"num_tokens": 225222.0,
|
||
|
|
"reward": -0.425,
|
||
|
|
"reward_std": 0.2809401035308838,
|
||
|
|
"rewards/reward_fn/mean": -0.425,
|
||
|
|
"rewards/reward_fn/std": 0.2809401035308838,
|
||
|
|
"step": 875,
|
||
|
|
"step_time": 0.6553768229670822
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.0,
|
||
|
|
"completions/max_terminated_length": 12.0,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.8609405964612961,
|
||
|
|
"epoch": 0.9072164948453608,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 3.329639069321221e-07,
|
||
|
|
"loss": 0.1672358274459839,
|
||
|
|
"num_tokens": 226454.0,
|
||
|
|
"reward": -0.36833333373069765,
|
||
|
|
"reward_std": 0.5804607629776001,
|
||
|
|
"rewards/reward_fn/mean": -0.36833333373069765,
|
||
|
|
"rewards/reward_fn/std": 0.5804607629776001,
|
||
|
|
"step": 880,
|
||
|
|
"step_time": 0.48266850765794517
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 10.6,
|
||
|
|
"completions/mean_terminated_length": 10.6,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 1.1898846477270126,
|
||
|
|
"epoch": 0.9123711340206185,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 3.308389005791871e-07,
|
||
|
|
"loss": 0.12706210613250732,
|
||
|
|
"num_tokens": 227674.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 885,
|
||
|
|
"step_time": 0.5870971732772887
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.2,
|
||
|
|
"completions/max_terminated_length": 11.2,
|
||
|
|
"completions/mean_length": 8.5,
|
||
|
|
"completions/mean_terminated_length": 8.5,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0536468535661698,
|
||
|
|
"epoch": 0.9175257731958762,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.287073513494261e-07,
|
||
|
|
"loss": 0.0786515474319458,
|
||
|
|
"num_tokens": 228868.0,
|
||
|
|
"reward": -0.4025000035762787,
|
||
|
|
"reward_std": 0.22273502349853516,
|
||
|
|
"rewards/reward_fn/mean": -0.4025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.22273502349853516,
|
||
|
|
"step": 890,
|
||
|
|
"step_time": 0.46912989038974046
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.0,
|
||
|
|
"completions/max_terminated_length": 22.0,
|
||
|
|
"completions/mean_length": 12.05,
|
||
|
|
"completions/mean_terminated_length": 12.05,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.8401788771152496,
|
||
|
|
"epoch": 0.9226804123711341,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 21.125,
|
||
|
|
"learning_rate": 3.265694317645328e-07,
|
||
|
|
"loss": 0.06491990685462952,
|
||
|
|
"num_tokens": 230125.0,
|
||
|
|
"reward": -0.4433333337306976,
|
||
|
|
"reward_std": 0.37993608713150023,
|
||
|
|
"rewards/reward_fn/mean": -0.4433333337306976,
|
||
|
|
"rewards/reward_fn/std": 0.37993608713150023,
|
||
|
|
"step": 895,
|
||
|
|
"step_time": 0.730783850606531
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 44.4,
|
||
|
|
"completions/max_terminated_length": 44.4,
|
||
|
|
"completions/mean_length": 22.75,
|
||
|
|
"completions/mean_terminated_length": 22.75,
|
||
|
|
"completions/min_length": 12.4,
|
||
|
|
"completions/min_terminated_length": 12.4,
|
||
|
|
"entropy": 0.8007380843162537,
|
||
|
|
"epoch": 0.9278350515463918,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 11.875,
|
||
|
|
"learning_rate": 3.244253148618001e-07,
|
||
|
|
"loss": 0.05340578556060791,
|
||
|
|
"num_tokens": 231620.0,
|
||
|
|
"reward": -0.4360606074333191,
|
||
|
|
"reward_std": 0.30347265005111695,
|
||
|
|
"rewards/reward_fn/mean": -0.4360606074333191,
|
||
|
|
"rewards/reward_fn/std": 0.3034726619720459,
|
||
|
|
"step": 900,
|
||
|
|
"step_time": 1.0834071793593467
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 67.4,
|
||
|
|
"completions/max_terminated_length": 21.6,
|
||
|
|
"completions/mean_length": 24.3,
|
||
|
|
"completions/mean_terminated_length": 12.25,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.9434569358825684,
|
||
|
|
"epoch": 0.9329896907216495,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 10.3125,
|
||
|
|
"learning_rate": 3.222751741801146e-07,
|
||
|
|
"loss": 0.07329256534576416,
|
||
|
|
"num_tokens": 233102.0,
|
||
|
|
"reward": -0.5100000023841857,
|
||
|
|
"reward_std": 0.1800000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.5100000023841857,
|
||
|
|
"rewards/reward_fn/std": 0.1800000011920929,
|
||
|
|
"step": 905,
|
||
|
|
"step_time": 1.5177901348099112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 12.05,
|
||
|
|
"completions/mean_terminated_length": 12.05,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8976251482963562,
|
||
|
|
"epoch": 0.9381443298969072,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.201191837459107e-07,
|
||
|
|
"loss": 0.02294694781303406,
|
||
|
|
"num_tokens": 234375.0,
|
||
|
|
"reward": -0.4889999985694885,
|
||
|
|
"reward_std": 0.1797350287437439,
|
||
|
|
"rewards/reward_fn/mean": -0.4889999985694885,
|
||
|
|
"rewards/reward_fn/std": 0.17973504066467286,
|
||
|
|
"step": 910,
|
||
|
|
"step_time": 0.5585914126597344
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.4,
|
||
|
|
"completions/max_terminated_length": 24.4,
|
||
|
|
"completions/mean_length": 14.5,
|
||
|
|
"completions/mean_terminated_length": 14.5,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 1.120745688676834,
|
||
|
|
"epoch": 0.9432989690721649,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.179575180590857e-07,
|
||
|
|
"loss": -0.041665342450141904,
|
||
|
|
"num_tokens": 235701.0,
|
||
|
|
"reward": -0.3719090908765793,
|
||
|
|
"reward_std": 0.26021685600280764,
|
||
|
|
"rewards/reward_fn/mean": -0.3719090908765793,
|
||
|
|
"rewards/reward_fn/std": 0.26021685600280764,
|
||
|
|
"step": 915,
|
||
|
|
"step_time": 0.7208410411141812
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 31.2,
|
||
|
|
"completions/max_terminated_length": 31.2,
|
||
|
|
"completions/mean_length": 18.15,
|
||
|
|
"completions/mean_terminated_length": 18.15,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.7649198234081268,
|
||
|
|
"epoch": 0.9484536082474226,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 15.9375,
|
||
|
|
"learning_rate": 3.1579035207887624e-07,
|
||
|
|
"loss": 0.09516732692718506,
|
||
|
|
"num_tokens": 237104.0,
|
||
|
|
"reward": -0.446000000834465,
|
||
|
|
"reward_std": 0.33894009590148927,
|
||
|
|
"rewards/reward_fn/mean": -0.446000000834465,
|
||
|
|
"rewards/reward_fn/std": 0.33894009590148927,
|
||
|
|
"step": 920,
|
||
|
|
"step_time": 0.8414601643569768
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.2,
|
||
|
|
"completions/max_terminated_length": 24.2,
|
||
|
|
"completions/mean_length": 12.7,
|
||
|
|
"completions/mean_terminated_length": 12.7,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.7877166599035264,
|
||
|
|
"epoch": 0.9536082474226805,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.136178612096973e-07,
|
||
|
|
"loss": 0.04098041653633118,
|
||
|
|
"num_tokens": 238386.0,
|
||
|
|
"reward": -0.3866666674613953,
|
||
|
|
"reward_std": 0.21086571216583253,
|
||
|
|
"rewards/reward_fn/mean": -0.3866666674613953,
|
||
|
|
"rewards/reward_fn/std": 0.21086571216583253,
|
||
|
|
"step": 925,
|
||
|
|
"step_time": 0.712461163289845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.2,
|
||
|
|
"completions/max_terminated_length": 17.2,
|
||
|
|
"completions/mean_length": 12.55,
|
||
|
|
"completions/mean_terminated_length": 12.55,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.8109033226966857,
|
||
|
|
"epoch": 0.9587628865979382,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 15.25,
|
||
|
|
"learning_rate": 3.114402212869458e-07,
|
||
|
|
"loss": 0.03635913133621216,
|
||
|
|
"num_tokens": 239649.0,
|
||
|
|
"reward": -0.30333333015441893,
|
||
|
|
"reward_std": 0.5505508422851563,
|
||
|
|
"rewards/reward_fn/mean": -0.30333333015441893,
|
||
|
|
"rewards/reward_fn/std": 0.5505508542060852,
|
||
|
|
"step": 930,
|
||
|
|
"step_time": 0.585801807884127
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.0,
|
||
|
|
"completions/max_terminated_length": 12.0,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.7167331606149674,
|
||
|
|
"epoch": 0.9639175257731959,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 21.5,
|
||
|
|
"learning_rate": 3.092576085627686e-07,
|
||
|
|
"loss": 0.04640328586101532,
|
||
|
|
"num_tokens": 240881.0,
|
||
|
|
"reward": -0.4983333319425583,
|
||
|
|
"reward_std": 0.1610683560371399,
|
||
|
|
"rewards/reward_fn/mean": -0.4983333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.16106836795806884,
|
||
|
|
"step": 935,
|
||
|
|
"step_time": 0.4838869537226856
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.2,
|
||
|
|
"completions/mean_terminated_length": 10.2,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.8657872408628464,
|
||
|
|
"epoch": 0.9690721649484536,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.070701996917976e-07,
|
||
|
|
"loss": 0.11601418256759644,
|
||
|
|
"num_tokens": 242105.0,
|
||
|
|
"reward": -0.0800000011920929,
|
||
|
|
"reward_std": 0.5176253199577332,
|
||
|
|
"rewards/reward_fn/mean": -0.0800000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.5176253080368042,
|
||
|
|
"step": 940,
|
||
|
|
"step_time": 0.5930247722193599
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 13.4,
|
||
|
|
"completions/mean_terminated_length": 13.4,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.8240721940994262,
|
||
|
|
"epoch": 0.9742268041237113,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 22.875,
|
||
|
|
"learning_rate": 3.048781717168513e-07,
|
||
|
|
"loss": -0.08106042742729187,
|
||
|
|
"num_tokens": 243389.0,
|
||
|
|
"reward": -0.2540909111499786,
|
||
|
|
"reward_std": 0.44955320954322814,
|
||
|
|
"rewards/reward_fn/mean": -0.2540909111499786,
|
||
|
|
"rewards/reward_fn/std": 0.44955320954322814,
|
||
|
|
"step": 945,
|
||
|
|
"step_time": 0.5971614887937904
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 34.0,
|
||
|
|
"completions/max_terminated_length": 34.0,
|
||
|
|
"completions/mean_length": 15.0,
|
||
|
|
"completions/mean_terminated_length": 15.0,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 1.3446922928094864,
|
||
|
|
"epoch": 0.979381443298969,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 24.125,
|
||
|
|
"learning_rate": 3.0268170205460566e-07,
|
||
|
|
"loss": -0.017026874423027038,
|
||
|
|
"num_tokens": 244697.0,
|
||
|
|
"reward": -0.3649999976158142,
|
||
|
|
"reward_std": 0.4780433356761932,
|
||
|
|
"rewards/reward_fn/mean": -0.3649999976158142,
|
||
|
|
"rewards/reward_fn/std": 0.4780433475971222,
|
||
|
|
"step": 950,
|
||
|
|
"step_time": 0.8890752191655338
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.4,
|
||
|
|
"completions/max_terminated_length": 13.4,
|
||
|
|
"completions/mean_length": 8.5,
|
||
|
|
"completions/mean_terminated_length": 8.5,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 1.2739114731550216,
|
||
|
|
"epoch": 0.9845360824742269,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 14.9375,
|
||
|
|
"learning_rate": 3.004809684812349e-07,
|
||
|
|
"loss": 0.057057321071624756,
|
||
|
|
"num_tokens": 245879.0,
|
||
|
|
"reward": -0.4950000047683716,
|
||
|
|
"reward_std": 0.26773502826690676,
|
||
|
|
"rewards/reward_fn/mean": -0.4950000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.26773502826690676,
|
||
|
|
"step": 955,
|
||
|
|
"step_time": 0.5191133976913989
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.2,
|
||
|
|
"completions/max_terminated_length": 26.2,
|
||
|
|
"completions/mean_length": 17.7,
|
||
|
|
"completions/mean_terminated_length": 17.7,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 1.1563934952020645,
|
||
|
|
"epoch": 0.9896907216494846,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.125,
|
||
|
|
"learning_rate": 2.98276149118022e-07,
|
||
|
|
"loss": 0.07603020668029785,
|
||
|
|
"num_tokens": 247273.0,
|
||
|
|
"reward": -0.5050000011920929,
|
||
|
|
"reward_std": 0.34321573972702024,
|
||
|
|
"rewards/reward_fn/mean": -0.5050000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.34321573972702024,
|
||
|
|
"step": 960,
|
||
|
|
"step_time": 0.764554787799716
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.0,
|
||
|
|
"completions/max_terminated_length": 12.0,
|
||
|
|
"completions/mean_length": 8.35,
|
||
|
|
"completions/mean_terminated_length": 8.35,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.2172771751880647,
|
||
|
|
"epoch": 0.9948453608247423,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.9606742241694265e-07,
|
||
|
|
"loss": 0.03891005516052246,
|
||
|
|
"num_tokens": 248456.0,
|
||
|
|
"reward": -0.4550000011920929,
|
||
|
|
"reward_std": 0.2651223957538605,
|
||
|
|
"rewards/reward_fn/mean": -0.4550000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2651223957538605,
|
||
|
|
"step": 965,
|
||
|
|
"step_time": 0.4914631086401641
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.2,
|
||
|
|
"completions/max_terminated_length": 10.2,
|
||
|
|
"completions/mean_length": 7.9,
|
||
|
|
"completions/mean_terminated_length": 7.9,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 1.2364417910575867,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.938549671462216e-07,
|
||
|
|
"loss": 0.016531860828399657,
|
||
|
|
"num_tokens": 249638.0,
|
||
|
|
"reward": -0.3433333277702332,
|
||
|
|
"reward_std": 0.1,
|
||
|
|
"rewards/reward_fn/mean": -0.3433333277702332,
|
||
|
|
"rewards/reward_fn/std": 0.1,
|
||
|
|
"step": 970,
|
||
|
|
"step_time": 0.4553267262876034
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.2,
|
||
|
|
"completions/max_terminated_length": 20.2,
|
||
|
|
"completions/mean_length": 14.65,
|
||
|
|
"completions/mean_terminated_length": 14.65,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.8596020042896271,
|
||
|
|
"epoch": 1.0051546391752577,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.6875,
|
||
|
|
"learning_rate": 2.916389623758636e-07,
|
||
|
|
"loss": 0.11736297607421875,
|
||
|
|
"num_tokens": 250955.0,
|
||
|
|
"reward": -0.22090909481048585,
|
||
|
|
"reward_std": 0.3392897605895996,
|
||
|
|
"rewards/reward_fn/mean": -0.22090909481048585,
|
||
|
|
"rewards/reward_fn/std": 0.3392897605895996,
|
||
|
|
"step": 975,
|
||
|
|
"step_time": 0.6459065237082541
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.2,
|
||
|
|
"completions/max_terminated_length": 11.2,
|
||
|
|
"completions/mean_length": 8.15,
|
||
|
|
"completions/mean_terminated_length": 8.15,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 0.8980956047773361,
|
||
|
|
"epoch": 1.0103092783505154,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 39.0,
|
||
|
|
"learning_rate": 2.8941958746316036e-07,
|
||
|
|
"loss": 0.032901537418365476,
|
||
|
|
"num_tokens": 252130.0,
|
||
|
|
"reward": -0.24749999940395356,
|
||
|
|
"reward_std": 0.3519443690776825,
|
||
|
|
"rewards/reward_fn/mean": -0.24749999940395356,
|
||
|
|
"rewards/reward_fn/std": 0.3519443690776825,
|
||
|
|
"step": 980,
|
||
|
|
"step_time": 0.47193123353645205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.6604278087615967,
|
||
|
|
"epoch": 1.0154639175257731,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.871970220381733e-07,
|
||
|
|
"loss": 0.14088039398193358,
|
||
|
|
"num_tokens": 253358.0,
|
||
|
|
"reward": -0.4949999988079071,
|
||
|
|
"reward_std": 0.42496423721313475,
|
||
|
|
"rewards/reward_fn/mean": -0.4949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.4249642491340637,
|
||
|
|
"step": 985,
|
||
|
|
"step_time": 0.49146103104576466
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 9.5,
|
||
|
|
"completions/mean_terminated_length": 9.5,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.7506533205509186,
|
||
|
|
"epoch": 1.0206185567010309,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 34.75,
|
||
|
|
"learning_rate": 2.8497144598919524e-07,
|
||
|
|
"loss": 0.2226261615753174,
|
||
|
|
"num_tokens": 254556.0,
|
||
|
|
"reward": -0.4300000071525574,
|
||
|
|
"reward_std": 0.4620167911052704,
|
||
|
|
"rewards/reward_fn/mean": -0.4300000071525574,
|
||
|
|
"rewards/reward_fn/std": 0.4620167911052704,
|
||
|
|
"step": 990,
|
||
|
|
"step_time": 0.5410918489098548
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.2,
|
||
|
|
"completions/mean_terminated_length": 10.2,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 1.0815811634063721,
|
||
|
|
"epoch": 1.0257731958762886,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 11.0625,
|
||
|
|
"learning_rate": 2.8274303944819045e-07,
|
||
|
|
"loss": 0.11785383224487304,
|
||
|
|
"num_tokens": 255784.0,
|
||
|
|
"reward": -0.19799999594688417,
|
||
|
|
"reward_std": 0.3982208132743835,
|
||
|
|
"rewards/reward_fn/mean": -0.19799999594688417,
|
||
|
|
"rewards/reward_fn/std": 0.3982208132743835,
|
||
|
|
"step": 995,
|
||
|
|
"step_time": 0.5314617343246937
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.8,
|
||
|
|
"completions/max_terminated_length": 21.8,
|
||
|
|
"completions/mean_length": 13.55,
|
||
|
|
"completions/mean_terminated_length": 13.55,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.7796996653079986,
|
||
|
|
"epoch": 1.0309278350515463,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 35.5,
|
||
|
|
"learning_rate": 2.8051198277621544e-07,
|
||
|
|
"loss": 0.11956088542938233,
|
||
|
|
"num_tokens": 257067.0,
|
||
|
|
"reward": -0.4350000023841858,
|
||
|
|
"reward_std": 0.21673332452774047,
|
||
|
|
"rewards/reward_fn/mean": -0.4350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.21673332452774047,
|
||
|
|
"step": 1000,
|
||
|
|
"step_time": 0.6741534044034779
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.6,
|
||
|
|
"completions/max_terminated_length": 27.6,
|
||
|
|
"completions/mean_length": 13.65,
|
||
|
|
"completions/mean_terminated_length": 13.65,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 1.1909677296876908,
|
||
|
|
"epoch": 1.0360824742268042,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.75,
|
||
|
|
"learning_rate": 2.782784565488211e-07,
|
||
|
|
"loss": 0.22131562232971191,
|
||
|
|
"num_tokens": 258368.0,
|
||
|
|
"reward": -0.26075000017881395,
|
||
|
|
"reward_std": 0.4857735872268677,
|
||
|
|
"rewards/reward_fn/mean": -0.26075000017881395,
|
||
|
|
"rewards/reward_fn/std": 0.48577356338500977,
|
||
|
|
"step": 1005,
|
||
|
|
"step_time": 0.8473773039877415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 44.2,
|
||
|
|
"completions/max_terminated_length": 44.2,
|
||
|
|
"completions/mean_length": 23.95,
|
||
|
|
"completions/mean_terminated_length": 23.95,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.8882090777158738,
|
||
|
|
"epoch": 1.041237113402062,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.760426415414371e-07,
|
||
|
|
"loss": 0.13278703689575194,
|
||
|
|
"num_tokens": 259863.0,
|
||
|
|
"reward": -0.6425000011920929,
|
||
|
|
"reward_std": 0.2727350234985352,
|
||
|
|
"rewards/reward_fn/mean": -0.6425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2727350234985352,
|
||
|
|
"step": 1010,
|
||
|
|
"step_time": 1.0934121519327165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.8,
|
||
|
|
"completions/max_terminated_length": 26.8,
|
||
|
|
"completions/mean_length": 15.75,
|
||
|
|
"completions/mean_terminated_length": 15.75,
|
||
|
|
"completions/min_length": 10.2,
|
||
|
|
"completions/min_terminated_length": 10.2,
|
||
|
|
"entropy": 0.9788724243640899,
|
||
|
|
"epoch": 1.0463917525773196,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.8125,
|
||
|
|
"learning_rate": 2.738047187147406e-07,
|
||
|
|
"loss": 0.035887351632118224,
|
||
|
|
"num_tokens": 261214.0,
|
||
|
|
"reward": -0.4050000011920929,
|
||
|
|
"reward_std": 0.20241495370864868,
|
||
|
|
"rewards/reward_fn/mean": -0.4050000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.20241495370864868,
|
||
|
|
"step": 1015,
|
||
|
|
"step_time": 0.7664700558409094
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.2,
|
||
|
|
"completions/max_terminated_length": 21.2,
|
||
|
|
"completions/mean_length": 13.0,
|
||
|
|
"completions/mean_terminated_length": 13.0,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.7931116044521331,
|
||
|
|
"epoch": 1.0515463917525774,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.0,
|
||
|
|
"learning_rate": 2.715648692000102e-07,
|
||
|
|
"loss": 0.1386839747428894,
|
||
|
|
"num_tokens": 262506.0,
|
||
|
|
"reward": -0.4069999992847443,
|
||
|
|
"reward_std": 0.4169401049613953,
|
||
|
|
"rewards/reward_fn/mean": -0.4069999992847443,
|
||
|
|
"rewards/reward_fn/std": 0.4169400930404663,
|
||
|
|
"step": 1020,
|
||
|
|
"step_time": 0.6620019136928021
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 38.0,
|
||
|
|
"completions/max_terminated_length": 38.0,
|
||
|
|
"completions/mean_length": 19.25,
|
||
|
|
"completions/mean_terminated_length": 19.25,
|
||
|
|
"completions/min_length": 9.8,
|
||
|
|
"completions/min_terminated_length": 9.8,
|
||
|
|
"entropy": 1.357260411977768,
|
||
|
|
"epoch": 1.056701030927835,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 9.75,
|
||
|
|
"learning_rate": 2.693232742844649e-07,
|
||
|
|
"loss": 0.09496567249298096,
|
||
|
|
"num_tokens": 263963.0,
|
||
|
|
"reward": -0.5210000038146972,
|
||
|
|
"reward_std": 0.2332816481590271,
|
||
|
|
"rewards/reward_fn/mean": -0.5210000038146972,
|
||
|
|
"rewards/reward_fn/std": 0.23328166007995604,
|
||
|
|
"step": 1025,
|
||
|
|
"step_time": 0.9738308310508728
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.8,
|
||
|
|
"completions/max_terminated_length": 22.8,
|
||
|
|
"completions/mean_length": 13.2,
|
||
|
|
"completions/mean_terminated_length": 13.2,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.9426976352930069,
|
||
|
|
"epoch": 1.0618556701030928,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.0,
|
||
|
|
"learning_rate": 2.6708011539659165e-07,
|
||
|
|
"loss": 0.1510265588760376,
|
||
|
|
"num_tokens": 265263.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 1030,
|
||
|
|
"step_time": 0.6886733044870198
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 11.3,
|
||
|
|
"completions/mean_terminated_length": 11.3,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.8660927087068557,
|
||
|
|
"epoch": 1.0670103092783505,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 42.75,
|
||
|
|
"learning_rate": 2.648355740914613e-07,
|
||
|
|
"loss": 0.15204911231994628,
|
||
|
|
"num_tokens": 266541.0,
|
||
|
|
"reward": -0.5639999985694886,
|
||
|
|
"reward_std": 0.3041954874992371,
|
||
|
|
"rewards/reward_fn/mean": -0.5639999985694886,
|
||
|
|
"rewards/reward_fn/std": 0.3041954874992371,
|
||
|
|
"step": 1035,
|
||
|
|
"step_time": 0.6211226489394903
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.4,
|
||
|
|
"completions/max_terminated_length": 16.4,
|
||
|
|
"completions/mean_length": 9.85,
|
||
|
|
"completions/mean_terminated_length": 9.85,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0684709191322326,
|
||
|
|
"epoch": 1.0721649484536082,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.6258983203603336e-07,
|
||
|
|
"loss": 0.16374725103378296,
|
||
|
|
"num_tokens": 267758.0,
|
||
|
|
"reward": -0.28499999940395354,
|
||
|
|
"reward_std": 0.44684439301490786,
|
||
|
|
"rewards/reward_fn/mean": -0.28499999940395354,
|
||
|
|
"rewards/reward_fn/std": 0.44684439301490786,
|
||
|
|
"step": 1040,
|
||
|
|
"step_time": 0.5705590627156198
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.8,
|
||
|
|
"completions/max_terminated_length": 17.8,
|
||
|
|
"completions/mean_length": 12.65,
|
||
|
|
"completions/mean_terminated_length": 12.65,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.8728094905614853,
|
||
|
|
"epoch": 1.077319587628866,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.125,
|
||
|
|
"learning_rate": 2.603430709944529e-07,
|
||
|
|
"loss": 0.08976652026176453,
|
||
|
|
"num_tokens": 269051.0,
|
||
|
|
"reward": -0.2824999988079071,
|
||
|
|
"reward_std": 0.3431844413280487,
|
||
|
|
"rewards/reward_fn/mean": -0.2824999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.3431844413280487,
|
||
|
|
"step": 1045,
|
||
|
|
"step_time": 0.600045460741967
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.8,
|
||
|
|
"completions/max_terminated_length": 13.8,
|
||
|
|
"completions/mean_length": 10.1,
|
||
|
|
"completions/mean_terminated_length": 10.1,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.7100171774625779,
|
||
|
|
"epoch": 1.0824742268041236,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.5,
|
||
|
|
"learning_rate": 2.58095472813339e-07,
|
||
|
|
"loss": 0.04074999690055847,
|
||
|
|
"num_tokens": 270285.0,
|
||
|
|
"reward": -0.4716666668653488,
|
||
|
|
"reward_std": 0.332260525226593,
|
||
|
|
"rewards/reward_fn/mean": -0.4716666668653488,
|
||
|
|
"rewards/reward_fn/std": 0.33226053714752196,
|
||
|
|
"step": 1050,
|
||
|
|
"step_time": 0.5243586430326104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 40.2,
|
||
|
|
"completions/max_terminated_length": 40.2,
|
||
|
|
"completions/mean_length": 15.9,
|
||
|
|
"completions/mean_terminated_length": 15.9,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0960396260023118,
|
||
|
|
"epoch": 1.0876288659793814,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.558472194070662e-07,
|
||
|
|
"loss": 0.06408804059028625,
|
||
|
|
"num_tokens": 271615.0,
|
||
|
|
"reward": -0.4333333373069763,
|
||
|
|
"reward_std": 0.20892719626426698,
|
||
|
|
"rewards/reward_fn/mean": -0.4333333373069763,
|
||
|
|
"rewards/reward_fn/std": 0.20892720818519592,
|
||
|
|
"step": 1055,
|
||
|
|
"step_time": 1.026824015751481
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.6,
|
||
|
|
"completions/max_terminated_length": 25.6,
|
||
|
|
"completions/mean_length": 15.3,
|
||
|
|
"completions/mean_terminated_length": 15.3,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 1.1162247240543366,
|
||
|
|
"epoch": 1.0927835051546393,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 12.375,
|
||
|
|
"learning_rate": 2.5359849274304163e-07,
|
||
|
|
"loss": -0.01492423415184021,
|
||
|
|
"num_tokens": 272933.0,
|
||
|
|
"reward": -0.7,
|
||
|
|
"reward_std": 0.2732050776481628,
|
||
|
|
"rewards/reward_fn/mean": -0.7,
|
||
|
|
"rewards/reward_fn/std": 0.2732050776481628,
|
||
|
|
"step": 1060,
|
||
|
|
"step_time": 0.7419044939801097
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.2,
|
||
|
|
"completions/max_terminated_length": 10.2,
|
||
|
|
"completions/mean_length": 6.65,
|
||
|
|
"completions/mean_terminated_length": 6.65,
|
||
|
|
"completions/min_length": 3.8,
|
||
|
|
"completions/min_terminated_length": 3.8,
|
||
|
|
"entropy": 1.1226806700229646,
|
||
|
|
"epoch": 1.097938144329897,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 31.5,
|
||
|
|
"learning_rate": 2.513494748269761e-07,
|
||
|
|
"loss": 0.20585639476776124,
|
||
|
|
"num_tokens": 274062.0,
|
||
|
|
"reward": -0.3850000023841858,
|
||
|
|
"reward_std": 0.3784202814102173,
|
||
|
|
"rewards/reward_fn/mean": -0.3850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.3784202814102173,
|
||
|
|
"step": 1065,
|
||
|
|
"step_time": 0.4867300412617624
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 9.05,
|
||
|
|
"completions/mean_terminated_length": 9.05,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.6160762578248977,
|
||
|
|
"epoch": 1.1030927835051547,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.625,
|
||
|
|
"learning_rate": 2.4910034768815383e-07,
|
||
|
|
"loss": 0.1410534143447876,
|
||
|
|
"num_tokens": 275279.0,
|
||
|
|
"reward": -0.5408333361148834,
|
||
|
|
"reward_std": 0.38424835205078123,
|
||
|
|
"rewards/reward_fn/mean": -0.5408333361148834,
|
||
|
|
"rewards/reward_fn/std": 0.3842483639717102,
|
||
|
|
"step": 1070,
|
||
|
|
"step_time": 0.48538791984319685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.2,
|
||
|
|
"completions/max_terminated_length": 13.2,
|
||
|
|
"completions/mean_length": 8.5,
|
||
|
|
"completions/mean_terminated_length": 8.5,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 1.0447974920272827,
|
||
|
|
"epoch": 1.1082474226804124,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 2.468512933646992e-07,
|
||
|
|
"loss": 0.14037505388259888,
|
||
|
|
"num_tokens": 276461.0,
|
||
|
|
"reward": -0.4675000011920929,
|
||
|
|
"reward_std": 0.36337884664535525,
|
||
|
|
"rewards/reward_fn/mean": -0.4675000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.36337885856628416,
|
||
|
|
"step": 1075,
|
||
|
|
"step_time": 0.5826695743016899
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 11.85,
|
||
|
|
"completions/mean_terminated_length": 11.85,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.7518991678953171,
|
||
|
|
"epoch": 1.1134020618556701,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 8.5,
|
||
|
|
"learning_rate": 2.4460249388884307e-07,
|
||
|
|
"loss": 0.039956438541412356,
|
||
|
|
"num_tokens": 277722.0,
|
||
|
|
"reward": -0.4816666662693024,
|
||
|
|
"reward_std": 0.3914422929286957,
|
||
|
|
"rewards/reward_fn/mean": -0.4816666662693024,
|
||
|
|
"rewards/reward_fn/std": 0.3914422929286957,
|
||
|
|
"step": 1080,
|
||
|
|
"step_time": 0.5958723069168628
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.8,
|
||
|
|
"completions/max_terminated_length": 16.8,
|
||
|
|
"completions/mean_length": 9.85,
|
||
|
|
"completions/mean_terminated_length": 9.85,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.8753577500581742,
|
||
|
|
"epoch": 1.1185567010309279,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 41.25,
|
||
|
|
"learning_rate": 2.423541312721896e-07,
|
||
|
|
"loss": 0.018834894895553587,
|
||
|
|
"num_tokens": 278939.0,
|
||
|
|
"reward": -0.34500000476837156,
|
||
|
|
"reward_std": 0.10847426652908325,
|
||
|
|
"rewards/reward_fn/mean": -0.34500000476837156,
|
||
|
|
"rewards/reward_fn/std": 0.10847426652908325,
|
||
|
|
"step": 1085,
|
||
|
|
"step_time": 0.5817077334038914
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.4,
|
||
|
|
"completions/max_terminated_length": 18.4,
|
||
|
|
"completions/mean_length": 13.3,
|
||
|
|
"completions/mean_terminated_length": 13.3,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.985750263929367,
|
||
|
|
"epoch": 1.1237113402061856,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 15.75,
|
||
|
|
"learning_rate": 2.401063874909848e-07,
|
||
|
|
"loss": 0.17092158794403076,
|
||
|
|
"num_tokens": 280241.0,
|
||
|
|
"reward": -0.5675000011920929,
|
||
|
|
"reward_std": 0.32273502349853517,
|
||
|
|
"rewards/reward_fn/mean": -0.5675000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.32273502349853517,
|
||
|
|
"step": 1090,
|
||
|
|
"step_time": 0.604114449955523
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 9.0,
|
||
|
|
"completions/max_terminated_length": 9.0,
|
||
|
|
"completions/mean_length": 7.1,
|
||
|
|
"completions/mean_terminated_length": 7.1,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.6755490630865097,
|
||
|
|
"epoch": 1.1288659793814433,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.375,
|
||
|
|
"learning_rate": 2.37859444471388e-07,
|
||
|
|
"loss": 0.07062238454818726,
|
||
|
|
"num_tokens": 281391.0,
|
||
|
|
"reward": -0.2740000009536743,
|
||
|
|
"reward_std": 0.4252050817012787,
|
||
|
|
"rewards/reward_fn/mean": -0.2740000009536743,
|
||
|
|
"rewards/reward_fn/std": 0.42520509362220765,
|
||
|
|
"step": 1095,
|
||
|
|
"step_time": 0.43289085691794754
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 38.6,
|
||
|
|
"completions/max_terminated_length": 38.6,
|
||
|
|
"completions/mean_length": 18.3,
|
||
|
|
"completions/mean_terminated_length": 18.3,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.9542642265558243,
|
||
|
|
"epoch": 1.134020618556701,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 38.5,
|
||
|
|
"learning_rate": 2.3561348407474678e-07,
|
||
|
|
"loss": 0.18230464458465576,
|
||
|
|
"num_tokens": 282765.0,
|
||
|
|
"reward": -0.5875,
|
||
|
|
"reward_std": 0.30516021251678466,
|
||
|
|
"rewards/reward_fn/mean": -0.5875,
|
||
|
|
"rewards/reward_fn/std": 0.30516021251678466,
|
||
|
|
"step": 1100,
|
||
|
|
"step_time": 0.9872395430691541
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 13.65,
|
||
|
|
"completions/mean_terminated_length": 13.65,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 1.1501558601856232,
|
||
|
|
"epoch": 1.1391752577319587,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.333686880828784e-07,
|
||
|
|
"loss": 0.01851527690887451,
|
||
|
|
"num_tokens": 284058.0,
|
||
|
|
"reward": -0.08750000596046448,
|
||
|
|
"reward_std": 0.23499999642372132,
|
||
|
|
"rewards/reward_fn/mean": -0.08750000596046448,
|
||
|
|
"rewards/reward_fn/std": 0.23499999642372132,
|
||
|
|
"step": 1105,
|
||
|
|
"step_time": 0.6418257012963295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 60.2,
|
||
|
|
"completions/max_terminated_length": 60.2,
|
||
|
|
"completions/mean_length": 22.15,
|
||
|
|
"completions/mean_terminated_length": 22.15,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.7380276054143906,
|
||
|
|
"epoch": 1.1443298969072164,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 24.625,
|
||
|
|
"learning_rate": 2.3112523818335605e-07,
|
||
|
|
"loss": 0.04171156883239746,
|
||
|
|
"num_tokens": 285521.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 1110,
|
||
|
|
"step_time": 1.4538947951979935
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.2,
|
||
|
|
"completions/max_terminated_length": 14.2,
|
||
|
|
"completions/mean_length": 8.9,
|
||
|
|
"completions/mean_terminated_length": 8.9,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.997655501961708,
|
||
|
|
"epoch": 1.1494845360824741,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 25.75,
|
||
|
|
"learning_rate": 2.2888331595480397e-07,
|
||
|
|
"loss": -0.0034986168146133425,
|
||
|
|
"num_tokens": 286707.0,
|
||
|
|
"reward": -0.08833332657814026,
|
||
|
|
"reward_std": 0.36946714818477633,
|
||
|
|
"rewards/reward_fn/mean": -0.08833332657814026,
|
||
|
|
"rewards/reward_fn/std": 0.36946714818477633,
|
||
|
|
"step": 1115,
|
||
|
|
"step_time": 0.5262484772130847
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.0,
|
||
|
|
"completions/max_terminated_length": 10.0,
|
||
|
|
"completions/mean_length": 7.6,
|
||
|
|
"completions/mean_terminated_length": 7.6,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 1.0448400229215622,
|
||
|
|
"epoch": 1.1546391752577319,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.2664310285220112e-07,
|
||
|
|
"loss": 0.10587654113769532,
|
||
|
|
"num_tokens": 287859.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.165283465385437,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.165283465385437,
|
||
|
|
"step": 1120,
|
||
|
|
"step_time": 0.4480899182148278
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.6,
|
||
|
|
"completions/max_terminated_length": 25.6,
|
||
|
|
"completions/mean_length": 13.3,
|
||
|
|
"completions/mean_terminated_length": 13.3,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 1.2160271763801576,
|
||
|
|
"epoch": 1.1597938144329896,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.2440478019219437e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 289173.0,
|
||
|
|
"reward": -0.5,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/reward_fn/mean": -0.5,
|
||
|
|
"rewards/reward_fn/std": 0.0,
|
||
|
|
"step": 1125,
|
||
|
|
"step_time": 0.7431498907506466
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.0,
|
||
|
|
"completions/max_terminated_length": 22.0,
|
||
|
|
"completions/mean_length": 10.95,
|
||
|
|
"completions/mean_terminated_length": 10.95,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.8884743601083755,
|
||
|
|
"epoch": 1.1649484536082475,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 57.5,
|
||
|
|
"learning_rate": 2.2216852913842347e-07,
|
||
|
|
"loss": 0.09710804224014283,
|
||
|
|
"num_tokens": 290424.0,
|
||
|
|
"reward": -0.3950000047683716,
|
||
|
|
"reward_std": 0.3127905666828156,
|
||
|
|
"rewards/reward_fn/mean": -0.3950000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.3127905666828156,
|
||
|
|
"step": 1130,
|
||
|
|
"step_time": 0.6823917054571211
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 56.6,
|
||
|
|
"completions/max_terminated_length": 56.6,
|
||
|
|
"completions/mean_length": 21.35,
|
||
|
|
"completions/mean_terminated_length": 21.35,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.8476454958319664,
|
||
|
|
"epoch": 1.1701030927835052,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 48.5,
|
||
|
|
"learning_rate": 2.1993453068685807e-07,
|
||
|
|
"loss": 0.21475210189819335,
|
||
|
|
"num_tokens": 291855.0,
|
||
|
|
"reward": -0.11125000417232514,
|
||
|
|
"reward_std": 0.3925000071525574,
|
||
|
|
"rewards/reward_fn/mean": -0.11125000417232514,
|
||
|
|
"rewards/reward_fn/std": 0.3925000011920929,
|
||
|
|
"step": 1135,
|
||
|
|
"step_time": 1.3351986355148255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.4,
|
||
|
|
"completions/max_terminated_length": 13.4,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 1.01234889626503,
|
||
|
|
"epoch": 1.175257731958763,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.1770296565114846e-07,
|
||
|
|
"loss": -0.0250102162361145,
|
||
|
|
"num_tokens": 293075.0,
|
||
|
|
"reward": -0.3625,
|
||
|
|
"reward_std": 0.2073760449886322,
|
||
|
|
"rewards/reward_fn/mean": -0.3625,
|
||
|
|
"rewards/reward_fn/std": 0.2073760449886322,
|
||
|
|
"step": 1140,
|
||
|
|
"step_time": 0.5155650717206299
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.0,
|
||
|
|
"completions/max_terminated_length": 18.0,
|
||
|
|
"completions/mean_length": 11.3,
|
||
|
|
"completions/mean_terminated_length": 11.3,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0420640349388122,
|
||
|
|
"epoch": 1.1804123711340206,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.1547401464799096e-07,
|
||
|
|
"loss": 0.12414703369140626,
|
||
|
|
"num_tokens": 294325.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 1145,
|
||
|
|
"step_time": 0.602156451717019
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 12.55,
|
||
|
|
"completions/mean_terminated_length": 12.55,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.7143237292766571,
|
||
|
|
"epoch": 1.1855670103092784,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.132478580825093e-07,
|
||
|
|
"loss": 0.041160008311271666,
|
||
|
|
"num_tokens": 295608.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.05,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.05,
|
||
|
|
"step": 1150,
|
||
|
|
"step_time": 0.5545094406232238
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 9.8,
|
||
|
|
"completions/mean_terminated_length": 9.8,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.8000463306903839,
|
||
|
|
"epoch": 1.190721649484536,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 17.375,
|
||
|
|
"learning_rate": 2.1102467613365334e-07,
|
||
|
|
"loss": -0.008892300724983215,
|
||
|
|
"num_tokens": 296856.0,
|
||
|
|
"reward": -0.3304166689515114,
|
||
|
|
"reward_std": 0.2501828670501709,
|
||
|
|
"rewards/reward_fn/mean": -0.3304166689515114,
|
||
|
|
"rewards/reward_fn/std": 0.25018285512924193,
|
||
|
|
"step": 1155,
|
||
|
|
"step_time": 0.5423662343993783
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.0,
|
||
|
|
"completions/max_terminated_length": 19.0,
|
||
|
|
"completions/mean_length": 11.5,
|
||
|
|
"completions/mean_terminated_length": 11.5,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0487438976764678,
|
||
|
|
"epoch": 1.1958762886597938,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.0880464873961553e-07,
|
||
|
|
"loss": 0.008957856893539428,
|
||
|
|
"num_tokens": 298118.0,
|
||
|
|
"reward": -0.33583333194255827,
|
||
|
|
"reward_std": 0.39479472637176516,
|
||
|
|
"rewards/reward_fn/mean": -0.33583333194255827,
|
||
|
|
"rewards/reward_fn/std": 0.39479475021362304,
|
||
|
|
"step": 1160,
|
||
|
|
"step_time": 0.6194642005488277
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 16.4,
|
||
|
|
"completions/mean_terminated_length": 16.4,
|
||
|
|
"completions/min_length": 12.2,
|
||
|
|
"completions/min_terminated_length": 12.2,
|
||
|
|
"entropy": 0.7133869394659996,
|
||
|
|
"epoch": 1.2010309278350515,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.065879555832674e-07,
|
||
|
|
"loss": -0.03838225901126861,
|
||
|
|
"num_tokens": 299458.0,
|
||
|
|
"reward": -0.45999999940395353,
|
||
|
|
"reward_std": 0.25230334997177123,
|
||
|
|
"rewards/reward_fn/mean": -0.45999999940395353,
|
||
|
|
"rewards/reward_fn/std": 0.25230334997177123,
|
||
|
|
"step": 1165,
|
||
|
|
"step_time": 0.6352453227154911
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.2,
|
||
|
|
"completions/max_terminated_length": 22.2,
|
||
|
|
"completions/mean_length": 12.05,
|
||
|
|
"completions/mean_terminated_length": 12.05,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.2859386861324311,
|
||
|
|
"epoch": 1.2061855670103092,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 15.5,
|
||
|
|
"learning_rate": 2.0437477607761655e-07,
|
||
|
|
"loss": 0.16457252502441405,
|
||
|
|
"num_tokens": 300711.0,
|
||
|
|
"reward": -0.3799999952316284,
|
||
|
|
"reward_std": 0.13106836080551149,
|
||
|
|
"rewards/reward_fn/mean": -0.3799999952316284,
|
||
|
|
"rewards/reward_fn/std": 0.13106836080551149,
|
||
|
|
"step": 1170,
|
||
|
|
"step_time": 0.6807820100337267
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.4,
|
||
|
|
"completions/max_terminated_length": 24.4,
|
||
|
|
"completions/mean_length": 14.95,
|
||
|
|
"completions/mean_terminated_length": 14.95,
|
||
|
|
"completions/min_length": 10.4,
|
||
|
|
"completions/min_terminated_length": 10.4,
|
||
|
|
"entropy": 1.0816936790943146,
|
||
|
|
"epoch": 1.211340206185567,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.375,
|
||
|
|
"learning_rate": 2.021652893512854e-07,
|
||
|
|
"loss": 0.25116305351257323,
|
||
|
|
"num_tokens": 302046.0,
|
||
|
|
"reward": -0.5069999992847443,
|
||
|
|
"reward_std": 0.28600000143051146,
|
||
|
|
"rewards/reward_fn/mean": -0.5069999992847443,
|
||
|
|
"rewards/reward_fn/std": 0.2859999895095825,
|
||
|
|
"step": 1175,
|
||
|
|
"step_time": 0.7220191801898181
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.2,
|
||
|
|
"completions/max_terminated_length": 19.2,
|
||
|
|
"completions/mean_length": 12.95,
|
||
|
|
"completions/mean_terminated_length": 12.95,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.9652112066745758,
|
||
|
|
"epoch": 1.2164948453608249,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.125,
|
||
|
|
"learning_rate": 1.9995967423401306e-07,
|
||
|
|
"loss": 0.10934715270996094,
|
||
|
|
"num_tokens": 303353.0,
|
||
|
|
"reward": -0.19833333194255828,
|
||
|
|
"reward_std": 0.3716987192630768,
|
||
|
|
"rewards/reward_fn/mean": -0.19833333194255828,
|
||
|
|
"rewards/reward_fn/std": 0.3716987192630768,
|
||
|
|
"step": 1180,
|
||
|
|
"step_time": 0.6236526501365006
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.4,
|
||
|
|
"completions/max_terminated_length": 19.4,
|
||
|
|
"completions/mean_length": 14.1,
|
||
|
|
"completions/mean_terminated_length": 14.1,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.9499202251434327,
|
||
|
|
"epoch": 1.2216494845360826,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 14.0,
|
||
|
|
"learning_rate": 1.977581092421812e-07,
|
||
|
|
"loss": 0.07946513891220093,
|
||
|
|
"num_tokens": 304687.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 1185,
|
||
|
|
"step_time": 0.6262445945292712
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.0,
|
||
|
|
"completions/max_terminated_length": 16.0,
|
||
|
|
"completions/mean_length": 9.95,
|
||
|
|
"completions/mean_terminated_length": 9.95,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.8996234953403472,
|
||
|
|
"epoch": 1.2268041237113403,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.9556077256436586e-07,
|
||
|
|
"loss": 0.10128650665283204,
|
||
|
|
"num_tokens": 305902.0,
|
||
|
|
"reward": -0.4949999988079071,
|
||
|
|
"reward_std": 0.20011107325553895,
|
||
|
|
"rewards/reward_fn/mean": -0.4949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.20011107325553895,
|
||
|
|
"step": 1190,
|
||
|
|
"step_time": 0.6217918395996094
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 76.4,
|
||
|
|
"completions/max_terminated_length": 26.8,
|
||
|
|
"completions/mean_length": 26.35,
|
||
|
|
"completions/mean_terminated_length": 13.88333339691162,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 1.2090439796447754,
|
||
|
|
"epoch": 1.231958762886598,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.933678420469147e-07,
|
||
|
|
"loss": -0.007263195514678955,
|
||
|
|
"num_tokens": 307457.0,
|
||
|
|
"reward": -0.5066666662693023,
|
||
|
|
"reward_std": 0.3612557053565979,
|
||
|
|
"rewards/reward_fn/mean": -0.5066666662693023,
|
||
|
|
"rewards/reward_fn/std": 0.3612557053565979,
|
||
|
|
"step": 1195,
|
||
|
|
"step_time": 1.6938601299189031
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.6,
|
||
|
|
"completions/max_terminated_length": 22.6,
|
||
|
|
"completions/mean_length": 13.8,
|
||
|
|
"completions/mean_terminated_length": 13.8,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.7910245209932327,
|
||
|
|
"epoch": 1.2371134020618557,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.75,
|
||
|
|
"learning_rate": 1.911794951795531e-07,
|
||
|
|
"loss": 0.04146863520145416,
|
||
|
|
"num_tokens": 308777.0,
|
||
|
|
"reward": -0.4508333384990692,
|
||
|
|
"reward_std": 0.41441943049430846,
|
||
|
|
"rewards/reward_fn/mean": -0.4508333384990692,
|
||
|
|
"rewards/reward_fn/std": 0.41441943049430846,
|
||
|
|
"step": 1200,
|
||
|
|
"step_time": 0.685492187552154
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.0,
|
||
|
|
"completions/max_terminated_length": 25.0,
|
||
|
|
"completions/mean_length": 15.25,
|
||
|
|
"completions/mean_terminated_length": 15.25,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.9586571276187896,
|
||
|
|
"epoch": 1.2422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 1.889959090810185e-07,
|
||
|
|
"loss": 0.11326441764831544,
|
||
|
|
"num_tokens": 310094.0,
|
||
|
|
"reward": -0.45833333134651183,
|
||
|
|
"reward_std": 0.165283465385437,
|
||
|
|
"rewards/reward_fn/mean": -0.45833333134651183,
|
||
|
|
"rewards/reward_fn/std": 0.165283465385437,
|
||
|
|
"step": 1205,
|
||
|
|
"step_time": 0.7412053862586617
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.2,
|
||
|
|
"completions/max_terminated_length": 21.2,
|
||
|
|
"completions/mean_length": 11.55,
|
||
|
|
"completions/mean_terminated_length": 11.55,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.8803381383419037,
|
||
|
|
"epoch": 1.2474226804123711,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.125,
|
||
|
|
"learning_rate": 1.8681726048472497e-07,
|
||
|
|
"loss": 0.23633675575256347,
|
||
|
|
"num_tokens": 311329.0,
|
||
|
|
"reward": -0.4,
|
||
|
|
"reward_std": 0.4,
|
||
|
|
"rewards/reward_fn/mean": -0.4,
|
||
|
|
"rewards/reward_fn/std": 0.4,
|
||
|
|
"step": 1210,
|
||
|
|
"step_time": 0.6590411723591387
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 53.6,
|
||
|
|
"completions/max_terminated_length": 53.6,
|
||
|
|
"completions/mean_length": 24.1,
|
||
|
|
"completions/mean_terminated_length": 24.1,
|
||
|
|
"completions/min_length": 11.8,
|
||
|
|
"completions/min_terminated_length": 11.8,
|
||
|
|
"entropy": 1.015632688999176,
|
||
|
|
"epoch": 1.2525773195876289,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 22.25,
|
||
|
|
"learning_rate": 1.8464372572445863e-07,
|
||
|
|
"loss": 0.07433696389198304,
|
||
|
|
"num_tokens": 312823.0,
|
||
|
|
"reward": -0.4983333319425583,
|
||
|
|
"reward_std": 0.20333333015441896,
|
||
|
|
"rewards/reward_fn/mean": -0.4983333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.2033333420753479,
|
||
|
|
"step": 1215,
|
||
|
|
"step_time": 1.2660644174553455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.4,
|
||
|
|
"completions/max_terminated_length": 22.4,
|
||
|
|
"completions/mean_length": 11.4,
|
||
|
|
"completions/mean_terminated_length": 11.4,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.7511100977659225,
|
||
|
|
"epoch": 1.2577319587628866,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 10.875,
|
||
|
|
"learning_rate": 1.8247548072010593e-07,
|
||
|
|
"loss": 0.20432829856872559,
|
||
|
|
"num_tokens": 314051.0,
|
||
|
|
"reward": -0.5385000020265579,
|
||
|
|
"reward_std": 0.4213788390159607,
|
||
|
|
"rewards/reward_fn/mean": -0.5385000020265579,
|
||
|
|
"rewards/reward_fn/std": 0.42137885093688965,
|
||
|
|
"step": 1220,
|
||
|
|
"step_time": 0.6830242997035384
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 28.8,
|
||
|
|
"completions/max_terminated_length": 28.8,
|
||
|
|
"completions/mean_length": 15.35,
|
||
|
|
"completions/mean_terminated_length": 15.35,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.7591416954994201,
|
||
|
|
"epoch": 1.2628865979381443,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 7.0625,
|
||
|
|
"learning_rate": 1.8031270096341533e-07,
|
||
|
|
"loss": 0.03440877199172974,
|
||
|
|
"num_tokens": 315406.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 1225,
|
||
|
|
"step_time": 0.8744408298283816
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.2,
|
||
|
|
"completions/max_terminated_length": 23.2,
|
||
|
|
"completions/mean_length": 13.15,
|
||
|
|
"completions/mean_terminated_length": 13.15,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.8470461785793304,
|
||
|
|
"epoch": 1.268041237113402,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 20.25,
|
||
|
|
"learning_rate": 1.7815556150379296e-07,
|
||
|
|
"loss": 0.12400050163269043,
|
||
|
|
"num_tokens": 316697.0,
|
||
|
|
"reward": -0.4033333361148834,
|
||
|
|
"reward_std": 0.4449474155902863,
|
||
|
|
"rewards/reward_fn/mean": -0.4033333361148834,
|
||
|
|
"rewards/reward_fn/std": 0.4449474155902863,
|
||
|
|
"step": 1230,
|
||
|
|
"step_time": 0.7029364148154855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 12.05,
|
||
|
|
"completions/mean_terminated_length": 12.05,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.7155319631099701,
|
||
|
|
"epoch": 1.2731958762886597,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 10.0,
|
||
|
|
"learning_rate": 1.7600423693413507e-07,
|
||
|
|
"loss": 0.1241507887840271,
|
||
|
|
"num_tokens": 317986.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.16547005176544188,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.16547005176544188,
|
||
|
|
"step": 1235,
|
||
|
|
"step_time": 0.6039237166754902
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.6,
|
||
|
|
"completions/max_terminated_length": 16.6,
|
||
|
|
"completions/mean_length": 11.1,
|
||
|
|
"completions/mean_terminated_length": 11.1,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.9233552157878876,
|
||
|
|
"epoch": 1.2783505154639174,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 13.75,
|
||
|
|
"learning_rate": 1.738589013766969e-07,
|
||
|
|
"loss": 0.028127449750900268,
|
||
|
|
"num_tokens": 319208.0,
|
||
|
|
"reward": -0.2425000011920929,
|
||
|
|
"reward_std": 0.3149999976158142,
|
||
|
|
"rewards/reward_fn/mean": -0.2425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3149999976158142,
|
||
|
|
"step": 1240,
|
||
|
|
"step_time": 0.5808714678511023
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 11.5,
|
||
|
|
"completions/mean_terminated_length": 11.5,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.9115808337926865,
|
||
|
|
"epoch": 1.2835051546391751,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 21.0,
|
||
|
|
"learning_rate": 1.7171972846899941e-07,
|
||
|
|
"loss": 0.05150402188301086,
|
||
|
|
"num_tokens": 320466.0,
|
||
|
|
"reward": -0.3074999988079071,
|
||
|
|
"reward_std": 0.469794100522995,
|
||
|
|
"rewards/reward_fn/mean": -0.3074999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.46979411244392394,
|
||
|
|
"step": 1245,
|
||
|
|
"step_time": 0.567639524769038
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 11.25,
|
||
|
|
"completions/mean_terminated_length": 11.25,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.6332197517156601,
|
||
|
|
"epoch": 1.2886597938144329,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 11.0625,
|
||
|
|
"learning_rate": 1.6958689134977587e-07,
|
||
|
|
"loss": 0.06763789653778077,
|
||
|
|
"num_tokens": 321707.0,
|
||
|
|
"reward": -0.5,
|
||
|
|
"reward_std": 0.3,
|
||
|
|
"rewards/reward_fn/mean": -0.5,
|
||
|
|
"rewards/reward_fn/std": 0.3,
|
||
|
|
"step": 1250,
|
||
|
|
"step_time": 0.5252592659555375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.4,
|
||
|
|
"completions/max_terminated_length": 16.4,
|
||
|
|
"completions/mean_length": 9.55,
|
||
|
|
"completions/mean_terminated_length": 9.55,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.0738073289394379,
|
||
|
|
"epoch": 1.2938144329896908,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.6746056264495846e-07,
|
||
|
|
"loss": 0.02612420916557312,
|
||
|
|
"num_tokens": 322922.0,
|
||
|
|
"reward": -0.425,
|
||
|
|
"reward_std": 0.30615527629852296,
|
||
|
|
"rewards/reward_fn/mean": -0.425,
|
||
|
|
"rewards/reward_fn/std": 0.30615527629852296,
|
||
|
|
"step": 1255,
|
||
|
|
"step_time": 0.5707598288543523
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 9.55,
|
||
|
|
"completions/mean_terminated_length": 9.55,
|
||
|
|
"completions/min_length": 5.0,
|
||
|
|
"completions/min_terminated_length": 5.0,
|
||
|
|
"entropy": 0.6144456475973129,
|
||
|
|
"epoch": 1.2989690721649485,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 23.5,
|
||
|
|
"learning_rate": 1.65340914453706e-07,
|
||
|
|
"loss": 0.20289556980133056,
|
||
|
|
"num_tokens": 324121.0,
|
||
|
|
"reward": -0.4025000035762787,
|
||
|
|
"reward_std": 0.4698511064052582,
|
||
|
|
"rewards/reward_fn/mean": -0.4025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.4698511064052582,
|
||
|
|
"step": 1260,
|
||
|
|
"step_time": 0.5400445449166
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 29.2,
|
||
|
|
"completions/max_terminated_length": 29.2,
|
||
|
|
"completions/mean_length": 15.3,
|
||
|
|
"completions/mean_terminated_length": 15.3,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.7304955810308457,
|
||
|
|
"epoch": 1.3041237113402062,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 13.75,
|
||
|
|
"learning_rate": 1.632281183344756e-07,
|
||
|
|
"loss": 0.13050819635391236,
|
||
|
|
"num_tokens": 325455.0,
|
||
|
|
"reward": -0.38333333730697633,
|
||
|
|
"reward_std": 0.4089271903038025,
|
||
|
|
"rewards/reward_fn/mean": -0.38333333730697633,
|
||
|
|
"rewards/reward_fn/std": 0.40892720222473145,
|
||
|
|
"step": 1265,
|
||
|
|
"step_time": 0.873385751992464
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 11.95,
|
||
|
|
"completions/mean_terminated_length": 11.95,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.6056427478790283,
|
||
|
|
"epoch": 1.309278350515464,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.6112234529113606e-07,
|
||
|
|
"loss": 0.10200026035308837,
|
||
|
|
"num_tokens": 326730.0,
|
||
|
|
"reward": -0.55,
|
||
|
|
"reward_std": 0.33094010353088377,
|
||
|
|
"rewards/reward_fn/mean": -0.55,
|
||
|
|
"rewards/reward_fn/std": 0.33094010353088377,
|
||
|
|
"step": 1270,
|
||
|
|
"step_time": 0.5555100217461586
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.4,
|
||
|
|
"completions/max_terminated_length": 19.4,
|
||
|
|
"completions/mean_length": 10.7,
|
||
|
|
"completions/mean_terminated_length": 10.7,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 1.0357014298439027,
|
||
|
|
"epoch": 1.3144329896907216,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 15.125,
|
||
|
|
"learning_rate": 1.5902376575912814e-07,
|
||
|
|
"loss": 0.2978448629379272,
|
||
|
|
"num_tokens": 327960.0,
|
||
|
|
"reward": -0.5262499988079071,
|
||
|
|
"reward_std": 0.4544562935829163,
|
||
|
|
"rewards/reward_fn/mean": -0.5262499988079071,
|
||
|
|
"rewards/reward_fn/std": 0.4544562935829163,
|
||
|
|
"step": 1275,
|
||
|
|
"step_time": 0.6274704253301024
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 9.85,
|
||
|
|
"completions/mean_terminated_length": 9.85,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.8286823898553848,
|
||
|
|
"epoch": 1.3195876288659794,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 1.569325495916696e-07,
|
||
|
|
"loss": 0.02840469479560852,
|
||
|
|
"num_tokens": 329177.0,
|
||
|
|
"reward": -0.3458333373069763,
|
||
|
|
"reward_std": 0.2295683205127716,
|
||
|
|
"rewards/reward_fn/mean": -0.3458333373069763,
|
||
|
|
"rewards/reward_fn/std": 0.22956833243370056,
|
||
|
|
"step": 1280,
|
||
|
|
"step_time": 0.5265060624107718
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.2,
|
||
|
|
"completions/max_terminated_length": 11.2,
|
||
|
|
"completions/mean_length": 7.75,
|
||
|
|
"completions/mean_terminated_length": 7.75,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.8311231344938278,
|
||
|
|
"epoch": 1.324742268041237,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 1.5484886604600795e-07,
|
||
|
|
"loss": 0.1401970148086548,
|
||
|
|
"num_tokens": 330348.0,
|
||
|
|
"reward": -0.36166666746139525,
|
||
|
|
"reward_std": 0.45088627338409426,
|
||
|
|
"rewards/reward_fn/mean": -0.36166666746139525,
|
||
|
|
"rewards/reward_fn/std": 0.45088627338409426,
|
||
|
|
"step": 1285,
|
||
|
|
"step_time": 0.4739662584848702
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 10.6,
|
||
|
|
"completions/mean_terminated_length": 10.6,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.894581812620163,
|
||
|
|
"epoch": 1.3298969072164948,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 16.375,
|
||
|
|
"learning_rate": 1.5277288376972114e-07,
|
||
|
|
"loss": 0.10380651950836181,
|
||
|
|
"num_tokens": 331592.0,
|
||
|
|
"reward": -0.19583333134651185,
|
||
|
|
"reward_std": 0.530957841873169,
|
||
|
|
"rewards/reward_fn/mean": -0.19583333134651185,
|
||
|
|
"rewards/reward_fn/std": 0.530957841873169,
|
||
|
|
"step": 1290,
|
||
|
|
"step_time": 0.5679388328455388
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.2,
|
||
|
|
"completions/max_terminated_length": 23.2,
|
||
|
|
"completions/mean_length": 14.7,
|
||
|
|
"completions/mean_terminated_length": 14.7,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.7432743966579437,
|
||
|
|
"epoch": 1.3350515463917525,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 10.875,
|
||
|
|
"learning_rate": 1.5070477078706755e-07,
|
||
|
|
"loss": 0.003392466902732849,
|
||
|
|
"num_tokens": 332938.0,
|
||
|
|
"reward": -0.3454999983310699,
|
||
|
|
"reward_std": 0.3838511049747467,
|
||
|
|
"rewards/reward_fn/mean": -0.3454999983310699,
|
||
|
|
"rewards/reward_fn/std": 0.3838511049747467,
|
||
|
|
"step": 1295,
|
||
|
|
"step_time": 0.6983847383409738
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.2,
|
||
|
|
"completions/max_terminated_length": 15.2,
|
||
|
|
"completions/mean_length": 10.9,
|
||
|
|
"completions/mean_terminated_length": 10.9,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.8243459284305572,
|
||
|
|
"epoch": 1.3402061855670104,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 23.625,
|
||
|
|
"learning_rate": 1.4864469448538696e-07,
|
||
|
|
"loss": 0.10561842918395996,
|
||
|
|
"num_tokens": 334168.0,
|
||
|
|
"reward": -0.4600000023841858,
|
||
|
|
"reward_std": 0.3377350211143494,
|
||
|
|
"rewards/reward_fn/mean": -0.4600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.3377350211143494,
|
||
|
|
"step": 1300,
|
||
|
|
"step_time": 0.6084295228123665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 10.0,
|
||
|
|
"completions/mean_terminated_length": 10.0,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.7640111565589904,
|
||
|
|
"epoch": 1.3453608247422681,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 33.25,
|
||
|
|
"learning_rate": 1.465928216015522e-07,
|
||
|
|
"loss": 0.027267986536026002,
|
||
|
|
"num_tokens": 335364.0,
|
||
|
|
"reward": -0.5425000011920929,
|
||
|
|
"reward_std": 0.11499999761581421,
|
||
|
|
"rewards/reward_fn/mean": -0.5425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.11499999761581421,
|
||
|
|
"step": 1305,
|
||
|
|
"step_time": 0.5023958536796271
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 61.0,
|
||
|
|
"completions/max_terminated_length": 61.0,
|
||
|
|
"completions/mean_length": 25.35,
|
||
|
|
"completions/mean_terminated_length": 25.35,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.8650216937065125,
|
||
|
|
"epoch": 1.3505154639175259,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 4.375,
|
||
|
|
"learning_rate": 1.445493182084743e-07,
|
||
|
|
"loss": -0.0330998033285141,
|
||
|
|
"num_tokens": 336903.0,
|
||
|
|
"reward": -0.5233333319425583,
|
||
|
|
"reward_std": 0.25333333015441895,
|
||
|
|
"rewards/reward_fn/mean": -0.5233333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.2533333420753479,
|
||
|
|
"step": 1310,
|
||
|
|
"step_time": 1.4153380719013513
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 9.8,
|
||
|
|
"completions/max_terminated_length": 9.8,
|
||
|
|
"completions/mean_length": 6.7,
|
||
|
|
"completions/mean_terminated_length": 6.7,
|
||
|
|
"completions/min_length": 4.8,
|
||
|
|
"completions/min_terminated_length": 4.8,
|
||
|
|
"entropy": 0.8046780616044998,
|
||
|
|
"epoch": 1.3556701030927836,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 18.625,
|
||
|
|
"learning_rate": 1.425143497016608e-07,
|
||
|
|
"loss": 0.1706416368484497,
|
||
|
|
"num_tokens": 338045.0,
|
||
|
|
"reward": -0.1850000023841858,
|
||
|
|
"reward_std": 0.2744683504104614,
|
||
|
|
"rewards/reward_fn/mean": -0.1850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2744683504104614,
|
||
|
|
"step": 1315,
|
||
|
|
"step_time": 0.4627469423227012
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.0,
|
||
|
|
"completions/max_terminated_length": 18.0,
|
||
|
|
"completions/mean_length": 9.65,
|
||
|
|
"completions/mean_terminated_length": 9.65,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0882736980915069,
|
||
|
|
"epoch": 1.3608247422680413,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 34.0,
|
||
|
|
"learning_rate": 1.4048808078582942e-07,
|
||
|
|
"loss": 0.1820279359817505,
|
||
|
|
"num_tokens": 339246.0,
|
||
|
|
"reward": -0.3949999988079071,
|
||
|
|
"reward_std": 0.342376047372818,
|
||
|
|
"rewards/reward_fn/mean": -0.3949999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.342376047372818,
|
||
|
|
"step": 1320,
|
||
|
|
"step_time": 0.6051038973964751
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 31.2,
|
||
|
|
"completions/max_terminated_length": 31.2,
|
||
|
|
"completions/mean_length": 15.0,
|
||
|
|
"completions/mean_terminated_length": 15.0,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 1.1348010390996932,
|
||
|
|
"epoch": 1.365979381443299,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 36.5,
|
||
|
|
"learning_rate": 1.384706754615767e-07,
|
||
|
|
"loss": -0.08873192071914673,
|
||
|
|
"num_tokens": 340586.0,
|
||
|
|
"reward": -0.15250000059604646,
|
||
|
|
"reward_std": 0.2449999988079071,
|
||
|
|
"rewards/reward_fn/mean": -0.15250000059604646,
|
||
|
|
"rewards/reward_fn/std": 0.2449999988079071,
|
||
|
|
"step": 1325,
|
||
|
|
"step_time": 0.853823087643832
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.2,
|
||
|
|
"completions/max_terminated_length": 24.2,
|
||
|
|
"completions/mean_length": 17.3,
|
||
|
|
"completions/mean_terminated_length": 17.3,
|
||
|
|
"completions/min_length": 13.0,
|
||
|
|
"completions/min_terminated_length": 13.0,
|
||
|
|
"entropy": 0.8157379269599915,
|
||
|
|
"epoch": 1.3711340206185567,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 5.6875,
|
||
|
|
"learning_rate": 1.3646229701210488e-07,
|
||
|
|
"loss": 0.0920086920261383,
|
||
|
|
"num_tokens": 341964.0,
|
||
|
|
"reward": -0.5612500071525574,
|
||
|
|
"reward_std": 0.3543502926826477,
|
||
|
|
"rewards/reward_fn/mean": -0.5612500071525574,
|
||
|
|
"rewards/reward_fn/std": 0.3543502926826477,
|
||
|
|
"step": 1330,
|
||
|
|
"step_time": 0.7886355902068317
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.0,
|
||
|
|
"completions/max_terminated_length": 13.0,
|
||
|
|
"completions/mean_length": 8.4,
|
||
|
|
"completions/mean_terminated_length": 8.4,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 1.2810046076774597,
|
||
|
|
"epoch": 1.3762886597938144,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.3446310799000575e-07,
|
||
|
|
"loss": 0.09169626832008362,
|
||
|
|
"num_tokens": 343160.0,
|
||
|
|
"reward": -0.4050000011920929,
|
||
|
|
"reward_std": 0.2223760485649109,
|
||
|
|
"rewards/reward_fn/mean": -0.4050000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.2223760485649109,
|
||
|
|
"step": 1335,
|
||
|
|
"step_time": 0.5072437161579728
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.6,
|
||
|
|
"completions/max_terminated_length": 26.6,
|
||
|
|
"completions/mean_length": 15.05,
|
||
|
|
"completions/mean_terminated_length": 15.05,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.6845301300287246,
|
||
|
|
"epoch": 1.3814432989690721,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 20.375,
|
||
|
|
"learning_rate": 1.3247327020410452e-07,
|
||
|
|
"loss": 0.016708952188491822,
|
||
|
|
"num_tokens": 344497.0,
|
||
|
|
"reward": -0.2340000033378601,
|
||
|
|
"reward_std": 0.37685110569000246,
|
||
|
|
"rewards/reward_fn/mean": -0.2340000033378601,
|
||
|
|
"rewards/reward_fn/std": 0.3768511176109314,
|
||
|
|
"step": 1340,
|
||
|
|
"step_time": 0.7704925104044378
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 10.4,
|
||
|
|
"completions/mean_terminated_length": 10.4,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.8923976421356201,
|
||
|
|
"epoch": 1.3865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.3049294470636302e-07,
|
||
|
|
"loss": 0.12361987829208373,
|
||
|
|
"num_tokens": 345705.0,
|
||
|
|
"reward": -0.2,
|
||
|
|
"reward_std": 0.3,
|
||
|
|
"rewards/reward_fn/mean": -0.2,
|
||
|
|
"rewards/reward_fn/std": 0.3,
|
||
|
|
"step": 1345,
|
||
|
|
"step_time": 0.6526330811902881
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.0,
|
||
|
|
"completions/max_terminated_length": 24.0,
|
||
|
|
"completions/mean_length": 12.75,
|
||
|
|
"completions/mean_terminated_length": 12.75,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 1.0238151282072068,
|
||
|
|
"epoch": 1.3917525773195876,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 29.125,
|
||
|
|
"learning_rate": 1.285222917788449e-07,
|
||
|
|
"loss": 0.10306930541992188,
|
||
|
|
"num_tokens": 347004.0,
|
||
|
|
"reward": -0.25499999821186065,
|
||
|
|
"reward_std": 0.3032050788402557,
|
||
|
|
"rewards/reward_fn/mean": -0.25499999821186065,
|
||
|
|
"rewards/reward_fn/std": 0.3032050788402557,
|
||
|
|
"step": 1350,
|
||
|
|
"step_time": 0.7229462908580899
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.4,
|
||
|
|
"completions/max_terminated_length": 16.4,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 4.8,
|
||
|
|
"completions/min_terminated_length": 4.8,
|
||
|
|
"entropy": 1.2665182173252105,
|
||
|
|
"epoch": 1.3969072164948453,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 34.25,
|
||
|
|
"learning_rate": 1.2656147092074276e-07,
|
||
|
|
"loss": 0.27374863624572754,
|
||
|
|
"num_tokens": 348196.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.20773502588272094,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.20773502588272094,
|
||
|
|
"step": 1355,
|
||
|
|
"step_time": 0.5801575570367277
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 10.6,
|
||
|
|
"completions/mean_terminated_length": 10.6,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.2069101631641388,
|
||
|
|
"epoch": 1.402061855670103,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.875,
|
||
|
|
"learning_rate": 1.2461064083546872e-07,
|
||
|
|
"loss": 0.16335666179656982,
|
||
|
|
"num_tokens": 349416.0,
|
||
|
|
"reward": -0.01749999225139618,
|
||
|
|
"reward_std": 0.32012745440006257,
|
||
|
|
"rewards/reward_fn/mean": -0.01749999225139618,
|
||
|
|
"rewards/reward_fn/std": 0.32012745440006257,
|
||
|
|
"step": 1360,
|
||
|
|
"step_time": 0.5706661109812557
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.2,
|
||
|
|
"completions/max_terminated_length": 12.2,
|
||
|
|
"completions/mean_length": 9.0,
|
||
|
|
"completions/mean_terminated_length": 9.0,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.8017214298248291,
|
||
|
|
"epoch": 1.4072164948453607,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.2266995941780933e-07,
|
||
|
|
"loss": 0.14401029348373412,
|
||
|
|
"num_tokens": 350640.0,
|
||
|
|
"reward": -0.30999999642372134,
|
||
|
|
"reward_std": 0.2600000023841858,
|
||
|
|
"rewards/reward_fn/mean": -0.30999999642372134,
|
||
|
|
"rewards/reward_fn/std": 0.2600000023841858,
|
||
|
|
"step": 1365,
|
||
|
|
"step_time": 0.5673609586432576
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 30.2,
|
||
|
|
"completions/max_terminated_length": 30.2,
|
||
|
|
"completions/mean_length": 17.1,
|
||
|
|
"completions/mean_terminated_length": 17.1,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.6343318462371826,
|
||
|
|
"epoch": 1.4123711340206184,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 19.5,
|
||
|
|
"learning_rate": 1.2073958374114608e-07,
|
||
|
|
"loss": 0.15244705677032472,
|
||
|
|
"num_tokens": 352022.0,
|
||
|
|
"reward": -0.5025000035762787,
|
||
|
|
"reward_std": 0.26985110640525817,
|
||
|
|
"rewards/reward_fn/mean": -0.5025000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.26985110640525817,
|
||
|
|
"step": 1370,
|
||
|
|
"step_time": 0.8377051496878266
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 76.4,
|
||
|
|
"completions/max_terminated_length": 27.2,
|
||
|
|
"completions/mean_length": 27.35,
|
||
|
|
"completions/mean_terminated_length": 14.866666698455811,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 1.4102221041917802,
|
||
|
|
"epoch": 1.4175257731958764,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 1.1881967004474255e-07,
|
||
|
|
"loss": 0.1017134428024292,
|
||
|
|
"num_tokens": 353585.0,
|
||
|
|
"reward": -0.30613636672496797,
|
||
|
|
"reward_std": 0.3646326959133148,
|
||
|
|
"rewards/reward_fn/mean": -0.30613636672496797,
|
||
|
|
"rewards/reward_fn/std": 0.3646326959133148,
|
||
|
|
"step": 1375,
|
||
|
|
"step_time": 1.7935701003298163
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 26.2,
|
||
|
|
"completions/max_terminated_length": 26.2,
|
||
|
|
"completions/mean_length": 16.55,
|
||
|
|
"completions/mean_terminated_length": 16.55,
|
||
|
|
"completions/min_length": 11.0,
|
||
|
|
"completions/min_terminated_length": 11.0,
|
||
|
|
"entropy": 1.0183711111545564,
|
||
|
|
"epoch": 1.422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.1691037372109832e-07,
|
||
|
|
"loss": -0.013654737174510956,
|
||
|
|
"num_tokens": 354944.0,
|
||
|
|
"reward": -0.35833333134651185,
|
||
|
|
"reward_std": 0.30754843950271604,
|
||
|
|
"rewards/reward_fn/mean": -0.35833333134651185,
|
||
|
|
"rewards/reward_fn/std": 0.30754843950271604,
|
||
|
|
"step": 1380,
|
||
|
|
"step_time": 0.7602020460180938
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.2,
|
||
|
|
"completions/max_terminated_length": 16.2,
|
||
|
|
"completions/mean_length": 10.75,
|
||
|
|
"completions/mean_terminated_length": 10.75,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.7927883446216584,
|
||
|
|
"epoch": 1.4278350515463918,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 18.75,
|
||
|
|
"learning_rate": 1.1501184930337235e-07,
|
||
|
|
"loss": 0.03684360682964325,
|
||
|
|
"num_tokens": 356155.0,
|
||
|
|
"reward": -0.36833333075046537,
|
||
|
|
"reward_std": 0.36272807121276857,
|
||
|
|
"rewards/reward_fn/mean": -0.36833333075046537,
|
||
|
|
"rewards/reward_fn/std": 0.36272807121276857,
|
||
|
|
"step": 1385,
|
||
|
|
"step_time": 0.5737183832563459
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.2,
|
||
|
|
"completions/max_terminated_length": 18.2,
|
||
|
|
"completions/mean_length": 10.85,
|
||
|
|
"completions/mean_terminated_length": 10.85,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.8488032281398773,
|
||
|
|
"epoch": 1.4329896907216495,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 20.0,
|
||
|
|
"learning_rate": 1.1312425045287524e-07,
|
||
|
|
"loss": 0.10943784713745117,
|
||
|
|
"num_tokens": 357380.0,
|
||
|
|
"reward": -0.5100000023841857,
|
||
|
|
"reward_std": 0.1800000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.5100000023841857,
|
||
|
|
"rewards/reward_fn/std": 0.1800000011920929,
|
||
|
|
"step": 1390,
|
||
|
|
"step_time": 0.670763380266726
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.2,
|
||
|
|
"completions/max_terminated_length": 10.2,
|
||
|
|
"completions/mean_length": 7.7,
|
||
|
|
"completions/mean_terminated_length": 7.7,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.6782841980457306,
|
||
|
|
"epoch": 1.4381443298969072,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.1124772994663256e-07,
|
||
|
|
"loss": 0.10716173648834229,
|
||
|
|
"num_tokens": 358538.0,
|
||
|
|
"reward": -0.21916666626930237,
|
||
|
|
"reward_std": 0.24967803657054902,
|
||
|
|
"rewards/reward_fn/mean": -0.21916666626930237,
|
||
|
|
"rewards/reward_fn/std": 0.24967803657054902,
|
||
|
|
"step": 1395,
|
||
|
|
"step_time": 0.46183384573087094
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 56.8,
|
||
|
|
"completions/max_terminated_length": 56.8,
|
||
|
|
"completions/mean_length": 23.45,
|
||
|
|
"completions/mean_terminated_length": 23.45,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 1.0040556222200394,
|
||
|
|
"epoch": 1.443298969072165,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.0938243966501929e-07,
|
||
|
|
"loss": 0.22731239795684816,
|
||
|
|
"num_tokens": 360027.0,
|
||
|
|
"reward": -0.4699999988079071,
|
||
|
|
"reward_std": 0.17793227434158326,
|
||
|
|
"rewards/reward_fn/mean": -0.4699999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.17793227434158326,
|
||
|
|
"step": 1400,
|
||
|
|
"step_time": 1.3617384696379304
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 11.55,
|
||
|
|
"completions/mean_terminated_length": 11.55,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.7580155253410339,
|
||
|
|
"epoch": 1.4484536082474226,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 1.075285305794671e-07,
|
||
|
|
"loss": 0.11078314781188965,
|
||
|
|
"num_tokens": 361294.0,
|
||
|
|
"reward": -0.3039999961853027,
|
||
|
|
"reward_std": 0.4995263934135437,
|
||
|
|
"rewards/reward_fn/mean": -0.3039999961853027,
|
||
|
|
"rewards/reward_fn/std": 0.4995263934135437,
|
||
|
|
"step": 1405,
|
||
|
|
"step_time": 0.6288512685336173
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 61.2,
|
||
|
|
"completions/max_terminated_length": 45.4,
|
||
|
|
"completions/mean_length": 30.8,
|
||
|
|
"completions/mean_terminated_length": 21.483334350585938,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.8140331745147705,
|
||
|
|
"epoch": 1.4536082474226804,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 3.734375,
|
||
|
|
"learning_rate": 1.056861527402452e-07,
|
||
|
|
"loss": 0.0343685507774353,
|
||
|
|
"num_tokens": 362922.0,
|
||
|
|
"reward": -0.625,
|
||
|
|
"reward_std": 0.20773502588272094,
|
||
|
|
"rewards/reward_fn/mean": -0.625,
|
||
|
|
"rewards/reward_fn/std": 0.20773502588272094,
|
||
|
|
"step": 1410,
|
||
|
|
"step_time": 1.5017110424116253
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 9.0,
|
||
|
|
"completions/mean_terminated_length": 9.0,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 1.2074449896812438,
|
||
|
|
"epoch": 1.458762886597938,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.0385545526431566e-07,
|
||
|
|
"loss": 0.154154634475708,
|
||
|
|
"num_tokens": 364126.0,
|
||
|
|
"reward": -0.3650000005960464,
|
||
|
|
"reward_std": 0.2099999964237213,
|
||
|
|
"rewards/reward_fn/mean": -0.3650000005960464,
|
||
|
|
"rewards/reward_fn/std": 0.21000000834465027,
|
||
|
|
"step": 1415,
|
||
|
|
"step_time": 0.5890727814286947
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 12.1,
|
||
|
|
"completions/mean_terminated_length": 12.1,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.8329922676086425,
|
||
|
|
"epoch": 1.463917525773196,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.0203658632326423e-07,
|
||
|
|
"loss": 0.11881840229034424,
|
||
|
|
"num_tokens": 365384.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.18773502707481385,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.18773502707481385,
|
||
|
|
"step": 1420,
|
||
|
|
"step_time": 0.7082118081860245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.4,
|
||
|
|
"completions/max_terminated_length": 19.4,
|
||
|
|
"completions/mean_length": 11.05,
|
||
|
|
"completions/mean_terminated_length": 11.05,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.9463332563638687,
|
||
|
|
"epoch": 1.4690721649484537,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 15.8125,
|
||
|
|
"learning_rate": 1.0022969313130773e-07,
|
||
|
|
"loss": -0.11116218566894531,
|
||
|
|
"num_tokens": 366617.0,
|
||
|
|
"reward": -0.4077777773141861,
|
||
|
|
"reward_std": 0.35764952898025515,
|
||
|
|
"rewards/reward_fn/mean": -0.4077777773141861,
|
||
|
|
"rewards/reward_fn/std": 0.35764952898025515,
|
||
|
|
"step": 1425,
|
||
|
|
"step_time": 0.6454660271294415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.8,
|
||
|
|
"completions/max_terminated_length": 18.8,
|
||
|
|
"completions/mean_length": 12.45,
|
||
|
|
"completions/mean_terminated_length": 12.45,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.8313162535429001,
|
||
|
|
"epoch": 1.4742268041237114,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 9.843492193337923e-08,
|
||
|
|
"loss": 0.10078572034835816,
|
||
|
|
"num_tokens": 367866.0,
|
||
|
|
"reward": -0.19883333146572113,
|
||
|
|
"reward_std": 0.5281381130218505,
|
||
|
|
"rewards/reward_fn/mean": -0.19883333146572113,
|
||
|
|
"rewards/reward_fn/std": 0.5281381130218505,
|
||
|
|
"step": 1430,
|
||
|
|
"step_time": 0.6323665215633809
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 7.4,
|
||
|
|
"completions/max_terminated_length": 7.4,
|
||
|
|
"completions/mean_length": 6.0,
|
||
|
|
"completions/mean_terminated_length": 6.0,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 0.7071571081876755,
|
||
|
|
"epoch": 1.4793814432989691,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 9.665241799329096e-08,
|
||
|
|
"loss": 0.05014275312423706,
|
||
|
|
"num_tokens": 368998.0,
|
||
|
|
"reward": 0.27749999761581423,
|
||
|
|
"reward_std": 0.4774251878261566,
|
||
|
|
"rewards/reward_fn/mean": 0.27749999761581423,
|
||
|
|
"rewards/reward_fn/std": 0.4774251878261566,
|
||
|
|
"step": 1435,
|
||
|
|
"step_time": 0.46755328895524145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.6,
|
||
|
|
"completions/max_terminated_length": 25.6,
|
||
|
|
"completions/mean_length": 12.1,
|
||
|
|
"completions/mean_terminated_length": 12.1,
|
||
|
|
"completions/min_length": 4.6,
|
||
|
|
"completions/min_terminated_length": 4.6,
|
||
|
|
"entropy": 0.8592796742916107,
|
||
|
|
"epoch": 1.4845360824742269,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.625,
|
||
|
|
"learning_rate": 9.488232558197731e-08,
|
||
|
|
"loss": 0.217482328414917,
|
||
|
|
"num_tokens": 370272.0,
|
||
|
|
"reward": -0.3266666650772095,
|
||
|
|
"reward_std": 0.3772513210773468,
|
||
|
|
"rewards/reward_fn/mean": -0.3266666650772095,
|
||
|
|
"rewards/reward_fn/std": 0.37725133299827573,
|
||
|
|
"step": 1440,
|
||
|
|
"step_time": 0.7577955449931324
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.6,
|
||
|
|
"completions/max_terminated_length": 16.6,
|
||
|
|
"completions/mean_length": 12.85,
|
||
|
|
"completions/mean_terminated_length": 12.85,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.6260365337133408,
|
||
|
|
"epoch": 1.4896907216494846,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 21.875,
|
||
|
|
"learning_rate": 9.312478796581791e-08,
|
||
|
|
"loss": 0.1230657935142517,
|
||
|
|
"num_tokens": 371541.0,
|
||
|
|
"reward": -0.46194444596767426,
|
||
|
|
"reward_std": 0.3338461399078369,
|
||
|
|
"rewards/reward_fn/mean": -0.46194444596767426,
|
||
|
|
"rewards/reward_fn/std": 0.33384612798690794,
|
||
|
|
"step": 1445,
|
||
|
|
"step_time": 0.5881177311763167
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 8.4,
|
||
|
|
"completions/max_terminated_length": 8.4,
|
||
|
|
"completions/mean_length": 6.35,
|
||
|
|
"completions/mean_terminated_length": 6.35,
|
||
|
|
"completions/min_length": 4.8,
|
||
|
|
"completions/min_terminated_length": 4.8,
|
||
|
|
"entropy": 1.007636147737503,
|
||
|
|
"epoch": 1.4948453608247423,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 9.137994739504199e-08,
|
||
|
|
"loss": 0.014282146096229553,
|
||
|
|
"num_tokens": 372688.0,
|
||
|
|
"reward": -0.31999999582767485,
|
||
|
|
"reward_std": 0.0800000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.31999999582767485,
|
||
|
|
"rewards/reward_fn/std": 0.0800000011920929,
|
||
|
|
"step": 1450,
|
||
|
|
"step_time": 0.4345912431366742
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 8.4,
|
||
|
|
"completions/max_terminated_length": 8.4,
|
||
|
|
"completions/mean_length": 6.8,
|
||
|
|
"completions/mean_terminated_length": 6.8,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 0.5342540591955185,
|
||
|
|
"epoch": 1.5,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 37.5,
|
||
|
|
"learning_rate": 8.964794509221507e-08,
|
||
|
|
"loss": 0.10609520673751831,
|
||
|
|
"num_tokens": 373848.0,
|
||
|
|
"reward": -0.0625,
|
||
|
|
"reward_std": 0.39801986813545226,
|
||
|
|
"rewards/reward_fn/mean": -0.0625,
|
||
|
|
"rewards/reward_fn/std": 0.3980198800563812,
|
||
|
|
"step": 1455,
|
||
|
|
"step_time": 0.5045116319321096
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.0,
|
||
|
|
"completions/max_terminated_length": 10.0,
|
||
|
|
"completions/mean_length": 7.8,
|
||
|
|
"completions/mean_terminated_length": 7.8,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.7498216450214386,
|
||
|
|
"epoch": 1.5051546391752577,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.79289212408088e-08,
|
||
|
|
"loss": 0.04797559082508087,
|
||
|
|
"num_tokens": 375004.0,
|
||
|
|
"reward": -0.28333333432674407,
|
||
|
|
"reward_std": 0.28076876401901246,
|
||
|
|
"rewards/reward_fn/mean": -0.28333333432674407,
|
||
|
|
"rewards/reward_fn/std": 0.28076878786087034,
|
||
|
|
"step": 1460,
|
||
|
|
"step_time": 0.4600403653457761
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.4,
|
||
|
|
"completions/max_terminated_length": 17.4,
|
||
|
|
"completions/mean_length": 12.9,
|
||
|
|
"completions/mean_terminated_length": 12.9,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.8871105581521987,
|
||
|
|
"epoch": 1.5103092783505154,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.622301497385506e-08,
|
||
|
|
"loss": -0.021851116418838502,
|
||
|
|
"num_tokens": 376282.0,
|
||
|
|
"reward": 0.06249999701976776,
|
||
|
|
"reward_std": 0.29870229959487915,
|
||
|
|
"rewards/reward_fn/mean": 0.06249999701976776,
|
||
|
|
"rewards/reward_fn/std": 0.2987023115158081,
|
||
|
|
"step": 1465,
|
||
|
|
"step_time": 0.6089170153252781
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.0,
|
||
|
|
"completions/max_terminated_length": 16.0,
|
||
|
|
"completions/mean_length": 11.45,
|
||
|
|
"completions/mean_terminated_length": 11.45,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.88620465695858,
|
||
|
|
"epoch": 1.5154639175257731,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 18.125,
|
||
|
|
"learning_rate": 8.453036436268457e-08,
|
||
|
|
"loss": 0.03725805878639221,
|
||
|
|
"num_tokens": 377543.0,
|
||
|
|
"reward": -0.3990000009536743,
|
||
|
|
"reward_std": 0.3792510271072388,
|
||
|
|
"rewards/reward_fn/mean": -0.3990000009536743,
|
||
|
|
"rewards/reward_fn/std": 0.3792510271072388,
|
||
|
|
"step": 1470,
|
||
|
|
"step_time": 0.5841910772025585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 49.6,
|
||
|
|
"completions/max_terminated_length": 49.6,
|
||
|
|
"completions/mean_length": 20.3,
|
||
|
|
"completions/mean_terminated_length": 20.3,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.589074644446373,
|
||
|
|
"epoch": 1.5206185567010309,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.285110640575199e-08,
|
||
|
|
"loss": -0.00042560696601867676,
|
||
|
|
"num_tokens": 378973.0,
|
||
|
|
"reward": -0.5600000023841858,
|
||
|
|
"reward_std": 0.21528346538543702,
|
||
|
|
"rewards/reward_fn/mean": -0.5600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.21528346538543702,
|
||
|
|
"step": 1475,
|
||
|
|
"step_time": 1.345842866692692
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 43.6,
|
||
|
|
"completions/max_terminated_length": 43.6,
|
||
|
|
"completions/mean_length": 20.05,
|
||
|
|
"completions/mean_terminated_length": 20.05,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 1.0215669929981233,
|
||
|
|
"epoch": 1.5257731958762886,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 10.6875,
|
||
|
|
"learning_rate": 8.118537701754766e-08,
|
||
|
|
"loss": 0.04796600043773651,
|
||
|
|
"num_tokens": 380426.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 1480,
|
||
|
|
"step_time": 1.1404463457874954
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.6,
|
||
|
|
"completions/max_terminated_length": 27.6,
|
||
|
|
"completions/mean_length": 13.8,
|
||
|
|
"completions/mean_terminated_length": 13.8,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.6877721130847931,
|
||
|
|
"epoch": 1.5309278350515463,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 20.375,
|
||
|
|
"learning_rate": 7.953331101759705e-08,
|
||
|
|
"loss": 0.06992399096488952,
|
||
|
|
"num_tokens": 381722.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.1154700517654419,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.1154700517654419,
|
||
|
|
"step": 1485,
|
||
|
|
"step_time": 0.8708475268445909
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.0,
|
||
|
|
"completions/max_terminated_length": 22.0,
|
||
|
|
"completions/mean_length": 12.35,
|
||
|
|
"completions/mean_terminated_length": 12.35,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.8309216499328613,
|
||
|
|
"epoch": 1.536082474226804,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 11.4375,
|
||
|
|
"learning_rate": 7.789504211954865e-08,
|
||
|
|
"loss": -0.06701267361640931,
|
||
|
|
"num_tokens": 382973.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 1490,
|
||
|
|
"step_time": 0.7045149521902203
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.25,
|
||
|
|
"completions/mean_terminated_length": 10.25,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.8640795558691025,
|
||
|
|
"epoch": 1.5412371134020617,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 7.627070292035201e-08,
|
||
|
|
"loss": -0.02734326720237732,
|
||
|
|
"num_tokens": 384206.0,
|
||
|
|
"reward": -0.39000000059604645,
|
||
|
|
"reward_std": 0.2600000023841858,
|
||
|
|
"rewards/reward_fn/mean": -0.39000000059604645,
|
||
|
|
"rewards/reward_fn/std": 0.2600000023841858,
|
||
|
|
"step": 1495,
|
||
|
|
"step_time": 0.5708974854089319
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.0,
|
||
|
|
"completions/max_terminated_length": 18.0,
|
||
|
|
"completions/mean_length": 11.4,
|
||
|
|
"completions/mean_terminated_length": 11.4,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.948620742559433,
|
||
|
|
"epoch": 1.5463917525773194,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 7.46604248895252e-08,
|
||
|
|
"loss": -0.024460136890411377,
|
||
|
|
"num_tokens": 385470.0,
|
||
|
|
"reward": -0.4800000011920929,
|
||
|
|
"reward_std": 0.27285742163658144,
|
||
|
|
"rewards/reward_fn/mean": -0.4800000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.27285742163658144,
|
||
|
|
"step": 1500,
|
||
|
|
"step_time": 0.7104676578193903
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.4,
|
||
|
|
"completions/max_terminated_length": 11.4,
|
||
|
|
"completions/mean_length": 8.3,
|
||
|
|
"completions/mean_terminated_length": 8.3,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.8726276814937591,
|
||
|
|
"epoch": 1.5515463917525774,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 7.306433835851422e-08,
|
||
|
|
"loss": 0.10536327362060546,
|
||
|
|
"num_tokens": 386652.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.05,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.05,
|
||
|
|
"step": 1505,
|
||
|
|
"step_time": 0.4956268293783069
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 7.35,
|
||
|
|
"completions/mean_terminated_length": 7.35,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 1.0207952618598939,
|
||
|
|
"epoch": 1.556701030927835,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 7.148257251014453e-08,
|
||
|
|
"loss": 0.024337270855903627,
|
||
|
|
"num_tokens": 387815.0,
|
||
|
|
"reward": -0.4983333319425583,
|
||
|
|
"reward_std": 0.20333333015441896,
|
||
|
|
"rewards/reward_fn/mean": -0.4983333319425583,
|
||
|
|
"rewards/reward_fn/std": 0.2033333420753479,
|
||
|
|
"step": 1510,
|
||
|
|
"step_time": 0.5121777442283928
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 7.8,
|
||
|
|
"completions/mean_terminated_length": 7.8,
|
||
|
|
"completions/min_length": 4.4,
|
||
|
|
"completions/min_terminated_length": 4.4,
|
||
|
|
"entropy": 1.017931866645813,
|
||
|
|
"epoch": 1.5618556701030928,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 32.5,
|
||
|
|
"learning_rate": 6.991525536816497e-08,
|
||
|
|
"loss": 0.13234100341796876,
|
||
|
|
"num_tokens": 388979.0,
|
||
|
|
"reward": -0.17000000178813934,
|
||
|
|
"reward_std": 0.33085031509399415,
|
||
|
|
"rewards/reward_fn/mean": -0.17000000178813934,
|
||
|
|
"rewards/reward_fn/std": 0.33085031509399415,
|
||
|
|
"step": 1515,
|
||
|
|
"step_time": 0.5632654077373445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 7.6,
|
||
|
|
"completions/max_terminated_length": 7.6,
|
||
|
|
"completions/mean_length": 6.35,
|
||
|
|
"completions/mean_terminated_length": 6.35,
|
||
|
|
"completions/min_length": 5.2,
|
||
|
|
"completions/min_terminated_length": 5.2,
|
||
|
|
"entropy": 0.9235482051968574,
|
||
|
|
"epoch": 1.5670103092783505,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 29.125,
|
||
|
|
"learning_rate": 6.836251378688631e-08,
|
||
|
|
"loss": 0.10203149318695068,
|
||
|
|
"num_tokens": 390118.0,
|
||
|
|
"reward": -0.3066666662693024,
|
||
|
|
"reward_std": 0.3993190586566925,
|
||
|
|
"rewards/reward_fn/mean": -0.3066666662693024,
|
||
|
|
"rewards/reward_fn/std": 0.3993190586566925,
|
||
|
|
"step": 1520,
|
||
|
|
"step_time": 0.49033016450703143
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05,
|
||
|
|
"completions/max_length": 65.2,
|
||
|
|
"completions/max_terminated_length": 18.6,
|
||
|
|
"completions/mean_length": 25.65,
|
||
|
|
"completions/mean_terminated_length": 13.7,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.6301730372011661,
|
||
|
|
"epoch": 1.5721649484536082,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 15.5,
|
||
|
|
"learning_rate": 6.682447344091363e-08,
|
||
|
|
"loss": 0.14748450517654418,
|
||
|
|
"num_tokens": 391663.0,
|
||
|
|
"reward": -0.43458333015441897,
|
||
|
|
"reward_std": 0.29935399293899534,
|
||
|
|
"rewards/reward_fn/mean": -0.43458333015441897,
|
||
|
|
"rewards/reward_fn/std": 0.2993540048599243,
|
||
|
|
"step": 1525,
|
||
|
|
"step_time": 1.5304752157069743
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.8,
|
||
|
|
"completions/max_terminated_length": 24.8,
|
||
|
|
"completions/mean_length": 15.95,
|
||
|
|
"completions/mean_terminated_length": 15.95,
|
||
|
|
"completions/min_length": 9.6,
|
||
|
|
"completions/min_terminated_length": 9.6,
|
||
|
|
"entropy": 1.306685358285904,
|
||
|
|
"epoch": 1.577319587628866,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.530125881497473e-08,
|
||
|
|
"loss": 0.06663668155670166,
|
||
|
|
"num_tokens": 393002.0,
|
||
|
|
"reward": -0.3899999976158142,
|
||
|
|
"reward_std": 0.2558645486831665,
|
||
|
|
"rewards/reward_fn/mean": -0.3899999976158142,
|
||
|
|
"rewards/reward_fn/std": 0.2558645486831665,
|
||
|
|
"step": 1530,
|
||
|
|
"step_time": 0.8066560757346452
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.4,
|
||
|
|
"completions/max_terminated_length": 13.4,
|
||
|
|
"completions/mean_length": 10.75,
|
||
|
|
"completions/mean_terminated_length": 10.75,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.8304047286510468,
|
||
|
|
"epoch": 1.5824742268041239,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.37929931938447e-08,
|
||
|
|
"loss": 0.008444239944219589,
|
||
|
|
"num_tokens": 394249.0,
|
||
|
|
"reward": -0.32750000059604645,
|
||
|
|
"reward_std": 0.24915027618408203,
|
||
|
|
"rewards/reward_fn/mean": -0.32750000059604645,
|
||
|
|
"rewards/reward_fn/std": 0.24915027618408203,
|
||
|
|
"step": 1535,
|
||
|
|
"step_time": 0.5235663454048336
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 10.5,
|
||
|
|
"completions/mean_terminated_length": 10.5,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.7640501141548157,
|
||
|
|
"epoch": 1.5876288659793816,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.229979865236768e-08,
|
||
|
|
"loss": -0.030684226751327516,
|
||
|
|
"num_tokens": 395455.0,
|
||
|
|
"reward": -0.3512500047683716,
|
||
|
|
"reward_std": 0.19023503065109254,
|
||
|
|
"rewards/reward_fn/mean": -0.3512500047683716,
|
||
|
|
"rewards/reward_fn/std": 0.19023503065109254,
|
||
|
|
"step": 1540,
|
||
|
|
"step_time": 0.6071365992538631
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.2,
|
||
|
|
"completions/max_terminated_length": 20.2,
|
||
|
|
"completions/mean_length": 12.7,
|
||
|
|
"completions/mean_terminated_length": 12.7,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.9274045705795289,
|
||
|
|
"epoch": 1.5927835051546393,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 38.25,
|
||
|
|
"learning_rate": 6.082179604557616e-08,
|
||
|
|
"loss": 0.047584915161132814,
|
||
|
|
"num_tokens": 396741.0,
|
||
|
|
"reward": -0.11250001192092896,
|
||
|
|
"reward_std": 0.21273502707481384,
|
||
|
|
"rewards/reward_fn/mean": -0.11250001192092896,
|
||
|
|
"rewards/reward_fn/std": 0.21273502707481384,
|
||
|
|
"step": 1545,
|
||
|
|
"step_time": 0.6507150128483772
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.6,
|
||
|
|
"completions/max_terminated_length": 20.6,
|
||
|
|
"completions/mean_length": 13.25,
|
||
|
|
"completions/mean_terminated_length": 13.25,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.9783258199691772,
|
||
|
|
"epoch": 1.597938144329897,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.5625,
|
||
|
|
"learning_rate": 5.9359104998909617e-08,
|
||
|
|
"loss": 0.1127540111541748,
|
||
|
|
"num_tokens": 398022.0,
|
||
|
|
"reward": -0.27166666388511657,
|
||
|
|
"reward_std": 0.5395560979843139,
|
||
|
|
"rewards/reward_fn/mean": -0.27166666388511657,
|
||
|
|
"rewards/reward_fn/std": 0.5395561099052429,
|
||
|
|
"step": 1550,
|
||
|
|
"step_time": 0.7179786643013358
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 11.65,
|
||
|
|
"completions/mean_terminated_length": 11.65,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.7478360235691071,
|
||
|
|
"epoch": 1.6030927835051547,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 8.8125,
|
||
|
|
"learning_rate": 5.791184389853212e-08,
|
||
|
|
"loss": 0.02044408619403839,
|
||
|
|
"num_tokens": 399287.0,
|
||
|
|
"reward": -0.23333333432674408,
|
||
|
|
"reward_std": 0.5817121744155884,
|
||
|
|
"rewards/reward_fn/mean": -0.23333333432674408,
|
||
|
|
"rewards/reward_fn/std": 0.5817121982574462,
|
||
|
|
"step": 1555,
|
||
|
|
"step_time": 0.5401966189965606
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 24.6,
|
||
|
|
"completions/max_terminated_length": 24.6,
|
||
|
|
"completions/mean_length": 11.9,
|
||
|
|
"completions/mean_terminated_length": 11.9,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 1.0005066066980361,
|
||
|
|
"epoch": 1.6082474226804124,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 5.648012988175074e-08,
|
||
|
|
"loss": 0.07772735953330993,
|
||
|
|
"num_tokens": 400533.0,
|
||
|
|
"reward": -0.325,
|
||
|
|
"reward_std": 0.3809401035308838,
|
||
|
|
"rewards/reward_fn/mean": -0.325,
|
||
|
|
"rewards/reward_fn/std": 0.3809401035308838,
|
||
|
|
"step": 1560,
|
||
|
|
"step_time": 0.7347628603689372
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 45.2,
|
||
|
|
"completions/max_terminated_length": 45.2,
|
||
|
|
"completions/mean_length": 20.0,
|
||
|
|
"completions/mean_terminated_length": 20.0,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.878107276558876,
|
||
|
|
"epoch": 1.6134020618556701,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 5.5064078827534554e-08,
|
||
|
|
"loss": -0.0046801909804344176,
|
||
|
|
"num_tokens": 401977.0,
|
||
|
|
"reward": -0.527999997138977,
|
||
|
|
"reward_std": 0.20173502564430237,
|
||
|
|
"rewards/reward_fn/mean": -0.527999997138977,
|
||
|
|
"rewards/reward_fn/std": 0.20173502564430237,
|
||
|
|
"step": 1565,
|
||
|
|
"step_time": 1.186389725934714
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 9.4,
|
||
|
|
"completions/mean_terminated_length": 9.4,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 0.9705159187316894,
|
||
|
|
"epoch": 1.6185567010309279,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 19.125,
|
||
|
|
"learning_rate": 5.36638053471358e-08,
|
||
|
|
"loss": -0.07211000919342041,
|
||
|
|
"num_tokens": 403185.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.18773502707481385,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.18773502707481385,
|
||
|
|
"step": 1570,
|
||
|
|
"step_time": 0.5305851818062365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.2,
|
||
|
|
"completions/max_terminated_length": 23.2,
|
||
|
|
"completions/mean_length": 15.4,
|
||
|
|
"completions/mean_terminated_length": 15.4,
|
||
|
|
"completions/min_length": 9.4,
|
||
|
|
"completions/min_terminated_length": 9.4,
|
||
|
|
"entropy": 0.7032105147838592,
|
||
|
|
"epoch": 1.6237113402061856,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 5.227942277481362e-08,
|
||
|
|
"loss": 0.010414773225784301,
|
||
|
|
"num_tokens": 404513.0,
|
||
|
|
"reward": -0.19199999868869783,
|
||
|
|
"reward_std": 0.2105449080467224,
|
||
|
|
"rewards/reward_fn/mean": -0.19199999868869783,
|
||
|
|
"rewards/reward_fn/std": 0.2105449080467224,
|
||
|
|
"step": 1575,
|
||
|
|
"step_time": 0.6994988144375384
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 9.15,
|
||
|
|
"completions/mean_terminated_length": 9.15,
|
||
|
|
"completions/min_length": 4.4,
|
||
|
|
"completions/min_terminated_length": 4.4,
|
||
|
|
"entropy": 1.0182935357093812,
|
||
|
|
"epoch": 1.6288659793814433,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 19.25,
|
||
|
|
"learning_rate": 5.091104315866101e-08,
|
||
|
|
"loss": 0.27594072818756105,
|
||
|
|
"num_tokens": 405704.0,
|
||
|
|
"reward": -0.4875000029802322,
|
||
|
|
"reward_std": 0.36525567770004275,
|
||
|
|
"rewards/reward_fn/mean": -0.4875000029802322,
|
||
|
|
"rewards/reward_fn/std": 0.36525567770004275,
|
||
|
|
"step": 1580,
|
||
|
|
"step_time": 0.6210156822577119
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.2,
|
||
|
|
"completions/max_terminated_length": 25.2,
|
||
|
|
"completions/mean_length": 12.6,
|
||
|
|
"completions/mean_terminated_length": 12.6,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.6944295287132263,
|
||
|
|
"epoch": 1.634020618556701,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.955877725153604e-08,
|
||
|
|
"loss": 0.25443143844604493,
|
||
|
|
"num_tokens": 406960.0,
|
||
|
|
"reward": -0.4175000011920929,
|
||
|
|
"reward_std": 0.3398511052131653,
|
||
|
|
"rewards/reward_fn/mean": -0.4175000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3398511052131653,
|
||
|
|
"step": 1585,
|
||
|
|
"step_time": 0.7473113781772553
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 50.0,
|
||
|
|
"completions/max_terminated_length": 50.0,
|
||
|
|
"completions/mean_length": 21.4,
|
||
|
|
"completions/mean_terminated_length": 21.4,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.065959817171097,
|
||
|
|
"epoch": 1.6391752577319587,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 22.375,
|
||
|
|
"learning_rate": 4.8222734502097655e-08,
|
||
|
|
"loss": 0.39557130336761476,
|
||
|
|
"num_tokens": 408396.0,
|
||
|
|
"reward": -0.5983333349227905,
|
||
|
|
"reward_std": 0.336662220954895,
|
||
|
|
"rewards/reward_fn/mean": -0.5983333349227905,
|
||
|
|
"rewards/reward_fn/std": 0.336662220954895,
|
||
|
|
"step": 1590,
|
||
|
|
"step_time": 1.2763810374774038
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 18.4,
|
||
|
|
"completions/max_terminated_length": 18.4,
|
||
|
|
"completions/mean_length": 9.7,
|
||
|
|
"completions/mean_terminated_length": 9.7,
|
||
|
|
"completions/min_length": 4.8,
|
||
|
|
"completions/min_terminated_length": 4.8,
|
||
|
|
"entropy": 0.8178534567356109,
|
||
|
|
"epoch": 1.6443298969072164,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 24.75,
|
||
|
|
"learning_rate": 4.690302304594754e-08,
|
||
|
|
"loss": 0.16044366359710693,
|
||
|
|
"num_tokens": 409602.0,
|
||
|
|
"reward": -0.23000001013278962,
|
||
|
|
"reward_std": 0.2588886618614197,
|
||
|
|
"rewards/reward_fn/mean": -0.23000001013278962,
|
||
|
|
"rewards/reward_fn/std": 0.2588886618614197,
|
||
|
|
"step": 1595,
|
||
|
|
"step_time": 0.628109234571457
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.6,
|
||
|
|
"completions/max_terminated_length": 12.6,
|
||
|
|
"completions/mean_length": 9.7,
|
||
|
|
"completions/mean_terminated_length": 9.7,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 1.0267748922109603,
|
||
|
|
"epoch": 1.6494845360824741,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 20.5,
|
||
|
|
"learning_rate": 4.559974969687755e-08,
|
||
|
|
"loss": 0.053169417381286624,
|
||
|
|
"num_tokens": 410820.0,
|
||
|
|
"reward": -0.5100000023841857,
|
||
|
|
"reward_std": 0.13773502707481383,
|
||
|
|
"rewards/reward_fn/mean": -0.5100000023841857,
|
||
|
|
"rewards/reward_fn/std": 0.13773502707481383,
|
||
|
|
"step": 1600,
|
||
|
|
"step_time": 0.5113442252390087
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.6,
|
||
|
|
"completions/max_terminated_length": 11.6,
|
||
|
|
"completions/mean_length": 7.3,
|
||
|
|
"completions/mean_terminated_length": 7.3,
|
||
|
|
"completions/min_length": 5.0,
|
||
|
|
"completions/min_terminated_length": 5.0,
|
||
|
|
"entropy": 0.8996017038822174,
|
||
|
|
"epoch": 1.6546391752577319,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.43130199382247e-08,
|
||
|
|
"loss": 0.21870932579040528,
|
||
|
|
"num_tokens": 411974.0,
|
||
|
|
"reward": -0.28500000238418577,
|
||
|
|
"reward_std": 0.4861552774906158,
|
||
|
|
"rewards/reward_fn/mean": -0.28500000238418577,
|
||
|
|
"rewards/reward_fn/std": 0.4861552774906158,
|
||
|
|
"step": 1605,
|
||
|
|
"step_time": 0.49553243080154064
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.4,
|
||
|
|
"completions/max_terminated_length": 10.4,
|
||
|
|
"completions/mean_length": 7.25,
|
||
|
|
"completions/mean_terminated_length": 7.25,
|
||
|
|
"completions/min_length": 5.0,
|
||
|
|
"completions/min_terminated_length": 5.0,
|
||
|
|
"entropy": 0.769911852478981,
|
||
|
|
"epoch": 1.6597938144329896,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 36.5,
|
||
|
|
"learning_rate": 4.304293791433358e-08,
|
||
|
|
"loss": 0.06664889454841613,
|
||
|
|
"num_tokens": 413139.0,
|
||
|
|
"reward": -0.325,
|
||
|
|
"reward_std": 0.25,
|
||
|
|
"rewards/reward_fn/mean": -0.325,
|
||
|
|
"rewards/reward_fn/std": 0.25,
|
||
|
|
"step": 1610,
|
||
|
|
"step_time": 0.5342640136368573
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.4,
|
||
|
|
"completions/max_terminated_length": 25.4,
|
||
|
|
"completions/mean_length": 15.65,
|
||
|
|
"completions/mean_terminated_length": 15.65,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 1.128392007946968,
|
||
|
|
"epoch": 1.6649484536082473,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 14.25,
|
||
|
|
"learning_rate": 4.1789606422127226e-08,
|
||
|
|
"loss": 0.07596928477287293,
|
||
|
|
"num_tokens": 414492.0,
|
||
|
|
"reward": -0.3168181836605072,
|
||
|
|
"reward_std": 0.353405237197876,
|
||
|
|
"rewards/reward_fn/mean": -0.3168181836605072,
|
||
|
|
"rewards/reward_fn/std": 0.353405225276947,
|
||
|
|
"step": 1615,
|
||
|
|
"step_time": 0.745933323726058
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.0,
|
||
|
|
"completions/max_terminated_length": 25.0,
|
||
|
|
"completions/mean_length": 17.4,
|
||
|
|
"completions/mean_terminated_length": 17.4,
|
||
|
|
"completions/min_length": 9.2,
|
||
|
|
"completions/min_terminated_length": 9.2,
|
||
|
|
"entropy": 0.685271966457367,
|
||
|
|
"epoch": 1.670103092783505,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 7.78125,
|
||
|
|
"learning_rate": 4.0553126902787005e-08,
|
||
|
|
"loss": 0.08662630915641785,
|
||
|
|
"num_tokens": 415836.0,
|
||
|
|
"reward": -0.38583333492279054,
|
||
|
|
"reward_std": 0.3215514063835144,
|
||
|
|
"rewards/reward_fn/mean": -0.38583333492279054,
|
||
|
|
"rewards/reward_fn/std": 0.3215514302253723,
|
||
|
|
"step": 1620,
|
||
|
|
"step_time": 0.7388565543107688
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 30.4,
|
||
|
|
"completions/max_terminated_length": 30.4,
|
||
|
|
"completions/mean_length": 14.75,
|
||
|
|
"completions/mean_terminated_length": 14.75,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 1.1614010393619538,
|
||
|
|
"epoch": 1.675257731958763,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 27.375,
|
||
|
|
"learning_rate": 3.933359943354228e-08,
|
||
|
|
"loss": 0.12167187929153442,
|
||
|
|
"num_tokens": 417167.0,
|
||
|
|
"reward": -0.2925000011920929,
|
||
|
|
"reward_std": 0.45460320115089414,
|
||
|
|
"rewards/reward_fn/mean": -0.2925000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.45460320115089414,
|
||
|
|
"step": 1625,
|
||
|
|
"step_time": 0.903398260846734
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.6,
|
||
|
|
"completions/max_terminated_length": 16.6,
|
||
|
|
"completions/mean_length": 10.45,
|
||
|
|
"completions/mean_terminated_length": 10.45,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.0969272643327712,
|
||
|
|
"epoch": 1.6804123711340206,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.813112271957039e-08,
|
||
|
|
"loss": 0.14156380891799927,
|
||
|
|
"num_tokens": 418412.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 1630,
|
||
|
|
"step_time": 0.5806521772406995
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.8,
|
||
|
|
"completions/max_terminated_length": 12.8,
|
||
|
|
"completions/mean_length": 9.35,
|
||
|
|
"completions/mean_terminated_length": 9.35,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 0.7530741691589355,
|
||
|
|
"epoch": 1.6855670103092784,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 53.75,
|
||
|
|
"learning_rate": 3.6945794086007705e-08,
|
||
|
|
"loss": 0.0776121437549591,
|
||
|
|
"num_tokens": 419619.0,
|
||
|
|
"reward": -0.1850000023841858,
|
||
|
|
"reward_std": 0.36603364944458006,
|
||
|
|
"rewards/reward_fn/mean": -0.1850000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.36603364944458006,
|
||
|
|
"step": 1635,
|
||
|
|
"step_time": 0.5080023482441902
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 27.6,
|
||
|
|
"completions/max_terminated_length": 27.6,
|
||
|
|
"completions/mean_length": 17.8,
|
||
|
|
"completions/mean_terminated_length": 17.8,
|
||
|
|
"completions/min_length": 10.6,
|
||
|
|
"completions/min_terminated_length": 10.6,
|
||
|
|
"entropy": 0.9120415449142456,
|
||
|
|
"epoch": 1.690721649484536,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 12.6875,
|
||
|
|
"learning_rate": 3.577770947007253e-08,
|
||
|
|
"loss": 0.12926599979400635,
|
||
|
|
"num_tokens": 421003.0,
|
||
|
|
"reward": -0.3950000047683716,
|
||
|
|
"reward_std": 0.29712470173835753,
|
||
|
|
"rewards/reward_fn/mean": -0.3950000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.29712470173835753,
|
||
|
|
"step": 1640,
|
||
|
|
"step_time": 0.8596902235411108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.6,
|
||
|
|
"completions/max_terminated_length": 10.6,
|
||
|
|
"completions/mean_length": 8.75,
|
||
|
|
"completions/mean_terminated_length": 8.75,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.6245357424020768,
|
||
|
|
"epoch": 1.6958762886597938,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 3.462696341329996e-08,
|
||
|
|
"loss": 0.09731593132019042,
|
||
|
|
"num_tokens": 422210.0,
|
||
|
|
"reward": -0.21000000238418579,
|
||
|
|
"reward_std": 0.43075350522994993,
|
||
|
|
"rewards/reward_fn/mean": -0.21000000238418579,
|
||
|
|
"rewards/reward_fn/std": 0.4307535171508789,
|
||
|
|
"step": 1645,
|
||
|
|
"step_time": 0.4654439380392432
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.8,
|
||
|
|
"completions/max_terminated_length": 21.8,
|
||
|
|
"completions/mean_length": 12.3,
|
||
|
|
"completions/mean_terminated_length": 12.3,
|
||
|
|
"completions/min_length": 6.0,
|
||
|
|
"completions/min_terminated_length": 6.0,
|
||
|
|
"entropy": 1.0625550508499146,
|
||
|
|
"epoch": 1.7010309278350515,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 24.25,
|
||
|
|
"learning_rate": 3.349364905389032e-08,
|
||
|
|
"loss": 0.025496214628219604,
|
||
|
|
"num_tokens": 423480.0,
|
||
|
|
"reward": -0.4909090906381607,
|
||
|
|
"reward_std": 0.218181836605072,
|
||
|
|
"rewards/reward_fn/mean": -0.4909090906381607,
|
||
|
|
"rewards/reward_fn/std": 0.2181818127632141,
|
||
|
|
"step": 1650,
|
||
|
|
"step_time": 0.6790977258235216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.2,
|
||
|
|
"completions/max_terminated_length": 12.2,
|
||
|
|
"completions/mean_length": 9.45,
|
||
|
|
"completions/mean_terminated_length": 9.45,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.7438314110040665,
|
||
|
|
"epoch": 1.7061855670103094,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 20.75,
|
||
|
|
"learning_rate": 3.237785811917049e-08,
|
||
|
|
"loss": 0.05355839133262634,
|
||
|
|
"num_tokens": 424693.0,
|
||
|
|
"reward": -0.5550000011920929,
|
||
|
|
"reward_std": 0.3228574156761169,
|
||
|
|
"rewards/reward_fn/mean": -0.5550000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3228574156761169,
|
||
|
|
"step": 1655,
|
||
|
|
"step_time": 0.5530511302873492
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 10.15,
|
||
|
|
"completions/mean_terminated_length": 10.15,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.8672337532043457,
|
||
|
|
"epoch": 1.7113402061855671,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 17.0,
|
||
|
|
"learning_rate": 3.127968091816985e-08,
|
||
|
|
"loss": 0.12111912965774536,
|
||
|
|
"num_tokens": 425920.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.10773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.10773502588272095,
|
||
|
|
"step": 1660,
|
||
|
|
"step_time": 0.5605017002671957
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.8,
|
||
|
|
"completions/max_terminated_length": 14.8,
|
||
|
|
"completions/mean_length": 10.65,
|
||
|
|
"completions/mean_terminated_length": 10.65,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.7234238475561142,
|
||
|
|
"epoch": 1.7164948453608249,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 18.125,
|
||
|
|
"learning_rate": 3.0199206334310945e-08,
|
||
|
|
"loss": -0.043664944171905515,
|
||
|
|
"num_tokens": 427157.0,
|
||
|
|
"reward": -0.10833333134651184,
|
||
|
|
"reward_std": 0.10754843950271606,
|
||
|
|
"rewards/reward_fn/mean": -0.10833333134651184,
|
||
|
|
"rewards/reward_fn/std": 0.10754843950271606,
|
||
|
|
"step": 1665,
|
||
|
|
"step_time": 0.5437676094472408
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 38.6,
|
||
|
|
"completions/max_terminated_length": 38.6,
|
||
|
|
"completions/mean_length": 18.1,
|
||
|
|
"completions/mean_terminated_length": 18.1,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 1.1868691265583038,
|
||
|
|
"epoch": 1.7216494845360826,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 15.5,
|
||
|
|
"learning_rate": 2.913652181821552e-08,
|
||
|
|
"loss": 0.12590534687042237,
|
||
|
|
"num_tokens": 428543.0,
|
||
|
|
"reward": -0.3076666623353958,
|
||
|
|
"reward_std": 0.4518565535545349,
|
||
|
|
"rewards/reward_fn/mean": -0.3076666623353958,
|
||
|
|
"rewards/reward_fn/std": 0.4518565535545349,
|
||
|
|
"step": 1670,
|
||
|
|
"step_time": 1.0609127783216536
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.0,
|
||
|
|
"completions/max_terminated_length": 20.0,
|
||
|
|
"completions/mean_length": 11.25,
|
||
|
|
"completions/mean_terminated_length": 11.25,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 1.1474024474620819,
|
||
|
|
"epoch": 1.7268041237113403,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.8091713380626493e-08,
|
||
|
|
"loss": -0.016077935695648193,
|
||
|
|
"num_tokens": 429788.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 1675,
|
||
|
|
"step_time": 0.6428652757778763
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.8,
|
||
|
|
"completions/max_terminated_length": 13.8,
|
||
|
|
"completions/mean_length": 11.05,
|
||
|
|
"completions/mean_terminated_length": 11.05,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.8905054450035095,
|
||
|
|
"epoch": 1.731958762886598,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 24.25,
|
||
|
|
"learning_rate": 2.7064865585446433e-08,
|
||
|
|
"loss": 0.0036585628986358643,
|
||
|
|
"num_tokens": 431053.0,
|
||
|
|
"reward": -0.28500000536441805,
|
||
|
|
"reward_std": 0.4567333281040192,
|
||
|
|
"rewards/reward_fn/mean": -0.28500000536441805,
|
||
|
|
"rewards/reward_fn/std": 0.4567333281040192,
|
||
|
|
"step": 1680,
|
||
|
|
"step_time": 0.5253810392692685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 40.0,
|
||
|
|
"completions/max_terminated_length": 40.0,
|
||
|
|
"completions/mean_length": 16.8,
|
||
|
|
"completions/mean_terminated_length": 16.8,
|
||
|
|
"completions/min_length": 7.6,
|
||
|
|
"completions/min_terminated_length": 7.6,
|
||
|
|
"entropy": 0.8407724499702454,
|
||
|
|
"epoch": 1.7371134020618557,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 6.78125,
|
||
|
|
"learning_rate": 2.6056061542893222e-08,
|
||
|
|
"loss": -0.14318621158599854,
|
||
|
|
"num_tokens": 432389.0,
|
||
|
|
"reward": -0.4699999988079071,
|
||
|
|
"reward_std": 0.15011107325553893,
|
||
|
|
"rewards/reward_fn/mean": -0.4699999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.15011107325553893,
|
||
|
|
"step": 1685,
|
||
|
|
"step_time": 1.0814969715662301
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.8,
|
||
|
|
"completions/max_terminated_length": 25.8,
|
||
|
|
"completions/mean_length": 16.65,
|
||
|
|
"completions/mean_terminated_length": 16.65,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 0.9036176294088364,
|
||
|
|
"epoch": 1.7422680412371134,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.875,
|
||
|
|
"learning_rate": 2.5065382902773435e-08,
|
||
|
|
"loss": 0.17168867588043213,
|
||
|
|
"num_tokens": 433758.0,
|
||
|
|
"reward": -0.5699999988079071,
|
||
|
|
"reward_std": 0.2934023141860962,
|
||
|
|
"rewards/reward_fn/mean": -0.5699999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.29340232610702516,
|
||
|
|
"step": 1690,
|
||
|
|
"step_time": 0.7546438516117633
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.8,
|
||
|
|
"completions/max_terminated_length": 17.8,
|
||
|
|
"completions/mean_length": 13.45,
|
||
|
|
"completions/mean_terminated_length": 13.45,
|
||
|
|
"completions/min_length": 9.6,
|
||
|
|
"completions/min_terminated_length": 9.6,
|
||
|
|
"entropy": 0.9092921882867813,
|
||
|
|
"epoch": 1.7474226804123711,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.409290984787371e-08,
|
||
|
|
"loss": -0.08626242876052856,
|
||
|
|
"num_tokens": 435043.0,
|
||
|
|
"reward": -0.65,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.65,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 1695,
|
||
|
|
"step_time": 0.6577458389103412
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 21.4,
|
||
|
|
"completions/max_terminated_length": 21.4,
|
||
|
|
"completions/mean_length": 14.8,
|
||
|
|
"completions/mean_terminated_length": 14.8,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.6884515404701232,
|
||
|
|
"epoch": 1.7525773195876289,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 13.9375,
|
||
|
|
"learning_rate": 2.313872108747092e-08,
|
||
|
|
"loss": -0.023072260618209838,
|
||
|
|
"num_tokens": 436375.0,
|
||
|
|
"reward": -0.45,
|
||
|
|
"reward_std": 0.2,
|
||
|
|
"rewards/reward_fn/mean": -0.45,
|
||
|
|
"rewards/reward_fn/std": 0.2,
|
||
|
|
"step": 1700,
|
||
|
|
"step_time": 0.6705449435859918
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.8,
|
||
|
|
"completions/max_terminated_length": 19.8,
|
||
|
|
"completions/mean_length": 9.65,
|
||
|
|
"completions/mean_terminated_length": 9.65,
|
||
|
|
"completions/min_length": 4.2,
|
||
|
|
"completions/min_terminated_length": 4.2,
|
||
|
|
"entropy": 1.0143103897571564,
|
||
|
|
"epoch": 1.7577319587628866,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 9.125,
|
||
|
|
"learning_rate": 2.220289385096194e-08,
|
||
|
|
"loss": 0.18412907123565675,
|
||
|
|
"num_tokens": 437580.0,
|
||
|
|
"reward": -0.35999999940395355,
|
||
|
|
"reward_std": 0.3762573838233948,
|
||
|
|
"rewards/reward_fn/mean": -0.35999999940395355,
|
||
|
|
"rewards/reward_fn/std": 0.3762573838233948,
|
||
|
|
"step": 1705,
|
||
|
|
"step_time": 0.7051920691505075
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 10.6,
|
||
|
|
"completions/mean_terminated_length": 10.6,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.6810103952884674,
|
||
|
|
"epoch": 1.7628865979381443,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 17.0,
|
||
|
|
"learning_rate": 2.1285503881612627e-08,
|
||
|
|
"loss": -0.005234807729721069,
|
||
|
|
"num_tokens": 438788.0,
|
||
|
|
"reward": 0.1824999988079071,
|
||
|
|
"reward_std": 0.2471906840801239,
|
||
|
|
"rewards/reward_fn/mean": 0.1824999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.2471906840801239,
|
||
|
|
"step": 1710,
|
||
|
|
"step_time": 0.5405286100693047
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 11.0,
|
||
|
|
"completions/mean_terminated_length": 11.0,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.78079913854599,
|
||
|
|
"epoch": 1.768041237113402,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 15.375,
|
||
|
|
"learning_rate": 2.0386625430427434e-08,
|
||
|
|
"loss": 0.11818069219589233,
|
||
|
|
"num_tokens": 440024.0,
|
||
|
|
"reward": -0.5425000011920929,
|
||
|
|
"reward_std": 0.24758613109588623,
|
||
|
|
"rewards/reward_fn/mean": -0.5425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.24758613109588623,
|
||
|
|
"step": 1715,
|
||
|
|
"step_time": 0.6267786368727684
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.4,
|
||
|
|
"completions/max_terminated_length": 14.4,
|
||
|
|
"completions/mean_length": 10.3,
|
||
|
|
"completions/mean_terminated_length": 10.3,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.9158282101154327,
|
||
|
|
"epoch": 1.7731958762886597,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 1.9506331250139896e-08,
|
||
|
|
"loss": 0.15086617469787597,
|
||
|
|
"num_tokens": 441246.0,
|
||
|
|
"reward": -0.4775000035762787,
|
||
|
|
"reward_std": 0.37412620186805723,
|
||
|
|
"rewards/reward_fn/mean": -0.4775000035762787,
|
||
|
|
"rewards/reward_fn/std": 0.37412620186805723,
|
||
|
|
"step": 1720,
|
||
|
|
"step_time": 0.5440597709268331
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.4,
|
||
|
|
"completions/max_terminated_length": 12.4,
|
||
|
|
"completions/mean_length": 8.6,
|
||
|
|
"completions/mean_terminated_length": 8.6,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 0.7902150690555573,
|
||
|
|
"epoch": 1.7783505154639174,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.8644692589323967e-08,
|
||
|
|
"loss": 0.05799698829650879,
|
||
|
|
"num_tokens": 442418.0,
|
||
|
|
"reward": -0.44666666388511655,
|
||
|
|
"reward_std": 0.20370726585388182,
|
||
|
|
"rewards/reward_fn/mean": -0.44666666388511655,
|
||
|
|
"rewards/reward_fn/std": 0.2037072777748108,
|
||
|
|
"step": 1725,
|
||
|
|
"step_time": 0.5111340196803212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.4,
|
||
|
|
"completions/max_terminated_length": 16.4,
|
||
|
|
"completions/mean_length": 10.65,
|
||
|
|
"completions/mean_terminated_length": 10.65,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.6754415512084961,
|
||
|
|
"epoch": 1.7835051546391751,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 1.7801779186627648e-08,
|
||
|
|
"loss": 0.1614132523536682,
|
||
|
|
"num_tokens": 443643.0,
|
||
|
|
"reward": -0.45499999821186066,
|
||
|
|
"reward_std": 0.22999999523162842,
|
||
|
|
"rewards/reward_fn/mean": -0.45499999821186066,
|
||
|
|
"rewards/reward_fn/std": 0.23000000715255736,
|
||
|
|
"step": 1730,
|
||
|
|
"step_time": 0.6408032779581845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 30.6,
|
||
|
|
"completions/max_terminated_length": 30.6,
|
||
|
|
"completions/mean_length": 17.55,
|
||
|
|
"completions/mean_terminated_length": 17.55,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.8354218006134033,
|
||
|
|
"epoch": 1.7886597938144329,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.697765926512823e-08,
|
||
|
|
"loss": 0.02780742049217224,
|
||
|
|
"num_tokens": 445034.0,
|
||
|
|
"reward": -0.5012500047683716,
|
||
|
|
"reward_std": 0.1390668272972107,
|
||
|
|
"rewards/reward_fn/mean": -0.5012500047683716,
|
||
|
|
"rewards/reward_fn/std": 0.1390668272972107,
|
||
|
|
"step": 1735,
|
||
|
|
"step_time": 0.8523523797281086
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.6,
|
||
|
|
"completions/max_terminated_length": 11.6,
|
||
|
|
"completions/mean_length": 8.1,
|
||
|
|
"completions/mean_terminated_length": 8.1,
|
||
|
|
"completions/min_length": 5.6,
|
||
|
|
"completions/min_terminated_length": 5.6,
|
||
|
|
"entropy": 0.8720086574554443,
|
||
|
|
"epoch": 1.7938144329896906,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.617239952681082e-08,
|
||
|
|
"loss": 0.12526261806488037,
|
||
|
|
"num_tokens": 446220.0,
|
||
|
|
"reward": -0.3550000011920929,
|
||
|
|
"reward_std": 0.32516661286354065,
|
||
|
|
"rewards/reward_fn/mean": -0.3550000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.32516661286354065,
|
||
|
|
"step": 1740,
|
||
|
|
"step_time": 0.5580897599458694
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 12.1,
|
||
|
|
"completions/mean_terminated_length": 12.1,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.9813948631286621,
|
||
|
|
"epoch": 1.7989690721649485,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.5,
|
||
|
|
"learning_rate": 1.538606514716939e-08,
|
||
|
|
"loss": 0.0448380708694458,
|
||
|
|
"num_tokens": 447486.0,
|
||
|
|
"reward": -0.4350000023841858,
|
||
|
|
"reward_std": 0.3048511028289795,
|
||
|
|
"rewards/reward_fn/mean": -0.4350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.3048511028289795,
|
||
|
|
"step": 1745,
|
||
|
|
"step_time": 0.5642483637668192
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 25.6,
|
||
|
|
"completions/max_terminated_length": 25.6,
|
||
|
|
"completions/mean_length": 16.5,
|
||
|
|
"completions/mean_terminated_length": 16.5,
|
||
|
|
"completions/min_length": 10.0,
|
||
|
|
"completions/min_terminated_length": 10.0,
|
||
|
|
"entropy": 0.971468162536621,
|
||
|
|
"epoch": 1.8041237113402062,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 12.1875,
|
||
|
|
"learning_rate": 1.4618719769931843e-08,
|
||
|
|
"loss": 0.2018949031829834,
|
||
|
|
"num_tokens": 448864.0,
|
||
|
|
"reward": -0.5600000023841858,
|
||
|
|
"reward_std": 0.25754843950271605,
|
||
|
|
"rewards/reward_fn/mean": -0.5600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.25754843950271605,
|
||
|
|
"step": 1750,
|
||
|
|
"step_time": 0.8179386577568948
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 37.4,
|
||
|
|
"completions/max_terminated_length": 37.4,
|
||
|
|
"completions/mean_length": 18.5,
|
||
|
|
"completions/mean_terminated_length": 18.5,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.6841971307992936,
|
||
|
|
"epoch": 1.809278350515464,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 1.3870425501908672e-08,
|
||
|
|
"loss": 0.1817900061607361,
|
||
|
|
"num_tokens": 450234.0,
|
||
|
|
"reward": -0.4199999988079071,
|
||
|
|
"reward_std": 0.21237728595733643,
|
||
|
|
"rewards/reward_fn/mean": -0.4199999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.21237727403640747,
|
||
|
|
"step": 1755,
|
||
|
|
"step_time": 1.0719246904365718
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 20.4,
|
||
|
|
"completions/max_terminated_length": 20.4,
|
||
|
|
"completions/mean_length": 12.0,
|
||
|
|
"completions/mean_terminated_length": 12.0,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.6847515821456909,
|
||
|
|
"epoch": 1.8144329896907216,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.314124290796656e-08,
|
||
|
|
"loss": 0.15299254655838013,
|
||
|
|
"num_tokens": 451514.0,
|
||
|
|
"reward": -0.4262499988079071,
|
||
|
|
"reward_std": 0.32143821716308596,
|
||
|
|
"rewards/reward_fn/mean": -0.4262499988079071,
|
||
|
|
"rewards/reward_fn/std": 0.32143821716308596,
|
||
|
|
"step": 1760,
|
||
|
|
"step_time": 0.6563241709955037
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.4,
|
||
|
|
"completions/max_terminated_length": 23.4,
|
||
|
|
"completions/mean_length": 12.65,
|
||
|
|
"completions/mean_terminated_length": 12.65,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.9309171110391616,
|
||
|
|
"epoch": 1.8195876288659794,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 14.875,
|
||
|
|
"learning_rate": 1.2431231006126003e-08,
|
||
|
|
"loss": 0.23779988288879395,
|
||
|
|
"num_tokens": 452787.0,
|
||
|
|
"reward": -0.2858333379030228,
|
||
|
|
"reward_std": 0.6213802993297577,
|
||
|
|
"rewards/reward_fn/mean": -0.2858333379030228,
|
||
|
|
"rewards/reward_fn/std": 0.6213802993297577,
|
||
|
|
"step": 1765,
|
||
|
|
"step_time": 0.788817260786891
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.8,
|
||
|
|
"completions/max_terminated_length": 22.8,
|
||
|
|
"completions/mean_length": 14.3,
|
||
|
|
"completions/mean_terminated_length": 14.3,
|
||
|
|
"completions/min_length": 9.8,
|
||
|
|
"completions/min_terminated_length": 9.8,
|
||
|
|
"entropy": 0.655038520693779,
|
||
|
|
"epoch": 1.824742268041237,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.8125,
|
||
|
|
"learning_rate": 1.174044726278478e-08,
|
||
|
|
"loss": 0.15090932846069335,
|
||
|
|
"num_tokens": 454117.0,
|
||
|
|
"reward": -0.1236666738986969,
|
||
|
|
"reward_std": 0.5083053946495056,
|
||
|
|
"rewards/reward_fn/mean": -0.1236666738986969,
|
||
|
|
"rewards/reward_fn/std": 0.5083053946495056,
|
||
|
|
"step": 1770,
|
||
|
|
"step_time": 0.7027385210618377
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 30.6,
|
||
|
|
"completions/max_terminated_length": 30.6,
|
||
|
|
"completions/mean_length": 16.4,
|
||
|
|
"completions/mean_terminated_length": 16.4,
|
||
|
|
"completions/min_length": 7.8,
|
||
|
|
"completions/min_terminated_length": 7.8,
|
||
|
|
"entropy": 0.8989251792430878,
|
||
|
|
"epoch": 1.829896907216495,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 1.1068947588066812e-08,
|
||
|
|
"loss": 0.07146713733673096,
|
||
|
|
"num_tokens": 455481.0,
|
||
|
|
"reward": -0.4300000011920929,
|
||
|
|
"reward_std": 0.3527463436126709,
|
||
|
|
"rewards/reward_fn/mean": -0.4300000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3527463436126709,
|
||
|
|
"step": 1775,
|
||
|
|
"step_time": 0.9245791801251471
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.6,
|
||
|
|
"completions/max_terminated_length": 22.6,
|
||
|
|
"completions/mean_length": 13.4,
|
||
|
|
"completions/mean_terminated_length": 13.4,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 1.0093101501464843,
|
||
|
|
"epoch": 1.8350515463917527,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 11.25,
|
||
|
|
"learning_rate": 1.0416786331296861e-08,
|
||
|
|
"loss": 0.11369644403457642,
|
||
|
|
"num_tokens": 456769.0,
|
||
|
|
"reward": -0.33700000643730166,
|
||
|
|
"reward_std": 0.5056420981884002,
|
||
|
|
"rewards/reward_fn/mean": -0.33700000643730166,
|
||
|
|
"rewards/reward_fn/std": 0.5056420981884002,
|
||
|
|
"step": 1780,
|
||
|
|
"step_time": 0.7672192833386362
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.0,
|
||
|
|
"completions/max_terminated_length": 14.0,
|
||
|
|
"completions/mean_length": 10.9,
|
||
|
|
"completions/mean_terminated_length": 10.9,
|
||
|
|
"completions/min_length": 8.0,
|
||
|
|
"completions/min_terminated_length": 8.0,
|
||
|
|
"entropy": 0.7461501806974411,
|
||
|
|
"epoch": 1.8402061855670104,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 9.784016276601609e-09,
|
||
|
|
"loss": 0.04938405454158783,
|
||
|
|
"num_tokens": 457979.0,
|
||
|
|
"reward": -0.3700000047683716,
|
||
|
|
"reward_std": 0.15847426652908325,
|
||
|
|
"rewards/reward_fn/mean": -0.3700000047683716,
|
||
|
|
"rewards/reward_fn/std": 0.15847426652908325,
|
||
|
|
"step": 1785,
|
||
|
|
"step_time": 0.5356096717528999
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 22.6,
|
||
|
|
"completions/max_terminated_length": 22.6,
|
||
|
|
"completions/mean_length": 14.65,
|
||
|
|
"completions/mean_terminated_length": 14.65,
|
||
|
|
"completions/min_length": 8.8,
|
||
|
|
"completions/min_terminated_length": 8.8,
|
||
|
|
"entropy": 0.9771642625331879,
|
||
|
|
"epoch": 1.8453608247422681,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 8.875,
|
||
|
|
"learning_rate": 9.170688638637653e-09,
|
||
|
|
"loss": 0.12961161136627197,
|
||
|
|
"num_tokens": 459316.0,
|
||
|
|
"reward": -0.4425000011920929,
|
||
|
|
"reward_std": 0.3898511052131653,
|
||
|
|
"rewards/reward_fn/mean": -0.4425000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.3898511052131653,
|
||
|
|
"step": 1790,
|
||
|
|
"step_time": 0.7050401211716235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.8,
|
||
|
|
"completions/max_terminated_length": 19.8,
|
||
|
|
"completions/mean_length": 13.75,
|
||
|
|
"completions/mean_terminated_length": 13.75,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 1.1843935072422027,
|
||
|
|
"epoch": 1.8505154639175259,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 14.5,
|
||
|
|
"learning_rate": 8.576853058446076e-09,
|
||
|
|
"loss": 0.09116497039794921,
|
||
|
|
"num_tokens": 460615.0,
|
||
|
|
"reward": -0.49600000083446505,
|
||
|
|
"reward_std": 0.20799999237060546,
|
||
|
|
"rewards/reward_fn/mean": -0.49600000083446505,
|
||
|
|
"rewards/reward_fn/std": 0.20799999237060546,
|
||
|
|
"step": 1795,
|
||
|
|
"step_time": 0.7088314037770033
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.2,
|
||
|
|
"completions/max_terminated_length": 11.2,
|
||
|
|
"completions/mean_length": 8.35,
|
||
|
|
"completions/mean_terminated_length": 8.35,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 1.0617978870868683,
|
||
|
|
"epoch": 1.8556701030927836,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.002557599434801e-09,
|
||
|
|
"loss": -0.02811015844345093,
|
||
|
|
"num_tokens": 461794.0,
|
||
|
|
"reward": -0.1716666638851166,
|
||
|
|
"reward_std": 0.1277350217103958,
|
||
|
|
"rewards/reward_fn/mean": -0.1716666638851166,
|
||
|
|
"rewards/reward_fn/std": 0.12773502469062806,
|
||
|
|
"step": 1800,
|
||
|
|
"step_time": 0.48176195118576287
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 10.1,
|
||
|
|
"completions/mean_terminated_length": 10.1,
|
||
|
|
"completions/min_length": 6.6,
|
||
|
|
"completions/min_terminated_length": 6.6,
|
||
|
|
"entropy": 0.7022380590438843,
|
||
|
|
"epoch": 1.8608247422680413,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 7.447848743488555e-09,
|
||
|
|
"loss": 0.13740893602371215,
|
||
|
|
"num_tokens": 463028.0,
|
||
|
|
"reward": -0.3600000023841858,
|
||
|
|
"reward_std": 0.23859126567840577,
|
||
|
|
"rewards/reward_fn/mean": -0.3600000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.2385912775993347,
|
||
|
|
"step": 1805,
|
||
|
|
"step_time": 0.6413388851098716
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.6,
|
||
|
|
"completions/max_terminated_length": 15.6,
|
||
|
|
"completions/mean_length": 12.85,
|
||
|
|
"completions/mean_terminated_length": 12.85,
|
||
|
|
"completions/min_length": 9.8,
|
||
|
|
"completions/min_terminated_length": 9.8,
|
||
|
|
"entropy": 0.5699267834424973,
|
||
|
|
"epoch": 1.865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.912771387206473e-09,
|
||
|
|
"loss": 0.11363432407379151,
|
||
|
|
"num_tokens": 464317.0,
|
||
|
|
"reward": -0.5100000023841857,
|
||
|
|
"reward_std": 0.1800000011920929,
|
||
|
|
"rewards/reward_fn/mean": -0.5100000023841857,
|
||
|
|
"rewards/reward_fn/std": 0.1800000011920929,
|
||
|
|
"step": 1810,
|
||
|
|
"step_time": 0.5692544280551374
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 48.4,
|
||
|
|
"completions/max_terminated_length": 48.4,
|
||
|
|
"completions/mean_length": 19.45,
|
||
|
|
"completions/mean_terminated_length": 19.45,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.8490649104118347,
|
||
|
|
"epoch": 1.8711340206185567,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 17.25,
|
||
|
|
"learning_rate": 6.397368838268496e-09,
|
||
|
|
"loss": 0.26308455467224123,
|
||
|
|
"num_tokens": 465726.0,
|
||
|
|
"reward": -0.4316666662693024,
|
||
|
|
"reward_std": 0.3493190586566925,
|
||
|
|
"rewards/reward_fn/mean": -0.4316666662693024,
|
||
|
|
"rewards/reward_fn/std": 0.3493190586566925,
|
||
|
|
"step": 1815,
|
||
|
|
"step_time": 1.2811257994733751
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 10.4,
|
||
|
|
"completions/max_terminated_length": 10.4,
|
||
|
|
"completions/mean_length": 8.2,
|
||
|
|
"completions/mean_terminated_length": 8.2,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.7323252975940704,
|
||
|
|
"epoch": 1.8762886597938144,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 29.25,
|
||
|
|
"learning_rate": 5.901682811930181e-09,
|
||
|
|
"loss": 0.09926102161407471,
|
||
|
|
"num_tokens": 466898.0,
|
||
|
|
"reward": -0.55,
|
||
|
|
"reward_std": 0.1,
|
||
|
|
"rewards/reward_fn/mean": -0.55,
|
||
|
|
"rewards/reward_fn/std": 0.1,
|
||
|
|
"step": 1820,
|
||
|
|
"step_time": 0.5264862560667097
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 42.0,
|
||
|
|
"completions/max_terminated_length": 42.0,
|
||
|
|
"completions/mean_length": 19.55,
|
||
|
|
"completions/mean_terminated_length": 19.55,
|
||
|
|
"completions/min_length": 8.4,
|
||
|
|
"completions/min_terminated_length": 8.4,
|
||
|
|
"entropy": 0.744831657409668,
|
||
|
|
"epoch": 1.8814432989690721,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 5.425753427646257e-09,
|
||
|
|
"loss": 0.16098196506500245,
|
||
|
|
"num_tokens": 468313.0,
|
||
|
|
"reward": -0.38875000178813934,
|
||
|
|
"reward_std": 0.3985625863075256,
|
||
|
|
"rewards/reward_fn/mean": -0.38875000178813934,
|
||
|
|
"rewards/reward_fn/std": 0.3985625863075256,
|
||
|
|
"step": 1825,
|
||
|
|
"step_time": 1.0875293998979032
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.6,
|
||
|
|
"completions/max_terminated_length": 16.6,
|
||
|
|
"completions/mean_length": 11.25,
|
||
|
|
"completions/mean_terminated_length": 11.25,
|
||
|
|
"completions/min_length": 7.0,
|
||
|
|
"completions/min_terminated_length": 7.0,
|
||
|
|
"entropy": 0.9310593396425247,
|
||
|
|
"epoch": 1.8865979381443299,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 27.25,
|
||
|
|
"learning_rate": 4.9696192058236164e-09,
|
||
|
|
"loss": 0.09600334167480469,
|
||
|
|
"num_tokens": 469550.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.33220337629318236,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.33220337629318236,
|
||
|
|
"step": 1830,
|
||
|
|
"step_time": 0.6508773274719715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 12.8,
|
||
|
|
"completions/max_terminated_length": 12.8,
|
||
|
|
"completions/mean_length": 8.7,
|
||
|
|
"completions/mean_terminated_length": 8.7,
|
||
|
|
"completions/min_length": 5.8,
|
||
|
|
"completions/min_terminated_length": 5.8,
|
||
|
|
"entropy": 0.6754906833171844,
|
||
|
|
"epoch": 1.8917525773195876,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 23.125,
|
||
|
|
"learning_rate": 4.53331706470339e-09,
|
||
|
|
"loss": 0.14865694046020508,
|
||
|
|
"num_tokens": 470724.0,
|
||
|
|
"reward": -0.4199999988079071,
|
||
|
|
"reward_std": 0.3779322624206543,
|
||
|
|
"rewards/reward_fn/mean": -0.4199999988079071,
|
||
|
|
"rewards/reward_fn/std": 0.3779322743415833,
|
||
|
|
"step": 1835,
|
||
|
|
"step_time": 0.518319695442915
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 16.0,
|
||
|
|
"completions/max_terminated_length": 16.0,
|
||
|
|
"completions/mean_length": 9.2,
|
||
|
|
"completions/mean_terminated_length": 9.2,
|
||
|
|
"completions/min_length": 5.4,
|
||
|
|
"completions/min_terminated_length": 5.4,
|
||
|
|
"entropy": 1.0082854956388474,
|
||
|
|
"epoch": 1.8969072164948453,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.116882317373116e-09,
|
||
|
|
"loss": -0.042311012744903564,
|
||
|
|
"num_tokens": 471920.0,
|
||
|
|
"reward": -0.3195833295583725,
|
||
|
|
"reward_std": 0.4602280676364899,
|
||
|
|
"rewards/reward_fn/mean": -0.3195833295583725,
|
||
|
|
"rewards/reward_fn/std": 0.46022807359695433,
|
||
|
|
"step": 1840,
|
||
|
|
"step_time": 0.6366292778402567
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.2,
|
||
|
|
"completions/max_terminated_length": 17.2,
|
||
|
|
"completions/mean_length": 10.65,
|
||
|
|
"completions/mean_terminated_length": 10.65,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.9684360563755036,
|
||
|
|
"epoch": 1.902061855670103,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 12.6875,
|
||
|
|
"learning_rate": 3.720348668908385e-09,
|
||
|
|
"loss": 0.10864226818084717,
|
||
|
|
"num_tokens": 473149.0,
|
||
|
|
"reward": -0.34166666567325593,
|
||
|
|
"reward_std": 0.25666666626930235,
|
||
|
|
"rewards/reward_fn/mean": -0.34166666567325593,
|
||
|
|
"rewards/reward_fn/std": 0.25666666626930235,
|
||
|
|
"step": 1845,
|
||
|
|
"step_time": 0.6000545247457921
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 8.6,
|
||
|
|
"completions/max_terminated_length": 8.6,
|
||
|
|
"completions/mean_length": 7.75,
|
||
|
|
"completions/mean_terminated_length": 7.75,
|
||
|
|
"completions/min_length": 6.2,
|
||
|
|
"completions/min_terminated_length": 6.2,
|
||
|
|
"entropy": 0.7907846450805665,
|
||
|
|
"epoch": 1.9072164948453607,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 11.875,
|
||
|
|
"learning_rate": 3.343748213645026e-09,
|
||
|
|
"loss": -0.018713557720184328,
|
||
|
|
"num_tokens": 474300.0,
|
||
|
|
"reward": -0.3925000011920929,
|
||
|
|
"reward_std": 0.20564382076263427,
|
||
|
|
"rewards/reward_fn/mean": -0.3925000011920929,
|
||
|
|
"rewards/reward_fn/std": 0.20564383268356323,
|
||
|
|
"step": 1850,
|
||
|
|
"step_time": 0.48386304890736936
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 15.8,
|
||
|
|
"completions/max_terminated_length": 15.8,
|
||
|
|
"completions/mean_length": 10.35,
|
||
|
|
"completions/mean_terminated_length": 10.35,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 1.116754847764969,
|
||
|
|
"epoch": 1.9123711340206184,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.9871114325812907e-09,
|
||
|
|
"loss": 0.028201365470886232,
|
||
|
|
"num_tokens": 475499.0,
|
||
|
|
"reward": -0.425,
|
||
|
|
"reward_std": 0.15,
|
||
|
|
"rewards/reward_fn/mean": -0.425,
|
||
|
|
"rewards/reward_fn/std": 0.15,
|
||
|
|
"step": 1855,
|
||
|
|
"step_time": 0.586841206252575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 11.35,
|
||
|
|
"completions/mean_terminated_length": 11.35,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.9712529420852661,
|
||
|
|
"epoch": 1.9175257731958761,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 13.1875,
|
||
|
|
"learning_rate": 2.6504671909109988e-09,
|
||
|
|
"loss": 0.055383771657943726,
|
||
|
|
"num_tokens": 476762.0,
|
||
|
|
"reward": -0.6,
|
||
|
|
"reward_std": 0.15773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.6,
|
||
|
|
"rewards/reward_fn/std": 0.15773502588272095,
|
||
|
|
"step": 1860,
|
||
|
|
"step_time": 0.5633186590857804
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 36.6,
|
||
|
|
"completions/max_terminated_length": 36.6,
|
||
|
|
"completions/mean_length": 15.85,
|
||
|
|
"completions/mean_terminated_length": 15.85,
|
||
|
|
"completions/min_length": 6.4,
|
||
|
|
"completions/min_terminated_length": 6.4,
|
||
|
|
"entropy": 1.062751868367195,
|
||
|
|
"epoch": 1.922680412371134,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 2.333842735687097e-09,
|
||
|
|
"loss": 0.10891166925430298,
|
||
|
|
"num_tokens": 478095.0,
|
||
|
|
"reward": -0.525,
|
||
|
|
"reward_std": 0.05,
|
||
|
|
"rewards/reward_fn/mean": -0.525,
|
||
|
|
"rewards/reward_fn/std": 0.05,
|
||
|
|
"step": 1865,
|
||
|
|
"step_time": 1.0668349410407245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 23.8,
|
||
|
|
"completions/max_terminated_length": 23.8,
|
||
|
|
"completions/mean_length": 16.4,
|
||
|
|
"completions/mean_terminated_length": 16.4,
|
||
|
|
"completions/min_length": 9.0,
|
||
|
|
"completions/min_terminated_length": 9.0,
|
||
|
|
"entropy": 1.1400189995765686,
|
||
|
|
"epoch": 1.9278350515463918,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 10.5,
|
||
|
|
"learning_rate": 2.0372636936165368e-09,
|
||
|
|
"loss": 0.017370551824569702,
|
||
|
|
"num_tokens": 479451.0,
|
||
|
|
"reward": -0.4213888913393021,
|
||
|
|
"reward_std": 0.3149572372436523,
|
||
|
|
"rewards/reward_fn/mean": -0.4213888913393021,
|
||
|
|
"rewards/reward_fn/std": 0.3149572491645813,
|
||
|
|
"step": 1870,
|
||
|
|
"step_time": 0.8135622454807162
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.2,
|
||
|
|
"completions/max_terminated_length": 17.2,
|
||
|
|
"completions/mean_length": 12.35,
|
||
|
|
"completions/mean_terminated_length": 12.35,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 1.0274198472499847,
|
||
|
|
"epoch": 1.9329896907216495,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.7607540689859035e-09,
|
||
|
|
"loss": 0.022668755054473876,
|
||
|
|
"num_tokens": 480746.0,
|
||
|
|
"reward": -0.5350000023841858,
|
||
|
|
"reward_std": 0.18773502707481385,
|
||
|
|
"rewards/reward_fn/mean": -0.5350000023841858,
|
||
|
|
"rewards/reward_fn/std": 0.18773502707481385,
|
||
|
|
"step": 1875,
|
||
|
|
"step_time": 0.6297862586565316
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 13.2,
|
||
|
|
"completions/max_terminated_length": 13.2,
|
||
|
|
"completions/mean_length": 9.95,
|
||
|
|
"completions/mean_terminated_length": 9.95,
|
||
|
|
"completions/min_length": 7.4,
|
||
|
|
"completions/min_terminated_length": 7.4,
|
||
|
|
"entropy": 0.7384592443704605,
|
||
|
|
"epoch": 1.9381443298969072,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.5043362417187787e-09,
|
||
|
|
"loss": 0.11721787452697754,
|
||
|
|
"num_tokens": 481953.0,
|
||
|
|
"reward": -0.2975000083446503,
|
||
|
|
"reward_std": 0.3525484263896942,
|
||
|
|
"rewards/reward_fn/mean": -0.2975000083446503,
|
||
|
|
"rewards/reward_fn/std": 0.35254843831062316,
|
||
|
|
"step": 1880,
|
||
|
|
"step_time": 0.6035839405842125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 19.2,
|
||
|
|
"completions/max_terminated_length": 19.2,
|
||
|
|
"completions/mean_length": 12.75,
|
||
|
|
"completions/mean_terminated_length": 12.75,
|
||
|
|
"completions/min_length": 6.8,
|
||
|
|
"completions/min_terminated_length": 6.8,
|
||
|
|
"entropy": 0.9961306691169739,
|
||
|
|
"epoch": 1.943298969072165,
|
||
|
|
"frac_reward_zero_std": 0.6,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.268030965564243e-09,
|
||
|
|
"loss": 0.061715888977050784,
|
||
|
|
"num_tokens": 483212.0,
|
||
|
|
"reward": -0.4899999976158142,
|
||
|
|
"reward_std": 0.1,
|
||
|
|
"rewards/reward_fn/mean": -0.4899999976158142,
|
||
|
|
"rewards/reward_fn/std": 0.1,
|
||
|
|
"step": 1885,
|
||
|
|
"step_time": 0.6409509646706283
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 11.8,
|
||
|
|
"completions/max_terminated_length": 11.8,
|
||
|
|
"completions/mean_length": 9.45,
|
||
|
|
"completions/mean_terminated_length": 9.45,
|
||
|
|
"completions/min_length": 7.2,
|
||
|
|
"completions/min_terminated_length": 7.2,
|
||
|
|
"entropy": 0.6782282710075378,
|
||
|
|
"epoch": 1.9484536082474226,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 1.0518573664172192e-09,
|
||
|
|
"loss": 0.09830584526062011,
|
||
|
|
"num_tokens": 484409.0,
|
||
|
|
"reward": -0.4433333337306976,
|
||
|
|
"reward_std": 0.3213032424449921,
|
||
|
|
"rewards/reward_fn/mean": -0.4433333337306976,
|
||
|
|
"rewards/reward_fn/std": 0.3213032424449921,
|
||
|
|
"step": 1890,
|
||
|
|
"step_time": 0.5699493732303381
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 14.6,
|
||
|
|
"completions/max_terminated_length": 14.6,
|
||
|
|
"completions/mean_length": 11.9,
|
||
|
|
"completions/mean_terminated_length": 11.9,
|
||
|
|
"completions/min_length": 8.6,
|
||
|
|
"completions/min_terminated_length": 8.6,
|
||
|
|
"entropy": 0.7531616747379303,
|
||
|
|
"epoch": 1.9536082474226806,
|
||
|
|
"frac_reward_zero_std": 0.4,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.558329407703513e-10,
|
||
|
|
"loss": 0.040635806322097776,
|
||
|
|
"num_tokens": 485675.0,
|
||
|
|
"reward": -0.30833333134651186,
|
||
|
|
"reward_std": 0.36525357961654664,
|
||
|
|
"rewards/reward_fn/mean": -0.30833333134651186,
|
||
|
|
"rewards/reward_fn/std": 0.3652535915374756,
|
||
|
|
"step": 1895,
|
||
|
|
"step_time": 0.5324823048897087
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 17.6,
|
||
|
|
"completions/max_terminated_length": 17.6,
|
||
|
|
"completions/mean_length": 12.4,
|
||
|
|
"completions/mean_terminated_length": 12.4,
|
||
|
|
"completions/min_length": 8.2,
|
||
|
|
"completions/min_terminated_length": 8.2,
|
||
|
|
"entropy": 0.8260966598987579,
|
||
|
|
"epoch": 1.9587628865979383,
|
||
|
|
"frac_reward_zero_std": 0.2,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.799735542979967e-10,
|
||
|
|
"loss": 0.06493161916732788,
|
||
|
|
"num_tokens": 486963.0,
|
||
|
|
"reward": -0.575,
|
||
|
|
"reward_std": 0.30773502588272095,
|
||
|
|
"rewards/reward_fn/mean": -0.575,
|
||
|
|
"rewards/reward_fn/std": 0.30773502588272095,
|
||
|
|
"step": 1900,
|
||
|
|
"step_time": 0.6176068928092718
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 1940,
|
||
|
|
"num_input_tokens_seen": 486963,
|
||
|
|
"num_train_epochs": 2,
|
||
|
|
"save_steps": 50,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|