875 lines
31 KiB
JSON
875 lines
31 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 2.0,
|
|
"eval_steps": 500,
|
|
"global_step": 300,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.525,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 378.9,
|
|
"completions/mean_length": 457.9125,
|
|
"completions/mean_terminated_length": 322.60096435546876,
|
|
"completions/min_length": 351.5,
|
|
"completions/min_terminated_length": 249.1,
|
|
"entropy": 0.3948360087350011,
|
|
"epoch": 0.06666666666666667,
|
|
"frac_reward_zero_std": 0.05,
|
|
"grad_norm": 2.03125,
|
|
"kl": 0.04399079610357148,
|
|
"learning_rate": 9.7e-06,
|
|
"loss": -0.004937342554330826,
|
|
"num_tokens": 49697.0,
|
|
"reward": 0.65526123046875,
|
|
"reward_std": 0.4537597060203552,
|
|
"rewards/JointRewardFunction/mean": 0.65526123046875,
|
|
"rewards/JointRewardFunction/std": 0.45375972241163254,
|
|
"step": 10,
|
|
"step_time": 22.810993660400346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/max_length": 508.7,
|
|
"completions/max_terminated_length": 417.6,
|
|
"completions/mean_length": 443.675,
|
|
"completions/mean_terminated_length": 342.31012573242185,
|
|
"completions/min_length": 326.9,
|
|
"completions/min_terminated_length": 275.7,
|
|
"entropy": 0.31964702922850846,
|
|
"epoch": 0.13333333333333333,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.1982421875,
|
|
"kl": 0.09551988991443068,
|
|
"learning_rate": 9.366666666666668e-06,
|
|
"loss": 0.00889739990234375,
|
|
"num_tokens": 97803.0,
|
|
"reward": 0.7364013671875,
|
|
"reward_std": 0.4511200308799744,
|
|
"rewards/JointRewardFunction/mean": 0.7364013671875,
|
|
"rewards/JointRewardFunction/std": 0.4511200487613678,
|
|
"step": 20,
|
|
"step_time": 23.522369655099464
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.675,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 379.6,
|
|
"completions/mean_length": 477.4875,
|
|
"completions/mean_terminated_length": 350.68857421875,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 319.8,
|
|
"entropy": 0.40151252076029775,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 0.35,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.11964645721018315,
|
|
"learning_rate": 9.033333333333334e-06,
|
|
"loss": -0.0028077010065317154,
|
|
"num_tokens": 148394.0,
|
|
"reward": 0.9680908203125,
|
|
"reward_std": 0.40886188447475436,
|
|
"rewards/JointRewardFunction/mean": 0.9680908203125,
|
|
"rewards/JointRewardFunction/std": 0.40886189937591555,
|
|
"step": 30,
|
|
"step_time": 23.04281468010122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 469.3,
|
|
"completions/mean_length": 438.0875,
|
|
"completions/mean_terminated_length": 393.87953491210936,
|
|
"completions/min_length": 335.8,
|
|
"completions/min_terminated_length": 335.8,
|
|
"entropy": 0.35483993515372275,
|
|
"epoch": 0.26666666666666666,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 2.34375,
|
|
"kl": 0.0885015036445111,
|
|
"learning_rate": 8.700000000000001e-06,
|
|
"loss": 0.03651041984558105,
|
|
"num_tokens": 197537.0,
|
|
"reward": 0.9726318359375,
|
|
"reward_std": 0.38605129569768903,
|
|
"rewards/JointRewardFunction/mean": 0.9726318359375,
|
|
"rewards/JointRewardFunction/std": 0.3860512986779213,
|
|
"step": 40,
|
|
"step_time": 23.371826623300876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 416.7,
|
|
"completions/max_terminated_length": 387.6,
|
|
"completions/mean_length": 307.5,
|
|
"completions/mean_terminated_length": 296.2386932373047,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.3532023083418608,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 2.484375,
|
|
"kl": 0.1458138041663915,
|
|
"learning_rate": 8.366666666666667e-06,
|
|
"loss": 0.058735775947570804,
|
|
"num_tokens": 235441.0,
|
|
"reward": 0.911224365234375,
|
|
"reward_std": 0.44634594917297366,
|
|
"rewards/JointRewardFunction/mean": 0.911224365234375,
|
|
"rewards/JointRewardFunction/std": 0.4463459521532059,
|
|
"step": 50,
|
|
"step_time": 19.590083589001733
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 439.0,
|
|
"completions/max_terminated_length": 411.3,
|
|
"completions/mean_length": 314.675,
|
|
"completions/mean_terminated_length": 302.16786193847656,
|
|
"completions/min_length": 213.5,
|
|
"completions/min_terminated_length": 213.5,
|
|
"entropy": 0.33970522740855813,
|
|
"epoch": 0.4,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 2.21875,
|
|
"kl": 0.13768328949809075,
|
|
"learning_rate": 8.033333333333335e-06,
|
|
"loss": 0.024235518276691438,
|
|
"num_tokens": 273659.0,
|
|
"reward": 1.012158203125,
|
|
"reward_std": 0.34636138631030916,
|
|
"rewards/JointRewardFunction/mean": 1.012158203125,
|
|
"rewards/JointRewardFunction/std": 0.3463613982312381,
|
|
"step": 60,
|
|
"step_time": 20.71416906509876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 405.4,
|
|
"completions/max_terminated_length": 397.6,
|
|
"completions/mean_length": 306.85,
|
|
"completions/mean_terminated_length": 298.9619079589844,
|
|
"completions/min_length": 215.4,
|
|
"completions/min_terminated_length": 215.4,
|
|
"entropy": 0.36216176860034466,
|
|
"epoch": 0.4666666666666667,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 2.375,
|
|
"kl": 0.11585550024174154,
|
|
"learning_rate": 7.7e-06,
|
|
"loss": -0.0011418772861361503,
|
|
"num_tokens": 311055.0,
|
|
"reward": 1.108551025390625,
|
|
"reward_std": 0.31149580857600084,
|
|
"rewards/JointRewardFunction/mean": 1.108551025390625,
|
|
"rewards/JointRewardFunction/std": 0.311495810066117,
|
|
"step": 70,
|
|
"step_time": 19.105042513498848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 425.2,
|
|
"completions/max_terminated_length": 399.6,
|
|
"completions/mean_length": 303.65,
|
|
"completions/mean_terminated_length": 296.1714324951172,
|
|
"completions/min_length": 221.7,
|
|
"completions/min_terminated_length": 221.7,
|
|
"entropy": 0.458731491304934,
|
|
"epoch": 0.5333333333333333,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 1.8515625,
|
|
"kl": 0.12144780233502388,
|
|
"learning_rate": 7.3666666666666676e-06,
|
|
"loss": 0.04609571397304535,
|
|
"num_tokens": 347759.0,
|
|
"reward": 1.0232421875,
|
|
"reward_std": 0.39059739662334325,
|
|
"rewards/JointRewardFunction/mean": 1.0232421875,
|
|
"rewards/JointRewardFunction/std": 0.39059740854427216,
|
|
"step": 80,
|
|
"step_time": 20.152326823800102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 458.2,
|
|
"completions/max_terminated_length": 434.3,
|
|
"completions/mean_length": 325.7,
|
|
"completions/mean_terminated_length": 320.2678619384766,
|
|
"completions/min_length": 231.3,
|
|
"completions/min_terminated_length": 231.3,
|
|
"entropy": 0.414226458594203,
|
|
"epoch": 0.6,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 2.71875,
|
|
"kl": 0.11777536268346012,
|
|
"learning_rate": 7.033333333333334e-06,
|
|
"loss": 0.017658862471580505,
|
|
"num_tokens": 386695.0,
|
|
"reward": 1.0622802734375,
|
|
"reward_std": 0.37295620441436766,
|
|
"rewards/JointRewardFunction/mean": 1.0622802734375,
|
|
"rewards/JointRewardFunction/std": 0.3729562103748322,
|
|
"step": 90,
|
|
"step_time": 21.401791930101172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 419.9,
|
|
"completions/max_terminated_length": 405.6,
|
|
"completions/mean_length": 305.0625,
|
|
"completions/mean_terminated_length": 297.75535888671874,
|
|
"completions/min_length": 201.6,
|
|
"completions/min_terminated_length": 201.6,
|
|
"entropy": 0.4645195137709379,
|
|
"epoch": 0.6666666666666666,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 3.21875,
|
|
"kl": 0.11028626803308725,
|
|
"learning_rate": 6.700000000000001e-06,
|
|
"loss": 0.019249849021434784,
|
|
"num_tokens": 423284.0,
|
|
"reward": 1.095147705078125,
|
|
"reward_std": 0.3025559388101101,
|
|
"rewards/JointRewardFunction/mean": 1.095147705078125,
|
|
"rewards/JointRewardFunction/std": 0.302555950731039,
|
|
"step": 100,
|
|
"step_time": 19.82219038610201
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 437.4,
|
|
"completions/max_terminated_length": 425.9,
|
|
"completions/mean_length": 325.925,
|
|
"completions/mean_terminated_length": 319.71428833007815,
|
|
"completions/min_length": 227.9,
|
|
"completions/min_terminated_length": 227.9,
|
|
"entropy": 0.43129814630374314,
|
|
"epoch": 0.7333333333333333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.40625,
|
|
"kl": 0.1031634843442589,
|
|
"learning_rate": 6.366666666666668e-06,
|
|
"loss": 0.026633352041244507,
|
|
"num_tokens": 462986.0,
|
|
"reward": 1.043157958984375,
|
|
"reward_std": 0.33559322796645574,
|
|
"rewards/JointRewardFunction/mean": 1.043157958984375,
|
|
"rewards/JointRewardFunction/std": 0.3355932369071525,
|
|
"step": 110,
|
|
"step_time": 20.435491250798805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 444.4,
|
|
"completions/max_terminated_length": 440.1,
|
|
"completions/mean_length": 337.825,
|
|
"completions/mean_terminated_length": 330.73964538574216,
|
|
"completions/min_length": 233.7,
|
|
"completions/min_terminated_length": 233.7,
|
|
"entropy": 0.43917944487184285,
|
|
"epoch": 0.8,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 4.03125,
|
|
"kl": 0.1373384237056598,
|
|
"learning_rate": 6.033333333333335e-06,
|
|
"loss": 0.051127517223358156,
|
|
"num_tokens": 504032.0,
|
|
"reward": 1.03193359375,
|
|
"reward_std": 0.326281476020813,
|
|
"rewards/JointRewardFunction/mean": 1.03193359375,
|
|
"rewards/JointRewardFunction/std": 0.3262814819812775,
|
|
"step": 120,
|
|
"step_time": 20.735252708198097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 412.0,
|
|
"completions/max_terminated_length": 392.5,
|
|
"completions/mean_length": 312.775,
|
|
"completions/mean_terminated_length": 300.4857147216797,
|
|
"completions/min_length": 222.2,
|
|
"completions/min_terminated_length": 222.2,
|
|
"entropy": 0.4507128844037652,
|
|
"epoch": 0.8666666666666667,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 2.09375,
|
|
"kl": 0.11878000139258801,
|
|
"learning_rate": 5.7e-06,
|
|
"loss": 0.011486038565635681,
|
|
"num_tokens": 544434.0,
|
|
"reward": 1.053509521484375,
|
|
"reward_std": 0.31438804864883424,
|
|
"rewards/JointRewardFunction/mean": 1.053509521484375,
|
|
"rewards/JointRewardFunction/std": 0.31438804864883424,
|
|
"step": 130,
|
|
"step_time": 19.64918669259896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 439.1,
|
|
"completions/max_terminated_length": 435.6,
|
|
"completions/mean_length": 333.05,
|
|
"completions/mean_terminated_length": 327.24464416503906,
|
|
"completions/min_length": 234.3,
|
|
"completions/min_terminated_length": 234.3,
|
|
"entropy": 0.40997486785054205,
|
|
"epoch": 0.9333333333333333,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 2.84375,
|
|
"kl": 0.10414444564376027,
|
|
"learning_rate": 5.366666666666666e-06,
|
|
"loss": 0.011959856748580933,
|
|
"num_tokens": 584866.0,
|
|
"reward": 1.083343505859375,
|
|
"reward_std": 0.3227477639913559,
|
|
"rewards/JointRewardFunction/mean": 1.083343505859375,
|
|
"rewards/JointRewardFunction/std": 0.3227477639913559,
|
|
"step": 140,
|
|
"step_time": 20.672387204799815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 434.4,
|
|
"completions/max_terminated_length": 412.6,
|
|
"completions/mean_length": 322.45,
|
|
"completions/mean_terminated_length": 308.4941680908203,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.4216183707118034,
|
|
"epoch": 1.0,
|
|
"frac_reward_zero_std": 0.35,
|
|
"grad_norm": 3.234375,
|
|
"kl": 0.10422139009460807,
|
|
"learning_rate": 5.033333333333333e-06,
|
|
"loss": 0.009627737849950791,
|
|
"num_tokens": 623390.0,
|
|
"reward": 1.072760009765625,
|
|
"reward_std": 0.298717375023989,
|
|
"rewards/JointRewardFunction/mean": 1.072760009765625,
|
|
"rewards/JointRewardFunction/std": 0.29871737800422127,
|
|
"step": 150,
|
|
"step_time": 20.43400644650028
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 477.3,
|
|
"completions/max_terminated_length": 441.4,
|
|
"completions/mean_length": 332.3375,
|
|
"completions/mean_terminated_length": 314.32142944335936,
|
|
"completions/min_length": 212.2,
|
|
"completions/min_terminated_length": 212.2,
|
|
"entropy": 0.4680627126246691,
|
|
"epoch": 1.0666666666666667,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 0.054443359375,
|
|
"kl": 0.09179632470477372,
|
|
"learning_rate": 4.7e-06,
|
|
"loss": 0.01482328325510025,
|
|
"num_tokens": 663349.0,
|
|
"reward": 1.0511962890625,
|
|
"reward_std": 0.33187133751343934,
|
|
"rewards/JointRewardFunction/mean": 1.0511962890625,
|
|
"rewards/JointRewardFunction/std": 0.33187134347390385,
|
|
"step": 160,
|
|
"step_time": 22.216746885597967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 431.7,
|
|
"completions/max_terminated_length": 406.5,
|
|
"completions/mean_length": 307.6625,
|
|
"completions/mean_terminated_length": 298.221435546875,
|
|
"completions/min_length": 207.6,
|
|
"completions/min_terminated_length": 207.6,
|
|
"entropy": 0.4726201189681888,
|
|
"epoch": 1.1333333333333333,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 2.078125,
|
|
"kl": 0.10375995303038507,
|
|
"learning_rate": 4.366666666666667e-06,
|
|
"loss": 0.008281752467155457,
|
|
"num_tokens": 699830.0,
|
|
"reward": 1.084967041015625,
|
|
"reward_std": 0.2650434322655201,
|
|
"rewards/JointRewardFunction/mean": 1.084967041015625,
|
|
"rewards/JointRewardFunction/std": 0.26504343524575236,
|
|
"step": 170,
|
|
"step_time": 20.110693615402123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 379.8,
|
|
"completions/max_terminated_length": 379.8,
|
|
"completions/mean_length": 297.7875,
|
|
"completions/mean_terminated_length": 297.7875,
|
|
"completions/min_length": 209.8,
|
|
"completions/min_terminated_length": 209.8,
|
|
"entropy": 0.4800571627914906,
|
|
"epoch": 1.2,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 2.859375,
|
|
"kl": 0.10157871665433049,
|
|
"learning_rate": 4.033333333333333e-06,
|
|
"loss": 0.015658122301101685,
|
|
"num_tokens": 735313.0,
|
|
"reward": 1.125604248046875,
|
|
"reward_std": 0.25403361991047857,
|
|
"rewards/JointRewardFunction/mean": 1.125604248046875,
|
|
"rewards/JointRewardFunction/std": 0.2540336221456528,
|
|
"step": 180,
|
|
"step_time": 18.15690856939873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 442.3,
|
|
"completions/max_terminated_length": 417.9,
|
|
"completions/mean_length": 322.4875,
|
|
"completions/mean_terminated_length": 312.0000061035156,
|
|
"completions/min_length": 221.9,
|
|
"completions/min_terminated_length": 221.9,
|
|
"entropy": 0.4642798786982894,
|
|
"epoch": 1.2666666666666666,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 2.1875,
|
|
"kl": 0.0930135941482149,
|
|
"learning_rate": 3.7e-06,
|
|
"loss": 0.022286814451217652,
|
|
"num_tokens": 773840.0,
|
|
"reward": 1.0874267578125,
|
|
"reward_std": 0.2804231091577094,
|
|
"rewards/JointRewardFunction/mean": 1.0874267578125,
|
|
"rewards/JointRewardFunction/std": 0.28042311511817386,
|
|
"step": 190,
|
|
"step_time": 20.867724073099453
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 448.1,
|
|
"completions/max_terminated_length": 431.5,
|
|
"completions/mean_length": 345.2375,
|
|
"completions/mean_terminated_length": 335.31964416503905,
|
|
"completions/min_length": 235.1,
|
|
"completions/min_terminated_length": 235.1,
|
|
"entropy": 0.4752088913694024,
|
|
"epoch": 1.3333333333333333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 2.140625,
|
|
"kl": 0.08606207200791686,
|
|
"learning_rate": 3.366666666666667e-06,
|
|
"loss": 0.019437333941459654,
|
|
"num_tokens": 814411.0,
|
|
"reward": 1.0824462890625,
|
|
"reward_std": 0.3011944264173508,
|
|
"rewards/JointRewardFunction/mean": 1.0824462890625,
|
|
"rewards/JointRewardFunction/std": 0.3011944264173508,
|
|
"step": 200,
|
|
"step_time": 21.037474609902347
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1125,
|
|
"completions/max_length": 447.2,
|
|
"completions/max_terminated_length": 440.3,
|
|
"completions/mean_length": 341.525,
|
|
"completions/mean_terminated_length": 324.12596130371094,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.45753496792167425,
|
|
"epoch": 1.4,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 1.7265625,
|
|
"kl": 0.08462402573786676,
|
|
"learning_rate": 3.0333333333333337e-06,
|
|
"loss": 0.03857302367687225,
|
|
"num_tokens": 854993.0,
|
|
"reward": 1.0694580078125,
|
|
"reward_std": 0.32212619576603174,
|
|
"rewards/JointRewardFunction/mean": 1.0694580078125,
|
|
"rewards/JointRewardFunction/std": 0.32212621364742516,
|
|
"step": 210,
|
|
"step_time": 20.958584549299484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 439.8,
|
|
"completions/max_terminated_length": 409.0,
|
|
"completions/mean_length": 325.825,
|
|
"completions/mean_terminated_length": 312.76429138183596,
|
|
"completions/min_length": 230.1,
|
|
"completions/min_terminated_length": 230.1,
|
|
"entropy": 0.43106496389955284,
|
|
"epoch": 1.4666666666666668,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.84375,
|
|
"kl": 0.08686103990767151,
|
|
"learning_rate": 2.7000000000000004e-06,
|
|
"loss": 0.019956868886947633,
|
|
"num_tokens": 894887.0,
|
|
"reward": 1.037451171875,
|
|
"reward_std": 0.27579919978743417,
|
|
"rewards/JointRewardFunction/mean": 1.037451171875,
|
|
"rewards/JointRewardFunction/std": 0.2757992057478987,
|
|
"step": 220,
|
|
"step_time": 20.81888391800021
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 370.2,
|
|
"completions/max_terminated_length": 357.7,
|
|
"completions/mean_length": 287.3625,
|
|
"completions/mean_terminated_length": 284.64464416503904,
|
|
"completions/min_length": 212.9,
|
|
"completions/min_terminated_length": 212.9,
|
|
"entropy": 0.5203873571008444,
|
|
"epoch": 1.5333333333333332,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 2.4375,
|
|
"kl": 0.09830583925358952,
|
|
"learning_rate": 2.3666666666666667e-06,
|
|
"loss": 0.013263462483882904,
|
|
"num_tokens": 929688.0,
|
|
"reward": 1.1615234375,
|
|
"reward_std": 0.1927501901984215,
|
|
"rewards/JointRewardFunction/mean": 1.1615234375,
|
|
"rewards/JointRewardFunction/std": 0.19275019615888594,
|
|
"step": 230,
|
|
"step_time": 17.954591237399292
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1375,
|
|
"completions/max_length": 455.5,
|
|
"completions/max_terminated_length": 442.9,
|
|
"completions/mean_length": 355.65,
|
|
"completions/mean_terminated_length": 341.68095703125,
|
|
"completions/min_length": 251.1,
|
|
"completions/min_terminated_length": 251.1,
|
|
"entropy": 0.42119636284187434,
|
|
"epoch": 1.6,
|
|
"frac_reward_zero_std": 0.35,
|
|
"grad_norm": 1.78125,
|
|
"kl": 0.08470410050358623,
|
|
"learning_rate": 2.0333333333333335e-06,
|
|
"loss": 0.01756674200296402,
|
|
"num_tokens": 972840.0,
|
|
"reward": 0.989080810546875,
|
|
"reward_std": 0.37197236269712447,
|
|
"rewards/JointRewardFunction/mean": 0.989080810546875,
|
|
"rewards/JointRewardFunction/std": 0.37197236269712447,
|
|
"step": 240,
|
|
"step_time": 21.474622961001295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 462.3,
|
|
"completions/max_terminated_length": 428.0,
|
|
"completions/mean_length": 342.35,
|
|
"completions/mean_terminated_length": 324.6696472167969,
|
|
"completions/min_length": 234.8,
|
|
"completions/min_terminated_length": 234.8,
|
|
"entropy": 0.4915314789861441,
|
|
"epoch": 1.6666666666666665,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.2109375,
|
|
"kl": 0.08292658934369683,
|
|
"learning_rate": 1.7000000000000002e-06,
|
|
"loss": 0.021336786448955536,
|
|
"num_tokens": 1013548.0,
|
|
"reward": 1.00546875,
|
|
"reward_std": 0.3482314422726631,
|
|
"rewards/JointRewardFunction/mean": 1.00546875,
|
|
"rewards/JointRewardFunction/std": 0.3482314541935921,
|
|
"step": 250,
|
|
"step_time": 21.594003204298495
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1125,
|
|
"completions/max_length": 458.6,
|
|
"completions/max_terminated_length": 413.7,
|
|
"completions/mean_length": 339.1875,
|
|
"completions/mean_terminated_length": 320.0369110107422,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.4891188982874155,
|
|
"epoch": 1.7333333333333334,
|
|
"frac_reward_zero_std": 0.45,
|
|
"grad_norm": 1.515625,
|
|
"kl": 0.08876022514887154,
|
|
"learning_rate": 1.3666666666666668e-06,
|
|
"loss": 0.04018712341785431,
|
|
"num_tokens": 1053691.0,
|
|
"reward": 1.0763916015625,
|
|
"reward_std": 0.33348723202943803,
|
|
"rewards/JointRewardFunction/mean": 1.0763916015625,
|
|
"rewards/JointRewardFunction/std": 0.33348723202943803,
|
|
"step": 260,
|
|
"step_time": 21.527129449401038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 424.1,
|
|
"completions/max_terminated_length": 398.5,
|
|
"completions/mean_length": 326.7125,
|
|
"completions/mean_terminated_length": 318.79107971191405,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.5027944987639785,
|
|
"epoch": 1.8,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.4140625,
|
|
"kl": 0.09775767312385142,
|
|
"learning_rate": 1.0333333333333333e-06,
|
|
"loss": 0.022160670161247252,
|
|
"num_tokens": 1093636.0,
|
|
"reward": 1.073553466796875,
|
|
"reward_std": 0.2811176300048828,
|
|
"rewards/JointRewardFunction/mean": 1.073553466796875,
|
|
"rewards/JointRewardFunction/std": 0.2811176344752312,
|
|
"step": 270,
|
|
"step_time": 19.999046050798643
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 418.1,
|
|
"completions/max_terminated_length": 399.8,
|
|
"completions/mean_length": 323.325,
|
|
"completions/mean_terminated_length": 314.7875,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.46924052368849517,
|
|
"epoch": 1.8666666666666667,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 1.890625,
|
|
"kl": 0.09717398346401751,
|
|
"learning_rate": 7.000000000000001e-07,
|
|
"loss": 0.011738638579845428,
|
|
"num_tokens": 1132610.0,
|
|
"reward": 1.1536865234375,
|
|
"reward_std": 0.23539954721927642,
|
|
"rewards/JointRewardFunction/mean": 1.1536865234375,
|
|
"rewards/JointRewardFunction/std": 0.2353995516896248,
|
|
"step": 280,
|
|
"step_time": 20.17876625719837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1375,
|
|
"completions/max_length": 465.2,
|
|
"completions/max_terminated_length": 430.9,
|
|
"completions/mean_length": 340.55,
|
|
"completions/mean_terminated_length": 316.59702758789064,
|
|
"completions/min_length": 220.1,
|
|
"completions/min_terminated_length": 220.1,
|
|
"entropy": 0.48896827585995195,
|
|
"epoch": 1.9333333333333333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.08544921875,
|
|
"kl": 0.09501179384533316,
|
|
"learning_rate": 3.666666666666667e-07,
|
|
"loss": 0.011296895891427993,
|
|
"num_tokens": 1175610.0,
|
|
"reward": 1.069940185546875,
|
|
"reward_std": 0.29926753640174864,
|
|
"rewards/JointRewardFunction/mean": 1.069940185546875,
|
|
"rewards/JointRewardFunction/std": 0.2992675393819809,
|
|
"step": 290,
|
|
"step_time": 21.805314568298854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 427.1,
|
|
"completions/max_terminated_length": 417.8,
|
|
"completions/mean_length": 321.225,
|
|
"completions/mean_terminated_length": 312.3041687011719,
|
|
"completions/min_length": 233.6,
|
|
"completions/min_terminated_length": 233.6,
|
|
"entropy": 0.49822530038654805,
|
|
"epoch": 2.0,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 2.28125,
|
|
"kl": 0.09350393395870923,
|
|
"learning_rate": 3.333333333333334e-08,
|
|
"loss": 0.0006398491561412812,
|
|
"num_tokens": 1214508.0,
|
|
"reward": 1.014300537109375,
|
|
"reward_std": 0.3215805938467383,
|
|
"rewards/JointRewardFunction/mean": 1.014300537109375,
|
|
"rewards/JointRewardFunction/std": 0.3215805938467383,
|
|
"step": 300,
|
|
"step_time": 20.291091689400854
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 300,
|
|
"num_input_tokens_seen": 1214508,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 10,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|