485 lines
17 KiB
JSON
485 lines
17 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 150,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 454.5,
|
|
"completions/max_terminated_length": 342.3,
|
|
"completions/mean_length": 139.7,
|
|
"completions/mean_terminated_length": 119.76785888671876,
|
|
"completions/min_length": 11.4,
|
|
"completions/min_terminated_length": 11.4,
|
|
"entropy": 0.45251130862161515,
|
|
"epoch": 0.06666666666666667,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 24.625,
|
|
"kl": 0.003444959384390245,
|
|
"learning_rate": 9.911436253643443e-07,
|
|
"loss": -0.056199127435684205,
|
|
"num_tokens": 22424.0,
|
|
"reward": 1.4375,
|
|
"reward_std": 0.44930775463581085,
|
|
"rewards/accuracy_reward_func/mean": 0.45,
|
|
"rewards/accuracy_reward_func/std": 0.42546978294849397,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 10,
|
|
"step_time": 17.857542922068387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 405.7,
|
|
"completions/max_terminated_length": 405.7,
|
|
"completions/mean_length": 104.75,
|
|
"completions/mean_terminated_length": 104.75,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.37909337431192397,
|
|
"epoch": 0.13333333333333333,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 33.25,
|
|
"kl": 0.0022808335735902572,
|
|
"learning_rate": 9.609315757942502e-07,
|
|
"loss": -0.12742023468017577,
|
|
"num_tokens": 41264.0,
|
|
"reward": 1.65,
|
|
"reward_std": 0.369350728392601,
|
|
"rewards/accuracy_reward_func/mean": 0.675,
|
|
"rewards/accuracy_reward_func/std": 0.32949312031269073,
|
|
"rewards/format_reward_func/mean": 0.975,
|
|
"rewards/format_reward_func/std": 0.046291005611419675,
|
|
"step": 20,
|
|
"step_time": 15.968829826638103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 770.7,
|
|
"completions/max_terminated_length": 607.1,
|
|
"completions/mean_length": 247.05,
|
|
"completions/mean_terminated_length": 189.3309555053711,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.4646949103684165,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 5.15625,
|
|
"kl": 0.003183109380734095,
|
|
"learning_rate": 9.10574604566852e-07,
|
|
"loss": 0.23813328742980958,
|
|
"num_tokens": 72036.0,
|
|
"reward": 1.6125,
|
|
"reward_std": 0.5259025603532791,
|
|
"rewards/accuracy_reward_func/mean": 0.675,
|
|
"rewards/accuracy_reward_func/std": 0.41283679008483887,
|
|
"rewards/format_reward_func/mean": 0.9375,
|
|
"rewards/format_reward_func/std": 0.15235702097415924,
|
|
"step": 30,
|
|
"step_time": 28.961046700924634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 651.1,
|
|
"completions/max_terminated_length": 495.6,
|
|
"completions/mean_length": 195.975,
|
|
"completions/mean_terminated_length": 155.1339324951172,
|
|
"completions/min_length": 12.4,
|
|
"completions/min_terminated_length": 12.4,
|
|
"entropy": 0.42483185429591686,
|
|
"epoch": 0.26666666666666666,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 12.0625,
|
|
"kl": 0.002366386147068633,
|
|
"learning_rate": 8.422735529643443e-07,
|
|
"loss": 0.07951252460479737,
|
|
"num_tokens": 98462.0,
|
|
"reward": 1.55,
|
|
"reward_std": 0.5763359367847443,
|
|
"rewards/accuracy_reward_func/mean": 0.625,
|
|
"rewards/accuracy_reward_func/std": 0.4401642054319382,
|
|
"rewards/format_reward_func/mean": 0.925,
|
|
"rewards/format_reward_func/std": 0.15782093703746797,
|
|
"step": 40,
|
|
"step_time": 24.76931145079434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 212.6,
|
|
"completions/max_terminated_length": 212.6,
|
|
"completions/mean_length": 77.7625,
|
|
"completions/mean_terminated_length": 77.7625,
|
|
"completions/min_length": 11.4,
|
|
"completions/min_terminated_length": 11.4,
|
|
"entropy": 0.5150532013736665,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 24.0,
|
|
"kl": 0.012370214833595128,
|
|
"learning_rate": 7.590135046865651e-07,
|
|
"loss": 0.07622140645980835,
|
|
"num_tokens": 115615.0,
|
|
"reward": 1.65,
|
|
"reward_std": 0.3610740154981613,
|
|
"rewards/accuracy_reward_func/mean": 0.6625,
|
|
"rewards/accuracy_reward_func/std": 0.32571869492530825,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 50,
|
|
"step_time": 9.345398245193064
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 391.9,
|
|
"completions/max_terminated_length": 391.9,
|
|
"completions/mean_length": 140.5125,
|
|
"completions/mean_terminated_length": 140.5125,
|
|
"completions/min_length": 10.5,
|
|
"completions/min_terminated_length": 10.5,
|
|
"entropy": 0.4693372020497918,
|
|
"epoch": 0.4,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 0.4921875,
|
|
"kl": 0.003207334756950786,
|
|
"learning_rate": 6.644333233692916e-07,
|
|
"loss": -0.014827732741832734,
|
|
"num_tokens": 137696.0,
|
|
"reward": 1.6375,
|
|
"reward_std": 0.37656808495521543,
|
|
"rewards/accuracy_reward_func/mean": 0.6625,
|
|
"rewards/accuracy_reward_func/std": 0.3202626138925552,
|
|
"rewards/format_reward_func/mean": 0.975,
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
|
"step": 60,
|
|
"step_time": 15.56080833701417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 588.2,
|
|
"completions/max_terminated_length": 532.9,
|
|
"completions/mean_length": 150.75,
|
|
"completions/mean_terminated_length": 106.68928833007813,
|
|
"completions/min_length": 11.1,
|
|
"completions/min_terminated_length": 11.1,
|
|
"entropy": 0.4096149187302217,
|
|
"epoch": 0.4666666666666667,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 14.75,
|
|
"kl": 0.0029691428710791,
|
|
"learning_rate": 5.626666167821521e-07,
|
|
"loss": -0.02979586124420166,
|
|
"num_tokens": 161620.0,
|
|
"reward": 1.525,
|
|
"reward_std": 0.48160980343818666,
|
|
"rewards/accuracy_reward_func/mean": 0.5625,
|
|
"rewards/accuracy_reward_func/std": 0.4115981698036194,
|
|
"rewards/format_reward_func/mean": 0.9625,
|
|
"rewards/format_reward_func/std": 0.10606601536273956,
|
|
"step": 70,
|
|
"step_time": 22.58805529791862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.8,
|
|
"completions/max_terminated_length": 296.8,
|
|
"completions/mean_length": 94.425,
|
|
"completions/mean_terminated_length": 94.425,
|
|
"completions/min_length": 11.5,
|
|
"completions/min_terminated_length": 11.5,
|
|
"entropy": 0.3293402023613453,
|
|
"epoch": 0.5333333333333333,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 1.1953125,
|
|
"kl": 0.0025371614843209046,
|
|
"learning_rate": 4.5816107833384233e-07,
|
|
"loss": -0.060630393028259275,
|
|
"num_tokens": 180298.0,
|
|
"reward": 1.8,
|
|
"reward_std": 0.3336638957262039,
|
|
"rewards/accuracy_reward_func/mean": 0.8125,
|
|
"rewards/accuracy_reward_func/std": 0.3055525213479996,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 80,
|
|
"step_time": 12.384980474878102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 571.0,
|
|
"completions/max_terminated_length": 396.8,
|
|
"completions/mean_length": 169.8875,
|
|
"completions/mean_terminated_length": 145.42857208251954,
|
|
"completions/min_length": 11.6,
|
|
"completions/min_terminated_length": 11.6,
|
|
"entropy": 0.32843760270625355,
|
|
"epoch": 0.6,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.006397408900011214,
|
|
"learning_rate": 3.554841015277641e-07,
|
|
"loss": 0.06048367619514465,
|
|
"num_tokens": 204697.0,
|
|
"reward": 1.725,
|
|
"reward_std": 0.2411646068096161,
|
|
"rewards/accuracy_reward_func/mean": 0.7375,
|
|
"rewards/accuracy_reward_func/std": 0.20580926835536956,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 90,
|
|
"step_time": 21.92367275925353
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 561.9,
|
|
"completions/max_terminated_length": 436.8,
|
|
"completions/mean_length": 148.8625,
|
|
"completions/mean_terminated_length": 124.88392944335938,
|
|
"completions/min_length": 11.8,
|
|
"completions/min_terminated_length": 11.8,
|
|
"entropy": 0.385774496383965,
|
|
"epoch": 0.6666666666666666,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.05419921875,
|
|
"kl": 0.006441528908105454,
|
|
"learning_rate": 2.591231629491423e-07,
|
|
"loss": -0.09045490622520447,
|
|
"num_tokens": 227338.0,
|
|
"reward": 1.6375,
|
|
"reward_std": 0.4069852590560913,
|
|
"rewards/accuracy_reward_func/mean": 0.65,
|
|
"rewards/accuracy_reward_func/std": 0.38802950978279116,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 100,
|
|
"step_time": 21.567665427271276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.2,
|
|
"completions/max_terminated_length": 276.2,
|
|
"completions/mean_length": 65.1375,
|
|
"completions/mean_terminated_length": 65.1375,
|
|
"completions/min_length": 10.5,
|
|
"completions/min_terminated_length": 10.5,
|
|
"entropy": 0.3241632700897753,
|
|
"epoch": 0.7333333333333333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.98046875,
|
|
"kl": 0.003847465490668345,
|
|
"learning_rate": 1.7328969800494726e-07,
|
|
"loss": 0.09001727104187011,
|
|
"num_tokens": 242669.0,
|
|
"reward": 1.7375,
|
|
"reward_std": 0.4489682465791702,
|
|
"rewards/accuracy_reward_func/mean": 0.75,
|
|
"rewards/accuracy_reward_func/std": 0.428018119931221,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 110,
|
|
"step_time": 11.50382228847593
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 851.4,
|
|
"completions/max_terminated_length": 603.7,
|
|
"completions/mean_length": 275.25,
|
|
"completions/mean_terminated_length": 206.3190490722656,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.41810003817081454,
|
|
"epoch": 0.8,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.28125,
|
|
"kl": 0.011894569395826693,
|
|
"learning_rate": 1.0173504098790186e-07,
|
|
"loss": 0.12610946893692015,
|
|
"num_tokens": 275633.0,
|
|
"reward": 1.6,
|
|
"reward_std": 0.45666705071926117,
|
|
"rewards/accuracy_reward_func/mean": 0.6375,
|
|
"rewards/accuracy_reward_func/std": 0.4069852650165558,
|
|
"rewards/format_reward_func/mean": 0.9625,
|
|
"rewards/format_reward_func/std": 0.0816463440656662,
|
|
"step": 120,
|
|
"step_time": 31.933318979479374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 568.3,
|
|
"completions/max_terminated_length": 374.9,
|
|
"completions/mean_length": 130.45,
|
|
"completions/mean_terminated_length": 105.21250019073486,
|
|
"completions/min_length": 11.2,
|
|
"completions/min_terminated_length": 11.2,
|
|
"entropy": 0.45407755924388765,
|
|
"epoch": 0.8666666666666667,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 51.75,
|
|
"kl": 0.008640645261036185,
|
|
"learning_rate": 4.758647376699032e-08,
|
|
"loss": 0.14615154266357422,
|
|
"num_tokens": 297813.0,
|
|
"reward": 1.5625,
|
|
"reward_std": 0.3699325889348984,
|
|
"rewards/accuracy_reward_func/mean": 0.5875,
|
|
"rewards/accuracy_reward_func/std": 0.316480353474617,
|
|
"rewards/format_reward_func/mean": 0.975,
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
|
"step": 130,
|
|
"step_time": 21.87842119038105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 117.7,
|
|
"completions/max_terminated_length": 117.7,
|
|
"completions/mean_length": 32.7375,
|
|
"completions/mean_terminated_length": 32.7375,
|
|
"completions/min_length": 10.8,
|
|
"completions/min_terminated_length": 10.8,
|
|
"entropy": 0.40678847576491534,
|
|
"epoch": 0.9333333333333333,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.03125,
|
|
"kl": 0.005609498374019495,
|
|
"learning_rate": 1.3210548563419855e-08,
|
|
"loss": 0.04860760569572449,
|
|
"num_tokens": 311608.0,
|
|
"reward": 1.6,
|
|
"reward_std": 0.31909499764442445,
|
|
"rewards/accuracy_reward_func/mean": 0.6125,
|
|
"rewards/accuracy_reward_func/std": 0.30129902362823485,
|
|
"rewards/format_reward_func/mean": 0.9875,
|
|
"rewards/format_reward_func/std": 0.03535533845424652,
|
|
"step": 140,
|
|
"step_time": 6.0708080609329045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 390.7,
|
|
"completions/max_terminated_length": 390.7,
|
|
"completions/mean_length": 116.95,
|
|
"completions/mean_terminated_length": 116.95,
|
|
"completions/min_length": 10.9,
|
|
"completions/min_terminated_length": 10.9,
|
|
"entropy": 0.41855254471302034,
|
|
"epoch": 1.0,
|
|
"frac_reward_zero_std": 0.55,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0024387751711611116,
|
|
"learning_rate": 1.0965826257725019e-10,
|
|
"loss": -0.009271913766860962,
|
|
"num_tokens": 332084.0,
|
|
"reward": 1.4875,
|
|
"reward_std": 0.4244658529758453,
|
|
"rewards/accuracy_reward_func/mean": 0.5125,
|
|
"rewards/accuracy_reward_func/std": 0.36824311017990113,
|
|
"rewards/format_reward_func/mean": 0.975,
|
|
"rewards/format_reward_func/std": 0.07071067690849304,
|
|
"step": 150,
|
|
"step_time": 15.536289924476296
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 150,
|
|
"num_input_tokens_seen": 332084,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|