12783 lines
372 KiB
JSON
12783 lines
372 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 4.997455470737913,
|
|
"eval_steps": 500,
|
|
"global_step": 980,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.1964416503906,
|
|
"epoch": 0.005089058524173028,
|
|
"grad_norm": 0.5131905682335655,
|
|
"kl": 0.0,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0083,
|
|
"reward": 0.3035714477300644,
|
|
"reward_std": 0.3146933540701866,
|
|
"rewards/accuracy_reward": 0.3035714477300644,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.7678756713867,
|
|
"epoch": 0.010178117048346057,
|
|
"grad_norm": 0.48550660503092874,
|
|
"kl": 0.00038623809814453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0022,
|
|
"reward": 0.2812500074505806,
|
|
"reward_std": 0.3097551055252552,
|
|
"rewards/accuracy_reward": 0.2812500074505806,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.9174270629883,
|
|
"epoch": 0.015267175572519083,
|
|
"grad_norm": 0.4755285363526245,
|
|
"kl": 0.0006351470947265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0043,
|
|
"reward": 0.3035714477300644,
|
|
"reward_std": 0.31310277432203293,
|
|
"rewards/accuracy_reward": 0.3035714477300644,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.6518020629883,
|
|
"epoch": 0.020356234096692113,
|
|
"grad_norm": 0.500511345750709,
|
|
"kl": 0.00102996826171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.3660714402794838,
|
|
"reward_std": 0.31569796800613403,
|
|
"rewards/accuracy_reward": 0.3660714402794838,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.5335006713867,
|
|
"epoch": 0.02544529262086514,
|
|
"grad_norm": 0.5780194120079271,
|
|
"kl": 0.001201629638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.4062500149011612,
|
|
"reward_std": 0.3942900151014328,
|
|
"rewards/accuracy_reward": 0.4062500149011612,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.3236846923828,
|
|
"epoch": 0.030534351145038167,
|
|
"grad_norm": 0.5128731556949468,
|
|
"kl": 0.00258636474609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0005,
|
|
"reward": 0.2566964440047741,
|
|
"reward_std": 0.31038618460297585,
|
|
"rewards/accuracy_reward": 0.2566964440047741,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.28572845458984,
|
|
"epoch": 0.035623409669211195,
|
|
"grad_norm": 0.5292397662077934,
|
|
"kl": 0.0038604736328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0214,
|
|
"reward": 0.3058035895228386,
|
|
"reward_std": 0.32114920020103455,
|
|
"rewards/accuracy_reward": 0.3058035895228386,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.28126525878906,
|
|
"epoch": 0.04071246819338423,
|
|
"grad_norm": 0.4923284300392425,
|
|
"kl": 0.003711700439453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0064,
|
|
"reward": 0.2901785895228386,
|
|
"reward_std": 0.3053254038095474,
|
|
"rewards/accuracy_reward": 0.2901785895228386,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.54689025878906,
|
|
"epoch": 0.04580152671755725,
|
|
"grad_norm": 0.5444443218780471,
|
|
"kl": 0.0039520263671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0072,
|
|
"reward": 0.2857142947614193,
|
|
"reward_std": 0.3841593265533447,
|
|
"rewards/accuracy_reward": 0.2857142947614193,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.0937728881836,
|
|
"epoch": 0.05089058524173028,
|
|
"grad_norm": 1.5777669521612345,
|
|
"kl": 0.00502777099609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.009,
|
|
"reward": 0.2433035857975483,
|
|
"reward_std": 0.3450405113399029,
|
|
"rewards/accuracy_reward": 0.2433035857975483,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.0759048461914,
|
|
"epoch": 0.05597964376590331,
|
|
"grad_norm": 0.5543684439921044,
|
|
"kl": 0.00626373291015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0009,
|
|
"reward": 0.3325893022119999,
|
|
"reward_std": 0.29412248730659485,
|
|
"rewards/accuracy_reward": 0.3325893022119999,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.1986770629883,
|
|
"epoch": 0.061068702290076333,
|
|
"grad_norm": 0.49762317088896774,
|
|
"kl": 0.0060577392578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0067,
|
|
"reward": 0.3794643133878708,
|
|
"reward_std": 0.30871570855379105,
|
|
"rewards/accuracy_reward": 0.3794643133878708,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 350.9977798461914,
|
|
"epoch": 0.06615776081424936,
|
|
"grad_norm": 1714.086045615442,
|
|
"kl": 0.655517578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0215,
|
|
"reward": 0.3638392984867096,
|
|
"reward_std": 0.34932900965213776,
|
|
"rewards/accuracy_reward": 0.3638392984867096,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 357.37947845458984,
|
|
"epoch": 0.07124681933842239,
|
|
"grad_norm": 0.5527423625236983,
|
|
"kl": 0.0115203857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0088,
|
|
"reward": 0.2700892984867096,
|
|
"reward_std": 0.2972116284072399,
|
|
"rewards/accuracy_reward": 0.2700892984867096,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 337.86608123779297,
|
|
"epoch": 0.07633587786259542,
|
|
"grad_norm": 0.6203881688811241,
|
|
"kl": 0.0127716064453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0117,
|
|
"reward": 0.3727678619325161,
|
|
"reward_std": 0.3377150446176529,
|
|
"rewards/accuracy_reward": 0.3727678619325161,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 365.1651916503906,
|
|
"epoch": 0.08142493638676845,
|
|
"grad_norm": 0.4485230364798742,
|
|
"kl": 0.0117340087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0035,
|
|
"reward": 0.3236607275903225,
|
|
"reward_std": 0.23699714615941048,
|
|
"rewards/accuracy_reward": 0.3236607275903225,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 373.04689025878906,
|
|
"epoch": 0.08651399491094147,
|
|
"grad_norm": 0.5003998737790313,
|
|
"kl": 0.01090240478515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0165,
|
|
"reward": 0.2343750074505806,
|
|
"reward_std": 0.23511192575097084,
|
|
"rewards/accuracy_reward": 0.2343750074505806,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.56697845458984,
|
|
"epoch": 0.0916030534351145,
|
|
"grad_norm": 0.5300248233778917,
|
|
"kl": 0.0104217529296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.002,
|
|
"reward": 0.2723214402794838,
|
|
"reward_std": 0.2561025284230709,
|
|
"rewards/accuracy_reward": 0.2723214402794838,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.7835006713867,
|
|
"epoch": 0.09669211195928754,
|
|
"grad_norm": 0.49467929686899487,
|
|
"kl": 0.009246826171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.3482142984867096,
|
|
"reward_std": 0.2795490361750126,
|
|
"rewards/accuracy_reward": 0.3482142984867096,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.15404510498047,
|
|
"epoch": 0.10178117048346055,
|
|
"grad_norm": 0.5179413808478568,
|
|
"kl": 0.0088653564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0286,
|
|
"reward": 0.3526785969734192,
|
|
"reward_std": 0.3307727947831154,
|
|
"rewards/accuracy_reward": 0.3526785969734192,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.2834930419922,
|
|
"epoch": 0.10687022900763359,
|
|
"grad_norm": 0.48607254188968196,
|
|
"kl": 0.011444091796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.3258928693830967,
|
|
"reward_std": 0.21202195063233376,
|
|
"rewards/accuracy_reward": 0.3258928693830967,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.3995666503906,
|
|
"epoch": 0.11195928753180662,
|
|
"grad_norm": 0.4940851576048368,
|
|
"kl": 0.0086669921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0179,
|
|
"reward": 0.2299107238650322,
|
|
"reward_std": 0.28535235673189163,
|
|
"rewards/accuracy_reward": 0.2299107238650322,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.1585006713867,
|
|
"epoch": 0.11704834605597965,
|
|
"grad_norm": 0.5219084458151091,
|
|
"kl": 0.00815582275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.404017873108387,
|
|
"reward_std": 0.3285277634859085,
|
|
"rewards/accuracy_reward": 0.404017873108387,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 436.0357360839844,
|
|
"epoch": 0.12213740458015267,
|
|
"grad_norm": 0.4937449249945028,
|
|
"kl": 0.00748443603515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0023,
|
|
"reward": 0.3995535969734192,
|
|
"reward_std": 0.31111307069659233,
|
|
"rewards/accuracy_reward": 0.3995535969734192,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 447.0312728881836,
|
|
"epoch": 0.1272264631043257,
|
|
"grad_norm": 0.5247292728268708,
|
|
"kl": 0.00972747802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0027,
|
|
"reward": 0.3325892984867096,
|
|
"reward_std": 0.27157899737358093,
|
|
"rewards/accuracy_reward": 0.3325892984867096,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 453.71653747558594,
|
|
"epoch": 0.13231552162849872,
|
|
"grad_norm": 0.5495977190025394,
|
|
"kl": 0.0119781494140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.4107143059372902,
|
|
"reward_std": 0.34284864366054535,
|
|
"rewards/accuracy_reward": 0.4107143059372902,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.21876525878906,
|
|
"epoch": 0.13740458015267176,
|
|
"grad_norm": 0.47937723371612206,
|
|
"kl": 0.009246826171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.31026787124574184,
|
|
"reward_std": 0.27487244457006454,
|
|
"rewards/accuracy_reward": 0.31026787124574184,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.98663330078125,
|
|
"epoch": 0.14249363867684478,
|
|
"grad_norm": 0.5276131424693773,
|
|
"kl": 0.00921630859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0092,
|
|
"reward": 0.3281250223517418,
|
|
"reward_std": 0.29099568352103233,
|
|
"rewards/accuracy_reward": 0.3281250223517418,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.6852798461914,
|
|
"epoch": 0.1475826972010178,
|
|
"grad_norm": 0.5582899993904399,
|
|
"kl": 0.0114593505859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0066,
|
|
"reward": 0.3660714477300644,
|
|
"reward_std": 0.3422120660543442,
|
|
"rewards/accuracy_reward": 0.3660714477300644,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.3638610839844,
|
|
"epoch": 0.15267175572519084,
|
|
"grad_norm": 0.5385443874335647,
|
|
"kl": 0.0098114013671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0095,
|
|
"reward": 0.3861607387661934,
|
|
"reward_std": 0.2970792390406132,
|
|
"rewards/accuracy_reward": 0.3861607387661934,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.87500762939453,
|
|
"epoch": 0.15776081424936386,
|
|
"grad_norm": 0.5037518936326002,
|
|
"kl": 0.0113983154296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0173,
|
|
"reward": 0.2812500111758709,
|
|
"reward_std": 0.30366747453808784,
|
|
"rewards/accuracy_reward": 0.2812500111758709,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.92859649658203,
|
|
"epoch": 0.1628498727735369,
|
|
"grad_norm": 0.4410514013813698,
|
|
"kl": 0.0113983154296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0067,
|
|
"reward": 0.274553582072258,
|
|
"reward_std": 0.2705678604543209,
|
|
"rewards/accuracy_reward": 0.274553582072258,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.7968978881836,
|
|
"epoch": 0.16793893129770993,
|
|
"grad_norm": 0.5691294204541579,
|
|
"kl": 0.012908935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.3482142984867096,
|
|
"reward_std": 0.3068159222602844,
|
|
"rewards/accuracy_reward": 0.3482142984867096,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 359.5134048461914,
|
|
"epoch": 0.17302798982188294,
|
|
"grad_norm": 0.5562831699180847,
|
|
"kl": 0.0130157470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.3236607238650322,
|
|
"reward_std": 0.3009896092116833,
|
|
"rewards/accuracy_reward": 0.3236607238650322,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.4486770629883,
|
|
"epoch": 0.178117048346056,
|
|
"grad_norm": 0.46591620037609793,
|
|
"kl": 0.0124053955078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0005,
|
|
"reward": 0.2924107238650322,
|
|
"reward_std": 0.23107879608869553,
|
|
"rewards/accuracy_reward": 0.2924107238650322,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.2343978881836,
|
|
"epoch": 0.183206106870229,
|
|
"grad_norm": 0.512032862767005,
|
|
"kl": 0.012298583984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0144,
|
|
"reward": 0.3080357275903225,
|
|
"reward_std": 0.2829487472772598,
|
|
"rewards/accuracy_reward": 0.3080357275903225,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.73216247558594,
|
|
"epoch": 0.18829516539440203,
|
|
"grad_norm": 0.4691396176787084,
|
|
"kl": 0.0124969482421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0063,
|
|
"reward": 0.2946428693830967,
|
|
"reward_std": 0.27999019995331764,
|
|
"rewards/accuracy_reward": 0.2946428693830967,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 441.99332427978516,
|
|
"epoch": 0.19338422391857507,
|
|
"grad_norm": 0.49734043814964074,
|
|
"kl": 0.013275146484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0126,
|
|
"reward": 0.368303582072258,
|
|
"reward_std": 0.37057018280029297,
|
|
"rewards/accuracy_reward": 0.368303582072258,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.02903747558594,
|
|
"epoch": 0.1984732824427481,
|
|
"grad_norm": 0.42504957050666886,
|
|
"kl": 0.01385498046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0051,
|
|
"reward": 0.377232164144516,
|
|
"reward_std": 0.25631672516465187,
|
|
"rewards/accuracy_reward": 0.377232164144516,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.84153747558594,
|
|
"epoch": 0.2035623409669211,
|
|
"grad_norm": 0.48761609257933625,
|
|
"kl": 0.013336181640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.2968750111758709,
|
|
"reward_std": 0.30066392943263054,
|
|
"rewards/accuracy_reward": 0.2968750111758709,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.01341247558594,
|
|
"epoch": 0.20865139949109415,
|
|
"grad_norm": 0.47812024733447134,
|
|
"kl": 0.012115478515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.3147321566939354,
|
|
"reward_std": 0.3126324266195297,
|
|
"rewards/accuracy_reward": 0.3147321566939354,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.2321548461914,
|
|
"epoch": 0.21374045801526717,
|
|
"grad_norm": 0.43396375187819597,
|
|
"kl": 0.0133819580078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0067,
|
|
"reward": 0.25000001676380634,
|
|
"reward_std": 0.22224192321300507,
|
|
"rewards/accuracy_reward": 0.25000001676380634,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 450.66519927978516,
|
|
"epoch": 0.21882951653944022,
|
|
"grad_norm": 0.4530485486842693,
|
|
"kl": 0.0116119384765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0046,
|
|
"reward": 0.2968750149011612,
|
|
"reward_std": 0.291314922273159,
|
|
"rewards/accuracy_reward": 0.2968750149011612,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.6205520629883,
|
|
"epoch": 0.22391857506361323,
|
|
"grad_norm": 0.4882091945408692,
|
|
"kl": 0.0125579833984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0024,
|
|
"reward": 0.357142873108387,
|
|
"reward_std": 0.3270977512001991,
|
|
"rewards/accuracy_reward": 0.357142873108387,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.82591247558594,
|
|
"epoch": 0.22900763358778625,
|
|
"grad_norm": 0.4906741745649979,
|
|
"kl": 0.0128021240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0061,
|
|
"reward": 0.4441964477300644,
|
|
"reward_std": 0.2941880486905575,
|
|
"rewards/accuracy_reward": 0.4441964477300644,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 438.5357360839844,
|
|
"epoch": 0.2340966921119593,
|
|
"grad_norm": 0.5026482980354947,
|
|
"kl": 0.011474609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.4174107313156128,
|
|
"reward_std": 0.3358287177979946,
|
|
"rewards/accuracy_reward": 0.4174107313156128,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.8861846923828,
|
|
"epoch": 0.23918575063613232,
|
|
"grad_norm": 0.45279918851719525,
|
|
"kl": 0.0128326416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0052,
|
|
"reward": 0.2276785783469677,
|
|
"reward_std": 0.25647593289613724,
|
|
"rewards/accuracy_reward": 0.2276785783469677,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.22769927978516,
|
|
"epoch": 0.24427480916030533,
|
|
"grad_norm": 0.5454276173522247,
|
|
"kl": 0.0137786865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0065,
|
|
"reward": 0.330357164144516,
|
|
"reward_std": 0.3306938484311104,
|
|
"rewards/accuracy_reward": 0.330357164144516,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.2299346923828,
|
|
"epoch": 0.24936386768447838,
|
|
"grad_norm": 0.5348141948553139,
|
|
"kl": 0.0132598876953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.4151785895228386,
|
|
"reward_std": 0.3523707911372185,
|
|
"rewards/accuracy_reward": 0.4151785895228386,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.0625228881836,
|
|
"epoch": 0.2544529262086514,
|
|
"grad_norm": 0.5085458953679328,
|
|
"kl": 0.0128021240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0021,
|
|
"reward": 0.522321455180645,
|
|
"reward_std": 0.33058052510023117,
|
|
"rewards/accuracy_reward": 0.522321455180645,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.6897506713867,
|
|
"epoch": 0.2595419847328244,
|
|
"grad_norm": 1518291951656.6533,
|
|
"kl": 7079985152.011017,
|
|
"learning_rate": 2e-06,
|
|
"loss": 117960256.0,
|
|
"reward": 0.3258928768336773,
|
|
"reward_std": 0.2740046866238117,
|
|
"rewards/accuracy_reward": 0.3258928768336773,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.23439025878906,
|
|
"epoch": 0.26463104325699743,
|
|
"grad_norm": 0.5514192754520774,
|
|
"kl": 0.0154571533203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.223214291036129,
|
|
"reward_std": 0.2859731763601303,
|
|
"rewards/accuracy_reward": 0.223214291036129,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.78572845458984,
|
|
"epoch": 0.2697201017811705,
|
|
"grad_norm": 0.4865275565102642,
|
|
"kl": 0.0140380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0111,
|
|
"reward": 0.2812500074505806,
|
|
"reward_std": 0.28903238475322723,
|
|
"rewards/accuracy_reward": 0.2812500074505806,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.07814025878906,
|
|
"epoch": 0.2748091603053435,
|
|
"grad_norm": 0.5784731742120693,
|
|
"kl": 0.015106201171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.3214285783469677,
|
|
"reward_std": 0.33232294768095016,
|
|
"rewards/accuracy_reward": 0.3214285783469677,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 361.86832427978516,
|
|
"epoch": 0.27989821882951654,
|
|
"grad_norm": 0.5517150744071869,
|
|
"kl": 0.017242431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0016,
|
|
"reward": 0.428571455180645,
|
|
"reward_std": 0.34388555958867073,
|
|
"rewards/accuracy_reward": 0.428571455180645,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.1629638671875,
|
|
"epoch": 0.28498727735368956,
|
|
"grad_norm": 0.4914644497587217,
|
|
"kl": 0.01690673828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.3794643022119999,
|
|
"reward_std": 0.27357952296733856,
|
|
"rewards/accuracy_reward": 0.3794643022119999,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.62501525878906,
|
|
"epoch": 0.2900763358778626,
|
|
"grad_norm": 0.5305247627017694,
|
|
"kl": 0.0151519775390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.3750000149011612,
|
|
"reward_std": 0.36491215601563454,
|
|
"rewards/accuracy_reward": 0.3750000149011612,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.9598388671875,
|
|
"epoch": 0.2951653944020356,
|
|
"grad_norm": 0.5355122953339883,
|
|
"kl": 0.0160980224609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0043,
|
|
"reward": 0.3147321566939354,
|
|
"reward_std": 0.33825405687093735,
|
|
"rewards/accuracy_reward": 0.3147321566939354,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.5580520629883,
|
|
"epoch": 0.30025445292620867,
|
|
"grad_norm": 0.4984115312555115,
|
|
"kl": 0.014129638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.3750000260770321,
|
|
"reward_std": 0.2658737301826477,
|
|
"rewards/accuracy_reward": 0.3750000260770321,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.7991256713867,
|
|
"epoch": 0.3053435114503817,
|
|
"grad_norm": 0.48379704997835704,
|
|
"kl": 0.0148162841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.4040178656578064,
|
|
"reward_std": 0.2832214683294296,
|
|
"rewards/accuracy_reward": 0.4040178656578064,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.7276916503906,
|
|
"epoch": 0.3104325699745547,
|
|
"grad_norm": 0.5414065913568801,
|
|
"kl": 0.013671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0064,
|
|
"reward": 0.4174107238650322,
|
|
"reward_std": 0.32154662907123566,
|
|
"rewards/accuracy_reward": 0.4174107238650322,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.85939025878906,
|
|
"epoch": 0.3155216284987277,
|
|
"grad_norm": 0.5719771747244955,
|
|
"kl": 0.0153045654296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.014,
|
|
"reward": 0.404017873108387,
|
|
"reward_std": 0.3457542136311531,
|
|
"rewards/accuracy_reward": 0.404017873108387,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.0736770629883,
|
|
"epoch": 0.32061068702290074,
|
|
"grad_norm": 0.5063681403358649,
|
|
"kl": 0.016845703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0128,
|
|
"reward": 0.3035714440047741,
|
|
"reward_std": 0.32448188215494156,
|
|
"rewards/accuracy_reward": 0.3035714440047741,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.88172149658203,
|
|
"epoch": 0.3256997455470738,
|
|
"grad_norm": 0.4876071845124516,
|
|
"kl": 0.015899658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.2633928693830967,
|
|
"reward_std": 0.27684302255511284,
|
|
"rewards/accuracy_reward": 0.2633928693830967,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.57814025878906,
|
|
"epoch": 0.33078880407124683,
|
|
"grad_norm": 0.5032443746200898,
|
|
"kl": 0.0156707763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0127,
|
|
"reward": 0.3950892984867096,
|
|
"reward_std": 0.3091568648815155,
|
|
"rewards/accuracy_reward": 0.3950892984867096,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.8973388671875,
|
|
"epoch": 0.33587786259541985,
|
|
"grad_norm": 0.5211229249047339,
|
|
"kl": 0.0162353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0062,
|
|
"reward": 0.4553571492433548,
|
|
"reward_std": 0.3269530087709427,
|
|
"rewards/accuracy_reward": 0.4553571492433548,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.35716247558594,
|
|
"epoch": 0.34096692111959287,
|
|
"grad_norm": 0.48245726026970265,
|
|
"kl": 0.01739501953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0172,
|
|
"reward": 0.379464291036129,
|
|
"reward_std": 0.30845845863223076,
|
|
"rewards/accuracy_reward": 0.379464291036129,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 354.5245666503906,
|
|
"epoch": 0.3460559796437659,
|
|
"grad_norm": 1531182.3047376974,
|
|
"kl": 7968.012969970703,
|
|
"learning_rate": 2e-06,
|
|
"loss": 295.9489,
|
|
"reward": 0.3236607238650322,
|
|
"reward_std": 0.3228748217225075,
|
|
"rewards/accuracy_reward": 0.3236607238650322,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 364.82591247558594,
|
|
"epoch": 0.3511450381679389,
|
|
"grad_norm": 0.4911830724591876,
|
|
"kl": 0.0185394287109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.016,
|
|
"reward": 0.3526785895228386,
|
|
"reward_std": 0.24602409452199936,
|
|
"rewards/accuracy_reward": 0.3526785895228386,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.0825958251953,
|
|
"epoch": 0.356234096692112,
|
|
"grad_norm": 0.49807944084033023,
|
|
"kl": 0.0153961181640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0054,
|
|
"reward": 0.4017857387661934,
|
|
"reward_std": 0.29982316493988037,
|
|
"rewards/accuracy_reward": 0.4017857387661934,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 373.26564025878906,
|
|
"epoch": 0.361323155216285,
|
|
"grad_norm": 0.5036334041571832,
|
|
"kl": 0.0206298828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.01,
|
|
"reward": 0.3236607313156128,
|
|
"reward_std": 0.29557934775948524,
|
|
"rewards/accuracy_reward": 0.3236607313156128,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.5089416503906,
|
|
"epoch": 0.366412213740458,
|
|
"grad_norm": 0.39407939476751,
|
|
"kl": 0.01800537109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.002,
|
|
"reward": 0.33258930779993534,
|
|
"reward_std": 0.21618272736668587,
|
|
"rewards/accuracy_reward": 0.33258930779993534,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.0826110839844,
|
|
"epoch": 0.37150127226463103,
|
|
"grad_norm": 0.5130185552836011,
|
|
"kl": 0.01806640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0027,
|
|
"reward": 0.33482143096625805,
|
|
"reward_std": 0.3136202208697796,
|
|
"rewards/accuracy_reward": 0.33482143096625805,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.50894927978516,
|
|
"epoch": 0.37659033078880405,
|
|
"grad_norm": 0.5177463812190622,
|
|
"kl": 0.019622802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.3169643096625805,
|
|
"reward_std": 0.2695329450070858,
|
|
"rewards/accuracy_reward": 0.3169643096625805,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.32592010498047,
|
|
"epoch": 0.3816793893129771,
|
|
"grad_norm": 0.5440731331635377,
|
|
"kl": 0.02008056640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.3102678656578064,
|
|
"reward_std": 0.2793062496930361,
|
|
"rewards/accuracy_reward": 0.3102678656578064,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 376.77010345458984,
|
|
"epoch": 0.38676844783715014,
|
|
"grad_norm": 0.547175246797069,
|
|
"kl": 0.01873779296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.3147321604192257,
|
|
"reward_std": 0.2940896600484848,
|
|
"rewards/accuracy_reward": 0.3147321604192257,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.57813262939453,
|
|
"epoch": 0.39185750636132316,
|
|
"grad_norm": 0.5229440696967318,
|
|
"kl": 0.0159454345703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0054,
|
|
"reward": 0.3125000111758709,
|
|
"reward_std": 0.3128367140889168,
|
|
"rewards/accuracy_reward": 0.3125000111758709,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.7254638671875,
|
|
"epoch": 0.3969465648854962,
|
|
"grad_norm": 0.49808120455381655,
|
|
"kl": 0.017059326171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.4598214477300644,
|
|
"reward_std": 0.331941694021225,
|
|
"rewards/accuracy_reward": 0.4598214477300644,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.88394927978516,
|
|
"epoch": 0.4020356234096692,
|
|
"grad_norm": 0.4672596874629748,
|
|
"kl": 0.0201416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0093,
|
|
"reward": 0.3214285895228386,
|
|
"reward_std": 0.269159983843565,
|
|
"rewards/accuracy_reward": 0.3214285895228386,
|
|
"step": 79
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.1964569091797,
|
|
"epoch": 0.4071246819338422,
|
|
"grad_norm": 0.5054012743482283,
|
|
"kl": 0.0167999267578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0139,
|
|
"reward": 0.4508928805589676,
|
|
"reward_std": 0.3098481632769108,
|
|
"rewards/accuracy_reward": 0.4508928805589676,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.5178756713867,
|
|
"epoch": 0.4122137404580153,
|
|
"grad_norm": 0.5139101758762243,
|
|
"kl": 0.0169830322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.33705358766019344,
|
|
"reward_std": 0.2844383306801319,
|
|
"rewards/accuracy_reward": 0.33705358766019344,
|
|
"step": 81
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.6317138671875,
|
|
"epoch": 0.4173027989821883,
|
|
"grad_norm": 0.48587084836771205,
|
|
"kl": 0.0179290771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0078,
|
|
"reward": 0.2611607238650322,
|
|
"reward_std": 0.3100224882364273,
|
|
"rewards/accuracy_reward": 0.2611607238650322,
|
|
"step": 82
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.8727798461914,
|
|
"epoch": 0.4223918575063613,
|
|
"grad_norm": 0.4928569542160694,
|
|
"kl": 0.018310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.3348214477300644,
|
|
"reward_std": 0.3120606653392315,
|
|
"rewards/accuracy_reward": 0.3348214477300644,
|
|
"step": 83
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.92189025878906,
|
|
"epoch": 0.42748091603053434,
|
|
"grad_norm": 0.42531892065233795,
|
|
"kl": 0.016448974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0082,
|
|
"reward": 0.3816964402794838,
|
|
"reward_std": 0.25463341921567917,
|
|
"rewards/accuracy_reward": 0.3816964402794838,
|
|
"step": 84
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.4174270629883,
|
|
"epoch": 0.43256997455470736,
|
|
"grad_norm": 0.5241773061730735,
|
|
"kl": 0.019378662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0074,
|
|
"reward": 0.3571428693830967,
|
|
"reward_std": 0.2619221694767475,
|
|
"rewards/accuracy_reward": 0.3571428693830967,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.58260345458984,
|
|
"epoch": 0.43765903307888043,
|
|
"grad_norm": 0.4781800627435873,
|
|
"kl": 0.0172882080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.3459821715950966,
|
|
"reward_std": 0.2790309302508831,
|
|
"rewards/accuracy_reward": 0.3459821715950966,
|
|
"step": 86
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.4241256713867,
|
|
"epoch": 0.44274809160305345,
|
|
"grad_norm": 0.5003740783291474,
|
|
"kl": 0.0181884765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0166,
|
|
"reward": 0.3147321492433548,
|
|
"reward_std": 0.2965996153652668,
|
|
"rewards/accuracy_reward": 0.3147321492433548,
|
|
"step": 87
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.17188262939453,
|
|
"epoch": 0.44783715012722647,
|
|
"grad_norm": 0.5173256984473651,
|
|
"kl": 0.01934814453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.341517873108387,
|
|
"reward_std": 0.28107255324721336,
|
|
"rewards/accuracy_reward": 0.341517873108387,
|
|
"step": 88
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.31028747558594,
|
|
"epoch": 0.4529262086513995,
|
|
"grad_norm": 0.4509863293454374,
|
|
"kl": 0.0174713134765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.007,
|
|
"reward": 0.2500000186264515,
|
|
"reward_std": 0.2574096694588661,
|
|
"rewards/accuracy_reward": 0.2500000186264515,
|
|
"step": 89
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.56251525878906,
|
|
"epoch": 0.4580152671755725,
|
|
"grad_norm": 0.4766616285716767,
|
|
"kl": 0.01788330078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0097,
|
|
"reward": 0.19419643841683865,
|
|
"reward_std": 0.2207874357700348,
|
|
"rewards/accuracy_reward": 0.19419643841683865,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.45314025878906,
|
|
"epoch": 0.4631043256997455,
|
|
"grad_norm": 0.4959895156830065,
|
|
"kl": 0.02020263671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0128,
|
|
"reward": 0.4129464402794838,
|
|
"reward_std": 0.2791617177426815,
|
|
"rewards/accuracy_reward": 0.4129464402794838,
|
|
"step": 91
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.78126525878906,
|
|
"epoch": 0.4681933842239186,
|
|
"grad_norm": 0.4507057246808986,
|
|
"kl": 0.0174560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0074,
|
|
"reward": 0.337053582072258,
|
|
"reward_std": 0.2914833575487137,
|
|
"rewards/accuracy_reward": 0.337053582072258,
|
|
"step": 92
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.45538330078125,
|
|
"epoch": 0.4732824427480916,
|
|
"grad_norm": 0.48879525342887997,
|
|
"kl": 0.019073486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0083,
|
|
"reward": 0.2633928656578064,
|
|
"reward_std": 0.25695963576436043,
|
|
"rewards/accuracy_reward": 0.2633928656578064,
|
|
"step": 93
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.94422149658203,
|
|
"epoch": 0.47837150127226463,
|
|
"grad_norm": 0.5190734752226046,
|
|
"kl": 0.018524169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0191,
|
|
"reward": 0.3861607313156128,
|
|
"reward_std": 0.36641061305999756,
|
|
"rewards/accuracy_reward": 0.3861607313156128,
|
|
"step": 94
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.5357360839844,
|
|
"epoch": 0.48346055979643765,
|
|
"grad_norm": 0.5370357401422521,
|
|
"kl": 0.0185546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0127,
|
|
"reward": 0.3571428768336773,
|
|
"reward_std": 0.2888643816113472,
|
|
"rewards/accuracy_reward": 0.3571428768336773,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.84376525878906,
|
|
"epoch": 0.48854961832061067,
|
|
"grad_norm": 0.4674788354621914,
|
|
"kl": 0.019195556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0152,
|
|
"reward": 0.2767857201397419,
|
|
"reward_std": 0.22408967837691307,
|
|
"rewards/accuracy_reward": 0.2767857201397419,
|
|
"step": 96
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.6049270629883,
|
|
"epoch": 0.49363867684478374,
|
|
"grad_norm": 0.5122986427613485,
|
|
"kl": 0.01995849609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.3392857275903225,
|
|
"reward_std": 0.29930707439780235,
|
|
"rewards/accuracy_reward": 0.3392857275903225,
|
|
"step": 97
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.98439025878906,
|
|
"epoch": 0.49872773536895676,
|
|
"grad_norm": 0.4555295504049037,
|
|
"kl": 0.0169677734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.464285746216774,
|
|
"reward_std": 0.2649185098707676,
|
|
"rewards/accuracy_reward": 0.464285746216774,
|
|
"step": 98
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.53572845458984,
|
|
"epoch": 0.5038167938931297,
|
|
"grad_norm": 0.4810120312253362,
|
|
"kl": 0.0220947265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0033,
|
|
"reward": 0.3191964477300644,
|
|
"reward_std": 0.2573278546333313,
|
|
"rewards/accuracy_reward": 0.3191964477300644,
|
|
"step": 99
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.0335006713867,
|
|
"epoch": 0.5089058524173028,
|
|
"grad_norm": 0.49415169467067754,
|
|
"kl": 0.020294189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.3214285857975483,
|
|
"reward_std": 0.3164416328072548,
|
|
"rewards/accuracy_reward": 0.3214285857975483,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.0424270629883,
|
|
"epoch": 0.5139949109414759,
|
|
"grad_norm": 0.5008631838848484,
|
|
"kl": 0.020355224609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.3638392984867096,
|
|
"reward_std": 0.312275193631649,
|
|
"rewards/accuracy_reward": 0.3638392984867096,
|
|
"step": 101
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.3527069091797,
|
|
"epoch": 0.5190839694656488,
|
|
"grad_norm": 0.4500688398225032,
|
|
"kl": 0.021392822265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0114,
|
|
"reward": 0.247767873108387,
|
|
"reward_std": 0.2502409480512142,
|
|
"rewards/accuracy_reward": 0.247767873108387,
|
|
"step": 102
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.0357360839844,
|
|
"epoch": 0.5241730279898219,
|
|
"grad_norm": 0.5522616500891663,
|
|
"kl": 0.019317626953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0139,
|
|
"reward": 0.4531250223517418,
|
|
"reward_std": 0.34994350373744965,
|
|
"rewards/accuracy_reward": 0.4531250223517418,
|
|
"step": 103
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 435.9174270629883,
|
|
"epoch": 0.5292620865139949,
|
|
"grad_norm": 0.5016605132656401,
|
|
"kl": 0.020416259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0019,
|
|
"reward": 0.4464285895228386,
|
|
"reward_std": 0.3158109188079834,
|
|
"rewards/accuracy_reward": 0.4464285895228386,
|
|
"step": 104
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.1830520629883,
|
|
"epoch": 0.5343511450381679,
|
|
"grad_norm": 0.5508999015132893,
|
|
"kl": 0.022491455078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.3593750260770321,
|
|
"reward_std": 0.29125937074422836,
|
|
"rewards/accuracy_reward": 0.3593750260770321,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.0067138671875,
|
|
"epoch": 0.539440203562341,
|
|
"grad_norm": 8.285535898929085,
|
|
"kl": 0.1497802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0127,
|
|
"reward": 0.3258928805589676,
|
|
"reward_std": 0.31499721854925156,
|
|
"rewards/accuracy_reward": 0.3258928805589676,
|
|
"step": 106
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.1629638671875,
|
|
"epoch": 0.544529262086514,
|
|
"grad_norm": 0.560681049322606,
|
|
"kl": 0.02166748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.4107143059372902,
|
|
"reward_std": 0.3652753680944443,
|
|
"rewards/accuracy_reward": 0.4107143059372902,
|
|
"step": 107
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.8951110839844,
|
|
"epoch": 0.549618320610687,
|
|
"grad_norm": 0.5057047183097904,
|
|
"kl": 0.02301025390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.003,
|
|
"reward": 0.3683035969734192,
|
|
"reward_std": 0.2857242077589035,
|
|
"rewards/accuracy_reward": 0.3683035969734192,
|
|
"step": 108
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 433.0134048461914,
|
|
"epoch": 0.55470737913486,
|
|
"grad_norm": 0.5291985319332276,
|
|
"kl": 0.0194854736328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0045,
|
|
"reward": 0.3415178619325161,
|
|
"reward_std": 0.34176725894212723,
|
|
"rewards/accuracy_reward": 0.3415178619325161,
|
|
"step": 109
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.2611770629883,
|
|
"epoch": 0.5597964376590331,
|
|
"grad_norm": 0.5318844814949354,
|
|
"kl": 0.023834228515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0058,
|
|
"reward": 0.3236607350409031,
|
|
"reward_std": 0.2054804302752018,
|
|
"rewards/accuracy_reward": 0.3236607350409031,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 458.0491256713867,
|
|
"epoch": 0.5648854961832062,
|
|
"grad_norm": 0.4732647168414915,
|
|
"kl": 0.017913818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.009,
|
|
"reward": 0.3593750223517418,
|
|
"reward_std": 0.3117176443338394,
|
|
"rewards/accuracy_reward": 0.3593750223517418,
|
|
"step": 111
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.0625228881836,
|
|
"epoch": 0.5699745547073791,
|
|
"grad_norm": 0.4403194436230484,
|
|
"kl": 0.01910400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.450892873108387,
|
|
"reward_std": 0.29527556151151657,
|
|
"rewards/accuracy_reward": 0.450892873108387,
|
|
"step": 112
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.1942138671875,
|
|
"epoch": 0.5750636132315522,
|
|
"grad_norm": 0.585103280065545,
|
|
"kl": 0.0179443359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0097,
|
|
"reward": 0.3147321492433548,
|
|
"reward_std": 0.32731254771351814,
|
|
"rewards/accuracy_reward": 0.3147321492433548,
|
|
"step": 113
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.4107360839844,
|
|
"epoch": 0.5801526717557252,
|
|
"grad_norm": 0.499338643029717,
|
|
"kl": 0.0187530517578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.3727678805589676,
|
|
"reward_std": 0.30609316378831863,
|
|
"rewards/accuracy_reward": 0.3727678805589676,
|
|
"step": 114
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.26564025878906,
|
|
"epoch": 0.5852417302798982,
|
|
"grad_norm": 0.5229032264133338,
|
|
"kl": 0.0181884765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0128,
|
|
"reward": 0.3950893059372902,
|
|
"reward_std": 0.2910423092544079,
|
|
"rewards/accuracy_reward": 0.3950893059372902,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.41295623779297,
|
|
"epoch": 0.5903307888040712,
|
|
"grad_norm": 0.46415048500586265,
|
|
"kl": 0.0173492431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.3482142984867096,
|
|
"reward_std": 0.2729547992348671,
|
|
"rewards/accuracy_reward": 0.3482142984867096,
|
|
"step": 116
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.7611770629883,
|
|
"epoch": 0.5954198473282443,
|
|
"grad_norm": 0.5151821080636549,
|
|
"kl": 0.017333984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.3236607350409031,
|
|
"reward_std": 0.3345264159142971,
|
|
"rewards/accuracy_reward": 0.3236607350409031,
|
|
"step": 117
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.51564025878906,
|
|
"epoch": 0.6005089058524173,
|
|
"grad_norm": 0.5375586636437247,
|
|
"kl": 0.018890380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0204,
|
|
"reward": 0.4263393059372902,
|
|
"reward_std": 0.35839200019836426,
|
|
"rewards/accuracy_reward": 0.4263393059372902,
|
|
"step": 118
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.0736770629883,
|
|
"epoch": 0.6055979643765903,
|
|
"grad_norm": 0.4459136396200577,
|
|
"kl": 0.01971435546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0028,
|
|
"reward": 0.3392857275903225,
|
|
"reward_std": 0.2719515413045883,
|
|
"rewards/accuracy_reward": 0.3392857275903225,
|
|
"step": 119
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.1272506713867,
|
|
"epoch": 0.6106870229007634,
|
|
"grad_norm": 0.44422677688912865,
|
|
"kl": 0.01959228515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0103,
|
|
"reward": 0.2812500074505806,
|
|
"reward_std": 0.24111849442124367,
|
|
"rewards/accuracy_reward": 0.2812500074505806,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.32144927978516,
|
|
"epoch": 0.6157760814249363,
|
|
"grad_norm": 0.5333976032222449,
|
|
"kl": 0.01995849609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0184,
|
|
"reward": 0.3861607313156128,
|
|
"reward_std": 0.3275603875517845,
|
|
"rewards/accuracy_reward": 0.3861607313156128,
|
|
"step": 121
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.2768020629883,
|
|
"epoch": 0.6208651399491094,
|
|
"grad_norm": 0.5631596318868146,
|
|
"kl": 0.020294189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.015,
|
|
"reward": 0.3794643059372902,
|
|
"reward_std": 0.2962530180811882,
|
|
"rewards/accuracy_reward": 0.3794643059372902,
|
|
"step": 122
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 373.7366180419922,
|
|
"epoch": 0.6259541984732825,
|
|
"grad_norm": 0.5473422513057858,
|
|
"kl": 0.02227783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.3526785895228386,
|
|
"reward_std": 0.3441708981990814,
|
|
"rewards/accuracy_reward": 0.3526785895228386,
|
|
"step": 123
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.9018096923828,
|
|
"epoch": 0.6310432569974554,
|
|
"grad_norm": 0.44013792020862513,
|
|
"kl": 0.018707275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0034,
|
|
"reward": 0.4575893133878708,
|
|
"reward_std": 0.24109597876667976,
|
|
"rewards/accuracy_reward": 0.4575893133878708,
|
|
"step": 124
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 353.28572845458984,
|
|
"epoch": 0.6361323155216285,
|
|
"grad_norm": 0.481985000927126,
|
|
"kl": 0.021728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0025,
|
|
"reward": 0.352678582072258,
|
|
"reward_std": 0.24602680280804634,
|
|
"rewards/accuracy_reward": 0.352678582072258,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 375.2701110839844,
|
|
"epoch": 0.6412213740458015,
|
|
"grad_norm": 0.4547008231526985,
|
|
"kl": 0.022247314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.2968750111758709,
|
|
"reward_std": 0.22678174823522568,
|
|
"rewards/accuracy_reward": 0.2968750111758709,
|
|
"step": 126
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.3192138671875,
|
|
"epoch": 0.6463104325699746,
|
|
"grad_norm": 0.8872411450857681,
|
|
"kl": 0.01995849609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0099,
|
|
"reward": 0.2611607238650322,
|
|
"reward_std": 0.26269694045186043,
|
|
"rewards/accuracy_reward": 0.2611607238650322,
|
|
"step": 127
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 379.0669860839844,
|
|
"epoch": 0.6513994910941476,
|
|
"grad_norm": 0.5408514885200469,
|
|
"kl": 0.021697998046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0106,
|
|
"reward": 0.4308035895228386,
|
|
"reward_std": 0.28540412336587906,
|
|
"rewards/accuracy_reward": 0.4308035895228386,
|
|
"step": 128
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.52457427978516,
|
|
"epoch": 0.6564885496183206,
|
|
"grad_norm": 0.5282900476478553,
|
|
"kl": 0.023101806640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0052,
|
|
"reward": 0.345982164144516,
|
|
"reward_std": 0.3362126648426056,
|
|
"rewards/accuracy_reward": 0.345982164144516,
|
|
"step": 129
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.7701110839844,
|
|
"epoch": 0.6615776081424937,
|
|
"grad_norm": 0.5250549946167662,
|
|
"kl": 0.02166748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.3013393022119999,
|
|
"reward_std": 0.27612515166401863,
|
|
"rewards/accuracy_reward": 0.3013393022119999,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.2768020629883,
|
|
"epoch": 0.6666666666666666,
|
|
"grad_norm": 0.500657457264774,
|
|
"kl": 0.01947021484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.357142873108387,
|
|
"reward_std": 0.31440506875514984,
|
|
"rewards/accuracy_reward": 0.357142873108387,
|
|
"step": 131
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.7098388671875,
|
|
"epoch": 0.6717557251908397,
|
|
"grad_norm": 0.5116768708151028,
|
|
"kl": 0.018798828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.2790178656578064,
|
|
"reward_std": 0.31369929388165474,
|
|
"rewards/accuracy_reward": 0.2790178656578064,
|
|
"step": 132
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.84154510498047,
|
|
"epoch": 0.6768447837150128,
|
|
"grad_norm": 0.5323305032149934,
|
|
"kl": 0.021087646484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0115,
|
|
"reward": 0.3526785783469677,
|
|
"reward_std": 0.324299618601799,
|
|
"rewards/accuracy_reward": 0.3526785783469677,
|
|
"step": 133
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.5491256713867,
|
|
"epoch": 0.6819338422391857,
|
|
"grad_norm": 0.49075673954093485,
|
|
"kl": 0.021514892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0009,
|
|
"reward": 0.2901785857975483,
|
|
"reward_std": 0.31014423072338104,
|
|
"rewards/accuracy_reward": 0.2901785857975483,
|
|
"step": 134
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.2053756713867,
|
|
"epoch": 0.6870229007633588,
|
|
"grad_norm": 0.5048769982735202,
|
|
"kl": 0.02349853515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0059,
|
|
"reward": 0.3035714402794838,
|
|
"reward_std": 0.2917110025882721,
|
|
"rewards/accuracy_reward": 0.3035714402794838,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 458.18082427978516,
|
|
"epoch": 0.6921119592875318,
|
|
"grad_norm": 0.4036350502302806,
|
|
"kl": 0.02081298828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0069,
|
|
"reward": 0.3660714402794838,
|
|
"reward_std": 0.22461625561118126,
|
|
"rewards/accuracy_reward": 0.3660714402794838,
|
|
"step": 136
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.10047149658203,
|
|
"epoch": 0.6972010178117048,
|
|
"grad_norm": 0.4490091972981054,
|
|
"kl": 0.022430419921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.2924107275903225,
|
|
"reward_std": 0.2619885765016079,
|
|
"rewards/accuracy_reward": 0.2924107275903225,
|
|
"step": 137
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.5714416503906,
|
|
"epoch": 0.7022900763358778,
|
|
"grad_norm": 0.4789993319261546,
|
|
"kl": 0.023834228515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.4062500149011612,
|
|
"reward_std": 0.22731737792491913,
|
|
"rewards/accuracy_reward": 0.4062500149011612,
|
|
"step": 138
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.3348388671875,
|
|
"epoch": 0.7073791348600509,
|
|
"grad_norm": 0.5687204248492608,
|
|
"kl": 0.02569580078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0093,
|
|
"reward": 0.321428582072258,
|
|
"reward_std": 0.26937170699238777,
|
|
"rewards/accuracy_reward": 0.321428582072258,
|
|
"step": 139
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.02010345458984,
|
|
"epoch": 0.712468193384224,
|
|
"grad_norm": 0.46015246488914435,
|
|
"kl": 0.021484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0171,
|
|
"reward": 0.5089285895228386,
|
|
"reward_std": 0.2650904916226864,
|
|
"rewards/accuracy_reward": 0.5089285895228386,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.76788330078125,
|
|
"epoch": 0.7175572519083969,
|
|
"grad_norm": 0.5545229461003499,
|
|
"kl": 0.022003173828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.3772321566939354,
|
|
"reward_std": 0.3813677653670311,
|
|
"rewards/accuracy_reward": 0.3772321566939354,
|
|
"step": 141
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.89957427978516,
|
|
"epoch": 0.72264631043257,
|
|
"grad_norm": 0.6084536376433793,
|
|
"kl": 0.025726318359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0156,
|
|
"reward": 0.3482143022119999,
|
|
"reward_std": 0.3355560079216957,
|
|
"rewards/accuracy_reward": 0.3482143022119999,
|
|
"step": 142
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.20091247558594,
|
|
"epoch": 0.727735368956743,
|
|
"grad_norm": 0.4972764291068402,
|
|
"kl": 0.024444580078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0005,
|
|
"reward": 0.3191964440047741,
|
|
"reward_std": 0.3096605874598026,
|
|
"rewards/accuracy_reward": 0.3191964440047741,
|
|
"step": 143
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 363.8817138671875,
|
|
"epoch": 0.732824427480916,
|
|
"grad_norm": 0.5557362711506282,
|
|
"kl": 0.027069091796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0124,
|
|
"reward": 0.3437500074505806,
|
|
"reward_std": 0.258017111569643,
|
|
"rewards/accuracy_reward": 0.3437500074505806,
|
|
"step": 144
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.1428680419922,
|
|
"epoch": 0.7379134860050891,
|
|
"grad_norm": 0.5670857388133471,
|
|
"kl": 0.026580810546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.3125000149011612,
|
|
"reward_std": 0.33708399534225464,
|
|
"rewards/accuracy_reward": 0.3125000149011612,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 370.0870666503906,
|
|
"epoch": 0.7430025445292621,
|
|
"grad_norm": 0.568961626764239,
|
|
"kl": 0.026275634765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0059,
|
|
"reward": 0.4598214402794838,
|
|
"reward_std": 0.315539114177227,
|
|
"rewards/accuracy_reward": 0.4598214402794838,
|
|
"step": 146
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 363.8147430419922,
|
|
"epoch": 0.7480916030534351,
|
|
"grad_norm": 0.4740036629407517,
|
|
"kl": 0.026611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0062,
|
|
"reward": 0.4754464477300644,
|
|
"reward_std": 0.26761702448129654,
|
|
"rewards/accuracy_reward": 0.4754464477300644,
|
|
"step": 147
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 375.73885345458984,
|
|
"epoch": 0.7531806615776081,
|
|
"grad_norm": 0.4295908650666131,
|
|
"kl": 0.02606201171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.4107143059372902,
|
|
"reward_std": 0.21619067713618279,
|
|
"rewards/accuracy_reward": 0.4107143059372902,
|
|
"step": 148
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 350.50001525878906,
|
|
"epoch": 0.7582697201017812,
|
|
"grad_norm": 0.5712708207232068,
|
|
"kl": 0.028656005859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.032,
|
|
"reward": 0.3325892947614193,
|
|
"reward_std": 0.3545599430799484,
|
|
"rewards/accuracy_reward": 0.3325892947614193,
|
|
"step": 149
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 353.7835006713867,
|
|
"epoch": 0.7633587786259542,
|
|
"grad_norm": 0.4762004238380701,
|
|
"kl": 0.028167724609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0093,
|
|
"reward": 0.4151785895228386,
|
|
"reward_std": 0.2631988301873207,
|
|
"rewards/accuracy_reward": 0.4151785895228386,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 362.5602722167969,
|
|
"epoch": 0.7684478371501272,
|
|
"grad_norm": 0.5534272693869284,
|
|
"kl": 0.0289306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.2879464477300644,
|
|
"reward_std": 0.3091566897928715,
|
|
"rewards/accuracy_reward": 0.2879464477300644,
|
|
"step": 151
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 323.91519927978516,
|
|
"epoch": 0.7735368956743003,
|
|
"grad_norm": 0.5997096089269098,
|
|
"kl": 0.032501220703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0006,
|
|
"reward": 0.2790178656578064,
|
|
"reward_std": 0.27878880500793457,
|
|
"rewards/accuracy_reward": 0.2790178656578064,
|
|
"step": 152
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 321.62947845458984,
|
|
"epoch": 0.7786259541984732,
|
|
"grad_norm": 0.660276718752937,
|
|
"kl": 0.0316162109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.3616071566939354,
|
|
"reward_std": 0.31992514431476593,
|
|
"rewards/accuracy_reward": 0.3616071566939354,
|
|
"step": 153
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 324.59822845458984,
|
|
"epoch": 0.7837150127226463,
|
|
"grad_norm": 0.5640062542576917,
|
|
"kl": 0.03253173828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.011,
|
|
"reward": 0.4464285969734192,
|
|
"reward_std": 0.27692829072475433,
|
|
"rewards/accuracy_reward": 0.4464285969734192,
|
|
"step": 154
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 302.19866943359375,
|
|
"epoch": 0.7888040712468194,
|
|
"grad_norm": 0.5839205172399542,
|
|
"kl": 0.03533935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0094,
|
|
"reward": 0.4598214477300644,
|
|
"reward_std": 0.2679207921028137,
|
|
"rewards/accuracy_reward": 0.4598214477300644,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 323.50670623779297,
|
|
"epoch": 0.7938931297709924,
|
|
"grad_norm": 0.6104848835648137,
|
|
"kl": 0.0283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0073,
|
|
"reward": 0.377232164144516,
|
|
"reward_std": 0.32067109644412994,
|
|
"rewards/accuracy_reward": 0.377232164144516,
|
|
"step": 156
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 313.3616180419922,
|
|
"epoch": 0.7989821882951654,
|
|
"grad_norm": 0.5720085283610616,
|
|
"kl": 0.029266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.4196428805589676,
|
|
"reward_std": 0.30029767006635666,
|
|
"rewards/accuracy_reward": 0.4196428805589676,
|
|
"step": 157
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 354.7410888671875,
|
|
"epoch": 0.8040712468193384,
|
|
"grad_norm": 0.521897463534276,
|
|
"kl": 0.024322509765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0084,
|
|
"reward": 0.3325893022119999,
|
|
"reward_std": 0.2593177370727062,
|
|
"rewards/accuracy_reward": 0.3325893022119999,
|
|
"step": 158
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.61832427978516,
|
|
"epoch": 0.8091603053435115,
|
|
"grad_norm": 0.45568937591045094,
|
|
"kl": 0.024078369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.3147321566939354,
|
|
"reward_std": 0.2266555279493332,
|
|
"rewards/accuracy_reward": 0.3147321566939354,
|
|
"step": 159
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.91519927978516,
|
|
"epoch": 0.8142493638676844,
|
|
"grad_norm": 0.5556974208639448,
|
|
"kl": 0.024993896484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.290178582072258,
|
|
"reward_std": 0.32750868797302246,
|
|
"rewards/accuracy_reward": 0.290178582072258,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.08260345458984,
|
|
"epoch": 0.8193384223918575,
|
|
"grad_norm": 0.47142602312686566,
|
|
"kl": 0.02667236328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0084,
|
|
"reward": 0.3392857387661934,
|
|
"reward_std": 0.2555834725499153,
|
|
"rewards/accuracy_reward": 0.3392857387661934,
|
|
"step": 161
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.03572845458984,
|
|
"epoch": 0.8244274809160306,
|
|
"grad_norm": 0.4937371352762309,
|
|
"kl": 0.025634765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0209,
|
|
"reward": 0.3549107350409031,
|
|
"reward_std": 0.26360073685646057,
|
|
"rewards/accuracy_reward": 0.3549107350409031,
|
|
"step": 162
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.8415298461914,
|
|
"epoch": 0.8295165394402035,
|
|
"grad_norm": 0.4831961543470371,
|
|
"kl": 0.02435302734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0242,
|
|
"reward": 0.4799107238650322,
|
|
"reward_std": 0.2922033295035362,
|
|
"rewards/accuracy_reward": 0.4799107238650322,
|
|
"step": 163
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 454.8080596923828,
|
|
"epoch": 0.8346055979643766,
|
|
"grad_norm": 0.4581637953094632,
|
|
"kl": 0.024139404296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0089,
|
|
"reward": 0.383928582072258,
|
|
"reward_std": 0.300465926527977,
|
|
"rewards/accuracy_reward": 0.383928582072258,
|
|
"step": 164
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 444.7745666503906,
|
|
"epoch": 0.8396946564885496,
|
|
"grad_norm": 0.412366914780704,
|
|
"kl": 0.02703857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0046,
|
|
"reward": 0.3236607275903225,
|
|
"reward_std": 0.18380586430430412,
|
|
"rewards/accuracy_reward": 0.3236607275903225,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 433.7857360839844,
|
|
"epoch": 0.8447837150127226,
|
|
"grad_norm": 0.4332577326939765,
|
|
"kl": 0.02471923828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.4799107387661934,
|
|
"reward_std": 0.27713901177048683,
|
|
"rewards/accuracy_reward": 0.4799107387661934,
|
|
"step": 166
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.2768096923828,
|
|
"epoch": 0.8498727735368957,
|
|
"grad_norm": 0.4934250138706889,
|
|
"kl": 0.02606201171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0074,
|
|
"reward": 0.2879464365541935,
|
|
"reward_std": 0.2666035257279873,
|
|
"rewards/accuracy_reward": 0.2879464365541935,
|
|
"step": 167
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.50672149658203,
|
|
"epoch": 0.8549618320610687,
|
|
"grad_norm": 0.5135332913276037,
|
|
"kl": 0.027008056640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.4486607387661934,
|
|
"reward_std": 0.34256667643785477,
|
|
"rewards/accuracy_reward": 0.4486607387661934,
|
|
"step": 168
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 445.5960006713867,
|
|
"epoch": 0.8600508905852418,
|
|
"grad_norm": 0.4428290082641468,
|
|
"kl": 0.025360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0083,
|
|
"reward": 0.310267873108387,
|
|
"reward_std": 0.28904811665415764,
|
|
"rewards/accuracy_reward": 0.310267873108387,
|
|
"step": 169
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.2500228881836,
|
|
"epoch": 0.8651399491094147,
|
|
"grad_norm": 0.5258875996753243,
|
|
"kl": 0.0260009765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.341517873108387,
|
|
"reward_std": 0.32597115635871887,
|
|
"rewards/accuracy_reward": 0.341517873108387,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 440.9598388671875,
|
|
"epoch": 0.8702290076335878,
|
|
"grad_norm": 0.45207686350373955,
|
|
"kl": 0.02337646484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0032,
|
|
"reward": 0.4107143022119999,
|
|
"reward_std": 0.2683451995253563,
|
|
"rewards/accuracy_reward": 0.4107143022119999,
|
|
"step": 171
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.3638610839844,
|
|
"epoch": 0.8753180661577609,
|
|
"grad_norm": 0.42381339196292467,
|
|
"kl": 0.02447509765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.3839285895228386,
|
|
"reward_std": 0.2377573773264885,
|
|
"rewards/accuracy_reward": 0.3839285895228386,
|
|
"step": 172
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 439.27903747558594,
|
|
"epoch": 0.8804071246819338,
|
|
"grad_norm": 0.35561623304524603,
|
|
"kl": 0.023101806640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0098,
|
|
"reward": 0.3191964477300644,
|
|
"reward_std": 0.1617190968245268,
|
|
"rewards/accuracy_reward": 0.3191964477300644,
|
|
"step": 173
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.92859649658203,
|
|
"epoch": 0.8854961832061069,
|
|
"grad_norm": 0.3927903996471991,
|
|
"kl": 0.025299072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0118,
|
|
"reward": 0.439732164144516,
|
|
"reward_std": 0.20010346360504627,
|
|
"rewards/accuracy_reward": 0.439732164144516,
|
|
"step": 174
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.2879638671875,
|
|
"epoch": 0.8905852417302799,
|
|
"grad_norm": 0.430060342574078,
|
|
"kl": 0.024322509765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0059,
|
|
"reward": 0.3616071715950966,
|
|
"reward_std": 0.22979383915662766,
|
|
"rewards/accuracy_reward": 0.3616071715950966,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.3817138671875,
|
|
"epoch": 0.8956743002544529,
|
|
"grad_norm": 0.4500570691242079,
|
|
"kl": 0.026123046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0077,
|
|
"reward": 0.4642857313156128,
|
|
"reward_std": 0.25467485189437866,
|
|
"rewards/accuracy_reward": 0.4642857313156128,
|
|
"step": 176
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.4330520629883,
|
|
"epoch": 0.9007633587786259,
|
|
"grad_norm": 0.4689466745754758,
|
|
"kl": 0.024078369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.3973214477300644,
|
|
"reward_std": 0.30610737577080727,
|
|
"rewards/accuracy_reward": 0.3973214477300644,
|
|
"step": 177
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 438.4174270629883,
|
|
"epoch": 0.905852417302799,
|
|
"grad_norm": 0.4388691039751368,
|
|
"kl": 0.02703857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.424107164144516,
|
|
"reward_std": 0.2609916552901268,
|
|
"rewards/accuracy_reward": 0.424107164144516,
|
|
"step": 178
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.7299270629883,
|
|
"epoch": 0.910941475826972,
|
|
"grad_norm": 0.4180145109965147,
|
|
"kl": 0.02935791015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0045,
|
|
"reward": 0.3772321566939354,
|
|
"reward_std": 0.25678665190935135,
|
|
"rewards/accuracy_reward": 0.3772321566939354,
|
|
"step": 179
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 433.75225830078125,
|
|
"epoch": 0.916030534351145,
|
|
"grad_norm": 0.45666481451744956,
|
|
"kl": 0.0283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0062,
|
|
"reward": 0.23660715110599995,
|
|
"reward_std": 0.2571524307131767,
|
|
"rewards/accuracy_reward": 0.23660715110599995,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.3705596923828,
|
|
"epoch": 0.9211195928753181,
|
|
"grad_norm": 0.4965736901194939,
|
|
"kl": 0.02862548828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0117,
|
|
"reward": 0.3928571492433548,
|
|
"reward_std": 0.32339557260274887,
|
|
"rewards/accuracy_reward": 0.3928571492433548,
|
|
"step": 181
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.8861770629883,
|
|
"epoch": 0.926208651399491,
|
|
"grad_norm": 0.4738101599027936,
|
|
"kl": 0.0289306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.003,
|
|
"reward": 0.3392857238650322,
|
|
"reward_std": 0.27634909749031067,
|
|
"rewards/accuracy_reward": 0.3392857238650322,
|
|
"step": 182
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.6607360839844,
|
|
"epoch": 0.9312977099236641,
|
|
"grad_norm": 0.44741536703750234,
|
|
"kl": 0.028106689453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.424107164144516,
|
|
"reward_std": 0.2569168284535408,
|
|
"rewards/accuracy_reward": 0.424107164144516,
|
|
"step": 183
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.1294860839844,
|
|
"epoch": 0.9363867684478372,
|
|
"grad_norm": 0.4993649877556307,
|
|
"kl": 0.02801513671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0058,
|
|
"reward": 0.3638393059372902,
|
|
"reward_std": 0.27742746472358704,
|
|
"rewards/accuracy_reward": 0.3638393059372902,
|
|
"step": 184
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.8080596923828,
|
|
"epoch": 0.9414758269720102,
|
|
"grad_norm": 0.4702756887679677,
|
|
"kl": 0.029876708984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.009,
|
|
"reward": 0.341517873108387,
|
|
"reward_std": 0.31923749297857285,
|
|
"rewards/accuracy_reward": 0.341517873108387,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.7098388671875,
|
|
"epoch": 0.9465648854961832,
|
|
"grad_norm": 0.5106143671517,
|
|
"kl": 0.03033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.3549107350409031,
|
|
"reward_std": 0.28500746190547943,
|
|
"rewards/accuracy_reward": 0.3549107350409031,
|
|
"step": 186
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.80359649658203,
|
|
"epoch": 0.9516539440203562,
|
|
"grad_norm": 0.5394363221726483,
|
|
"kl": 0.03045654296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.4754464477300644,
|
|
"reward_std": 0.3313308395445347,
|
|
"rewards/accuracy_reward": 0.4754464477300644,
|
|
"step": 187
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.23885345458984,
|
|
"epoch": 0.9567430025445293,
|
|
"grad_norm": 0.48598201018594284,
|
|
"kl": 0.029998779296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0042,
|
|
"reward": 0.3125000111758709,
|
|
"reward_std": 0.2686355970799923,
|
|
"rewards/accuracy_reward": 0.3125000111758709,
|
|
"step": 188
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 363.5580520629883,
|
|
"epoch": 0.9618320610687023,
|
|
"grad_norm": 0.577078574347706,
|
|
"kl": 0.02972412109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0117,
|
|
"reward": 0.3058035895228386,
|
|
"reward_std": 0.3343946039676666,
|
|
"rewards/accuracy_reward": 0.3058035895228386,
|
|
"step": 189
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.7835006713867,
|
|
"epoch": 0.9669211195928753,
|
|
"grad_norm": 0.5538549733946959,
|
|
"kl": 0.03118896484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0086,
|
|
"reward": 0.3705357313156128,
|
|
"reward_std": 0.31720490753650665,
|
|
"rewards/accuracy_reward": 0.3705357313156128,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.1116180419922,
|
|
"epoch": 0.9720101781170484,
|
|
"grad_norm": 0.4602172125795691,
|
|
"kl": 0.03082275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0045,
|
|
"reward": 0.4419643133878708,
|
|
"reward_std": 0.2720428891479969,
|
|
"rewards/accuracy_reward": 0.4419643133878708,
|
|
"step": 191
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 441.6964416503906,
|
|
"epoch": 0.9770992366412213,
|
|
"grad_norm": 0.44511046078232314,
|
|
"kl": 0.0264892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0051,
|
|
"reward": 0.4129464440047741,
|
|
"reward_std": 0.204301031306386,
|
|
"rewards/accuracy_reward": 0.4129464440047741,
|
|
"step": 192
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.27233123779297,
|
|
"epoch": 0.9821882951653944,
|
|
"grad_norm": 0.6368128639064,
|
|
"kl": 0.036834716796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.424107164144516,
|
|
"reward_std": 0.3008757568895817,
|
|
"rewards/accuracy_reward": 0.424107164144516,
|
|
"step": 193
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.9107360839844,
|
|
"epoch": 0.9872773536895675,
|
|
"grad_norm": 0.42714163361676444,
|
|
"kl": 0.029388427734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.37053573317825794,
|
|
"reward_std": 0.19569924473762512,
|
|
"rewards/accuracy_reward": 0.37053573317825794,
|
|
"step": 194
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.43751525878906,
|
|
"epoch": 0.9923664122137404,
|
|
"grad_norm": 0.5126159949792717,
|
|
"kl": 0.028045654296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0046,
|
|
"reward": 0.4933035932481289,
|
|
"reward_std": 0.3095056377351284,
|
|
"rewards/accuracy_reward": 0.4933035932481289,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 354.4423141479492,
|
|
"epoch": 0.9974554707379135,
|
|
"grad_norm": 0.477609157386284,
|
|
"kl": 0.029541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.012,
|
|
"reward": 0.4218750223517418,
|
|
"reward_std": 0.22727755084633827,
|
|
"rewards/accuracy_reward": 0.4218750223517418,
|
|
"step": 196
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 434.3928756713867,
|
|
"epoch": 1.005089058524173,
|
|
"grad_norm": 0.45629019648922187,
|
|
"kl": 0.028839111328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0129,
|
|
"reward": 0.4151785895228386,
|
|
"reward_std": 0.20347346365451813,
|
|
"rewards/accuracy_reward": 0.4151785895228386,
|
|
"step": 197
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.9397506713867,
|
|
"epoch": 1.0101781170483461,
|
|
"grad_norm": 0.4852725436095131,
|
|
"kl": 0.03228759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.5714285969734192,
|
|
"reward_std": 0.24667690321803093,
|
|
"rewards/accuracy_reward": 0.5714285969734192,
|
|
"step": 198
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.6919860839844,
|
|
"epoch": 1.015267175572519,
|
|
"grad_norm": 0.46684070027513186,
|
|
"kl": 0.03106689453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0063,
|
|
"reward": 0.3973214477300644,
|
|
"reward_std": 0.24830005317926407,
|
|
"rewards/accuracy_reward": 0.3973214477300644,
|
|
"step": 199
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.1518020629883,
|
|
"epoch": 1.020356234096692,
|
|
"grad_norm": 0.47365885205499736,
|
|
"kl": 0.0308837890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.3928571715950966,
|
|
"reward_std": 0.2442369982600212,
|
|
"rewards/accuracy_reward": 0.3928571715950966,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.6294860839844,
|
|
"epoch": 1.025445292620865,
|
|
"grad_norm": 0.5216914587175241,
|
|
"kl": 0.03045654296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0095,
|
|
"reward": 0.5781250223517418,
|
|
"reward_std": 0.24771291390061378,
|
|
"rewards/accuracy_reward": 0.5781250223517418,
|
|
"step": 201
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.7143020629883,
|
|
"epoch": 1.0305343511450382,
|
|
"grad_norm": 0.49886249418845297,
|
|
"kl": 0.034149169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0167,
|
|
"reward": 0.466517873108387,
|
|
"reward_std": 0.26621802896261215,
|
|
"rewards/accuracy_reward": 0.466517873108387,
|
|
"step": 202
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.1049270629883,
|
|
"epoch": 1.0356234096692112,
|
|
"grad_norm": 0.49883207537024243,
|
|
"kl": 0.035430908203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.4888393059372902,
|
|
"reward_std": 0.29329938814044,
|
|
"rewards/accuracy_reward": 0.4888393059372902,
|
|
"step": 203
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.4710006713867,
|
|
"epoch": 1.0407124681933841,
|
|
"grad_norm": 0.4743305957167603,
|
|
"kl": 0.0341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0058,
|
|
"reward": 0.6093750223517418,
|
|
"reward_std": 0.27687905728816986,
|
|
"rewards/accuracy_reward": 0.6093750223517418,
|
|
"step": 204
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.2366256713867,
|
|
"epoch": 1.0458015267175573,
|
|
"grad_norm": 0.41185213945826243,
|
|
"kl": 0.031585693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.005,
|
|
"reward": 0.667410746216774,
|
|
"reward_std": 0.1963009387254715,
|
|
"rewards/accuracy_reward": 0.667410746216774,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.06251525878906,
|
|
"epoch": 1.0508905852417303,
|
|
"grad_norm": 0.4319517303591335,
|
|
"kl": 0.029632568359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0061,
|
|
"reward": 0.542410746216774,
|
|
"reward_std": 0.2033972479403019,
|
|
"rewards/accuracy_reward": 0.542410746216774,
|
|
"step": 206
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.9486846923828,
|
|
"epoch": 1.0559796437659033,
|
|
"grad_norm": 0.4428543206563001,
|
|
"kl": 0.030517578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0139,
|
|
"reward": 0.5781250149011612,
|
|
"reward_std": 0.21149570867419243,
|
|
"rewards/accuracy_reward": 0.5781250149011612,
|
|
"step": 207
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.04466247558594,
|
|
"epoch": 1.0610687022900764,
|
|
"grad_norm": 0.4095348359436766,
|
|
"kl": 0.03240966796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0137,
|
|
"reward": 0.448660746216774,
|
|
"reward_std": 0.18329654354602098,
|
|
"rewards/accuracy_reward": 0.448660746216774,
|
|
"step": 208
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.1138610839844,
|
|
"epoch": 1.0661577608142494,
|
|
"grad_norm": 0.4796397496935904,
|
|
"kl": 0.030426025390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.004,
|
|
"reward": 0.4732143059372902,
|
|
"reward_std": 0.24544759839773178,
|
|
"rewards/accuracy_reward": 0.4732143059372902,
|
|
"step": 209
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.3973388671875,
|
|
"epoch": 1.0712468193384224,
|
|
"grad_norm": 0.9274552338240573,
|
|
"kl": 0.039306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0106,
|
|
"reward": 0.5982143059372902,
|
|
"reward_std": 0.2766333222389221,
|
|
"rewards/accuracy_reward": 0.5982143059372902,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.5335006713867,
|
|
"epoch": 1.0763358778625953,
|
|
"grad_norm": 0.4468713413539593,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0031,
|
|
"reward": 0.562500037252903,
|
|
"reward_std": 0.19667476043105125,
|
|
"rewards/accuracy_reward": 0.562500037252903,
|
|
"step": 211
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.85491943359375,
|
|
"epoch": 1.0814249363867685,
|
|
"grad_norm": 0.4865455605392445,
|
|
"kl": 0.031768798828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.600446455180645,
|
|
"reward_std": 0.27646007761359215,
|
|
"rewards/accuracy_reward": 0.600446455180645,
|
|
"step": 212
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 368.4665298461914,
|
|
"epoch": 1.0865139949109415,
|
|
"grad_norm": 0.4958354129620625,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.5334821715950966,
|
|
"reward_std": 0.26529859378933907,
|
|
"rewards/accuracy_reward": 0.5334821715950966,
|
|
"step": 213
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.0424270629883,
|
|
"epoch": 1.0916030534351144,
|
|
"grad_norm": 0.47123769690556155,
|
|
"kl": 0.03216552734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0105,
|
|
"reward": 0.4866071715950966,
|
|
"reward_std": 0.22155790403485298,
|
|
"rewards/accuracy_reward": 0.4866071715950966,
|
|
"step": 214
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.37500762939453,
|
|
"epoch": 1.0966921119592876,
|
|
"grad_norm": 0.326585929171639,
|
|
"kl": 0.03814697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.642857164144516,
|
|
"reward_std": 0.11197268962860107,
|
|
"rewards/accuracy_reward": 0.642857164144516,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.30135345458984,
|
|
"epoch": 1.1017811704834606,
|
|
"grad_norm": 0.44026320719542633,
|
|
"kl": 0.03521728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0104,
|
|
"reward": 0.464285746216774,
|
|
"reward_std": 0.28557220473885536,
|
|
"rewards/accuracy_reward": 0.464285746216774,
|
|
"step": 216
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.1540298461914,
|
|
"epoch": 1.1068702290076335,
|
|
"grad_norm": 0.446293170783821,
|
|
"kl": 0.03741455078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.4910714477300644,
|
|
"reward_std": 0.17842019442468882,
|
|
"rewards/accuracy_reward": 0.4910714477300644,
|
|
"step": 217
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.4352798461914,
|
|
"epoch": 1.1119592875318065,
|
|
"grad_norm": 0.3950797046488124,
|
|
"kl": 0.036041259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0117,
|
|
"reward": 0.5491071715950966,
|
|
"reward_std": 0.18994246423244476,
|
|
"rewards/accuracy_reward": 0.5491071715950966,
|
|
"step": 218
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.73663330078125,
|
|
"epoch": 1.1170483460559797,
|
|
"grad_norm": 0.4317642165996371,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0164,
|
|
"reward": 0.5000000149011612,
|
|
"reward_std": 0.18737347051501274,
|
|
"rewards/accuracy_reward": 0.5000000149011612,
|
|
"step": 219
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.8727798461914,
|
|
"epoch": 1.1221374045801527,
|
|
"grad_norm": 0.48090345298021026,
|
|
"kl": 0.037322998046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.5915178805589676,
|
|
"reward_std": 0.1756830532103777,
|
|
"rewards/accuracy_reward": 0.5915178805589676,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.70314025878906,
|
|
"epoch": 1.1272264631043256,
|
|
"grad_norm": 0.3540583574823124,
|
|
"kl": 0.0330810546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0116,
|
|
"reward": 0.5803571864962578,
|
|
"reward_std": 0.14323088619858027,
|
|
"rewards/accuracy_reward": 0.5803571864962578,
|
|
"step": 221
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.71876525878906,
|
|
"epoch": 1.1323155216284988,
|
|
"grad_norm": 0.4245122051677136,
|
|
"kl": 0.033111572265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0217,
|
|
"reward": 0.502232164144516,
|
|
"reward_std": 0.2278486266732216,
|
|
"rewards/accuracy_reward": 0.502232164144516,
|
|
"step": 222
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.1272506713867,
|
|
"epoch": 1.1374045801526718,
|
|
"grad_norm": 0.3779360649176341,
|
|
"kl": 0.03619384765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.5312500298023224,
|
|
"reward_std": 0.21440864354372025,
|
|
"rewards/accuracy_reward": 0.5312500298023224,
|
|
"step": 223
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.6942138671875,
|
|
"epoch": 1.1424936386768447,
|
|
"grad_norm": 0.3296850361604471,
|
|
"kl": 0.032379150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0079,
|
|
"reward": 0.6584821790456772,
|
|
"reward_std": 0.11233820579946041,
|
|
"rewards/accuracy_reward": 0.6584821790456772,
|
|
"step": 224
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 435.2812728881836,
|
|
"epoch": 1.1475826972010177,
|
|
"grad_norm": 0.44279309621022384,
|
|
"kl": 0.03387451171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0073,
|
|
"reward": 0.5000000074505806,
|
|
"reward_std": 0.23925619572401047,
|
|
"rewards/accuracy_reward": 0.5000000074505806,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 445.60269927978516,
|
|
"epoch": 1.1526717557251909,
|
|
"grad_norm": 0.3810375840986681,
|
|
"kl": 0.03192138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.435267873108387,
|
|
"reward_std": 0.18524267710745335,
|
|
"rewards/accuracy_reward": 0.435267873108387,
|
|
"step": 226
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.0446548461914,
|
|
"epoch": 1.1577608142493638,
|
|
"grad_norm": 0.45017396993067865,
|
|
"kl": 0.035614013671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0098,
|
|
"reward": 0.491071455180645,
|
|
"reward_std": 0.2348989974707365,
|
|
"rewards/accuracy_reward": 0.491071455180645,
|
|
"step": 227
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.5513610839844,
|
|
"epoch": 1.162849872773537,
|
|
"grad_norm": 0.3586093802630662,
|
|
"kl": 0.031402587890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.014,
|
|
"reward": 0.5848214402794838,
|
|
"reward_std": 0.2014264166355133,
|
|
"rewards/accuracy_reward": 0.5848214402794838,
|
|
"step": 228
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 450.0111846923828,
|
|
"epoch": 1.16793893129771,
|
|
"grad_norm": 0.3888403716091177,
|
|
"kl": 0.029754638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0123,
|
|
"reward": 0.6383928954601288,
|
|
"reward_std": 0.16469183936715126,
|
|
"rewards/accuracy_reward": 0.6383928954601288,
|
|
"step": 229
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.9308166503906,
|
|
"epoch": 1.173027989821883,
|
|
"grad_norm": 0.3757730072446275,
|
|
"kl": 0.035491943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.006,
|
|
"reward": 0.6205357387661934,
|
|
"reward_std": 0.1877366341650486,
|
|
"rewards/accuracy_reward": 0.6205357387661934,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.8995590209961,
|
|
"epoch": 1.178117048346056,
|
|
"grad_norm": 0.36476781267980535,
|
|
"kl": 0.030426025390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0085,
|
|
"reward": 0.705357164144516,
|
|
"reward_std": 0.16365888342261314,
|
|
"rewards/accuracy_reward": 0.705357164144516,
|
|
"step": 231
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.4085006713867,
|
|
"epoch": 1.183206106870229,
|
|
"grad_norm": 0.42247243573537296,
|
|
"kl": 0.029266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.5089285895228386,
|
|
"reward_std": 0.2098158746957779,
|
|
"rewards/accuracy_reward": 0.5089285895228386,
|
|
"step": 232
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.1183166503906,
|
|
"epoch": 1.188295165394402,
|
|
"grad_norm": 0.3372301931726151,
|
|
"kl": 0.029815673828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0018,
|
|
"reward": 0.4933035969734192,
|
|
"reward_std": 0.19880928099155426,
|
|
"rewards/accuracy_reward": 0.4933035969734192,
|
|
"step": 233
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.52457427978516,
|
|
"epoch": 1.193384223918575,
|
|
"grad_norm": 0.3984313133748342,
|
|
"kl": 0.033966064453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0109,
|
|
"reward": 0.5223214626312256,
|
|
"reward_std": 0.17286018654704094,
|
|
"rewards/accuracy_reward": 0.5223214626312256,
|
|
"step": 234
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.57591247558594,
|
|
"epoch": 1.1984732824427482,
|
|
"grad_norm": 0.4800689228399824,
|
|
"kl": 0.0406494140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.5223214402794838,
|
|
"reward_std": 0.2609461396932602,
|
|
"rewards/accuracy_reward": 0.5223214402794838,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.08483123779297,
|
|
"epoch": 1.2035623409669212,
|
|
"grad_norm": 0.4084541086945262,
|
|
"kl": 0.0341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0039,
|
|
"reward": 0.5424107387661934,
|
|
"reward_std": 0.17931898683309555,
|
|
"rewards/accuracy_reward": 0.5424107387661934,
|
|
"step": 236
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.9665298461914,
|
|
"epoch": 1.2086513994910941,
|
|
"grad_norm": 0.4019034905125619,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0024,
|
|
"reward": 0.647321455180645,
|
|
"reward_std": 0.18874311447143555,
|
|
"rewards/accuracy_reward": 0.647321455180645,
|
|
"step": 237
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.82814025878906,
|
|
"epoch": 1.213740458015267,
|
|
"grad_norm": 0.48533269021127756,
|
|
"kl": 0.0325927734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0053,
|
|
"reward": 0.5468750298023224,
|
|
"reward_std": 0.254442248493433,
|
|
"rewards/accuracy_reward": 0.5468750298023224,
|
|
"step": 238
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.8214569091797,
|
|
"epoch": 1.2188295165394403,
|
|
"grad_norm": 0.3491733026298112,
|
|
"kl": 0.029266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.517857164144516,
|
|
"reward_std": 0.1553249955177307,
|
|
"rewards/accuracy_reward": 0.517857164144516,
|
|
"step": 239
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.12501525878906,
|
|
"epoch": 1.2239185750636132,
|
|
"grad_norm": 0.3738114254251958,
|
|
"kl": 0.030853271484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0029,
|
|
"reward": 0.5200893208384514,
|
|
"reward_std": 0.22691549360752106,
|
|
"rewards/accuracy_reward": 0.5200893208384514,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 369.85045623779297,
|
|
"epoch": 1.2290076335877862,
|
|
"grad_norm": 0.4317542008409837,
|
|
"kl": 0.034698486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.4241071566939354,
|
|
"reward_std": 0.19226149097085,
|
|
"rewards/accuracy_reward": 0.4241071566939354,
|
|
"step": 241
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 378.1227798461914,
|
|
"epoch": 1.2340966921119594,
|
|
"grad_norm": 0.39238353022547695,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.008,
|
|
"reward": 0.569196455180645,
|
|
"reward_std": 0.17385854944586754,
|
|
"rewards/accuracy_reward": 0.569196455180645,
|
|
"step": 242
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.10716247558594,
|
|
"epoch": 1.2391857506361323,
|
|
"grad_norm": 0.43920936279747014,
|
|
"kl": 0.029815673828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0136,
|
|
"reward": 0.3593750223517418,
|
|
"reward_std": 0.24526547640562057,
|
|
"rewards/accuracy_reward": 0.3593750223517418,
|
|
"step": 243
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.65626525878906,
|
|
"epoch": 1.2442748091603053,
|
|
"grad_norm": 0.41848082095968575,
|
|
"kl": 0.032684326171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.5825893133878708,
|
|
"reward_std": 0.21210570260882378,
|
|
"rewards/accuracy_reward": 0.5825893133878708,
|
|
"step": 244
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.6562728881836,
|
|
"epoch": 1.2493638676844783,
|
|
"grad_norm": 0.5011391731279294,
|
|
"kl": 0.03106689453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0091,
|
|
"reward": 0.5781250149011612,
|
|
"reward_std": 0.26321787759661674,
|
|
"rewards/accuracy_reward": 0.5781250149011612,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.5982360839844,
|
|
"epoch": 1.2544529262086515,
|
|
"grad_norm": 0.3997580217150404,
|
|
"kl": 0.035614013671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.4687500074505806,
|
|
"reward_std": 0.15454928297549486,
|
|
"rewards/accuracy_reward": 0.4687500074505806,
|
|
"step": 246
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.3995666503906,
|
|
"epoch": 1.2595419847328244,
|
|
"grad_norm": 0.487790323685233,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0148,
|
|
"reward": 0.511160746216774,
|
|
"reward_std": 0.24315107986330986,
|
|
"rewards/accuracy_reward": 0.511160746216774,
|
|
"step": 247
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 438.16297149658203,
|
|
"epoch": 1.2646310432569974,
|
|
"grad_norm": 0.38761795177162306,
|
|
"kl": 0.029388427734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0152,
|
|
"reward": 0.611607164144516,
|
|
"reward_std": 0.21232472360134125,
|
|
"rewards/accuracy_reward": 0.611607164144516,
|
|
"step": 248
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.4442138671875,
|
|
"epoch": 1.2697201017811706,
|
|
"grad_norm": 0.49600390157546337,
|
|
"kl": 0.031982421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0028,
|
|
"reward": 0.3750000149011612,
|
|
"reward_std": 0.2584892399609089,
|
|
"rewards/accuracy_reward": 0.3750000149011612,
|
|
"step": 249
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.50001525878906,
|
|
"epoch": 1.2748091603053435,
|
|
"grad_norm": 0.42773230362935344,
|
|
"kl": 0.036529541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0068,
|
|
"reward": 0.4375000149011612,
|
|
"reward_std": 0.2202574610710144,
|
|
"rewards/accuracy_reward": 0.4375000149011612,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 352.59153747558594,
|
|
"epoch": 1.2798982188295165,
|
|
"grad_norm": 0.5330881236514794,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0071,
|
|
"reward": 0.618303582072258,
|
|
"reward_std": 0.27262888848781586,
|
|
"rewards/accuracy_reward": 0.618303582072258,
|
|
"step": 251
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.53126525878906,
|
|
"epoch": 1.2849872773536894,
|
|
"grad_norm": 0.48866138610644344,
|
|
"kl": 0.031524658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0105,
|
|
"reward": 0.5848214626312256,
|
|
"reward_std": 0.2368122972548008,
|
|
"rewards/accuracy_reward": 0.5848214626312256,
|
|
"step": 252
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.4620666503906,
|
|
"epoch": 1.2900763358778626,
|
|
"grad_norm": 0.3896532735098542,
|
|
"kl": 0.03399658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0163,
|
|
"reward": 0.7366071790456772,
|
|
"reward_std": 0.18220902606844902,
|
|
"rewards/accuracy_reward": 0.7366071790456772,
|
|
"step": 253
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.18082427978516,
|
|
"epoch": 1.2951653944020356,
|
|
"grad_norm": 0.486253566989,
|
|
"kl": 0.03790283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0016,
|
|
"reward": 0.4933036044239998,
|
|
"reward_std": 0.2729470394551754,
|
|
"rewards/accuracy_reward": 0.4933036044239998,
|
|
"step": 254
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.66966247558594,
|
|
"epoch": 1.3002544529262088,
|
|
"grad_norm": 0.45345429544975063,
|
|
"kl": 0.0303955078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.4531250149011612,
|
|
"reward_std": 0.2670621797442436,
|
|
"rewards/accuracy_reward": 0.4531250149011612,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 374.8058319091797,
|
|
"epoch": 1.3053435114503817,
|
|
"grad_norm": 0.438097016293131,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.6450893133878708,
|
|
"reward_std": 0.16849614307284355,
|
|
"rewards/accuracy_reward": 0.6450893133878708,
|
|
"step": 256
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.25447845458984,
|
|
"epoch": 1.3104325699745547,
|
|
"grad_norm": 0.359204988846788,
|
|
"kl": 0.035003662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0191,
|
|
"reward": 0.651785746216774,
|
|
"reward_std": 0.16984091699123383,
|
|
"rewards/accuracy_reward": 0.651785746216774,
|
|
"step": 257
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.47322845458984,
|
|
"epoch": 1.3155216284987277,
|
|
"grad_norm": 0.46896489998912644,
|
|
"kl": 0.034515380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0054,
|
|
"reward": 0.5558035969734192,
|
|
"reward_std": 0.2598639354109764,
|
|
"rewards/accuracy_reward": 0.5558035969734192,
|
|
"step": 258
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.02903747558594,
|
|
"epoch": 1.3206106870229006,
|
|
"grad_norm": 0.3417142397006384,
|
|
"kl": 0.031768798828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.651785746216774,
|
|
"reward_std": 0.15594056900590658,
|
|
"rewards/accuracy_reward": 0.651785746216774,
|
|
"step": 259
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.66072845458984,
|
|
"epoch": 1.3256997455470738,
|
|
"grad_norm": 0.4718016390791712,
|
|
"kl": 0.03558349609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.5312500223517418,
|
|
"reward_std": 0.22380899265408516,
|
|
"rewards/accuracy_reward": 0.5312500223517418,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.72547149658203,
|
|
"epoch": 1.3307888040712468,
|
|
"grad_norm": 0.3798765837581581,
|
|
"kl": 0.0338134765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.5044643059372902,
|
|
"reward_std": 0.15109536796808243,
|
|
"rewards/accuracy_reward": 0.5044643059372902,
|
|
"step": 261
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.7143020629883,
|
|
"epoch": 1.33587786259542,
|
|
"grad_norm": 0.48318812132563077,
|
|
"kl": 0.035064697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.5870536044239998,
|
|
"reward_std": 0.2547951564192772,
|
|
"rewards/accuracy_reward": 0.5870536044239998,
|
|
"step": 262
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.0022506713867,
|
|
"epoch": 1.340966921119593,
|
|
"grad_norm": 0.39300607799916687,
|
|
"kl": 0.031982421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0131,
|
|
"reward": 0.5825893133878708,
|
|
"reward_std": 0.21780962124466896,
|
|
"rewards/accuracy_reward": 0.5825893133878708,
|
|
"step": 263
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.43528747558594,
|
|
"epoch": 1.3460559796437659,
|
|
"grad_norm": 0.490922251385516,
|
|
"kl": 0.0289306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.5089286044239998,
|
|
"reward_std": 0.2536001205444336,
|
|
"rewards/accuracy_reward": 0.5089286044239998,
|
|
"step": 264
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.38394927978516,
|
|
"epoch": 1.3511450381679388,
|
|
"grad_norm": 0.3978566875964569,
|
|
"kl": 0.03375244140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.6651786118745804,
|
|
"reward_std": 0.17543086595833302,
|
|
"rewards/accuracy_reward": 0.6651786118745804,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.9933166503906,
|
|
"epoch": 1.356234096692112,
|
|
"grad_norm": 0.4691451616822436,
|
|
"kl": 0.033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.5133928805589676,
|
|
"reward_std": 0.2103098053485155,
|
|
"rewards/accuracy_reward": 0.5133928805589676,
|
|
"step": 266
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 440.1361846923828,
|
|
"epoch": 1.361323155216285,
|
|
"grad_norm": 0.371972857118148,
|
|
"kl": 0.029083251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0142,
|
|
"reward": 0.6718750298023224,
|
|
"reward_std": 0.17460346966981888,
|
|
"rewards/accuracy_reward": 0.6718750298023224,
|
|
"step": 267
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.89734649658203,
|
|
"epoch": 1.366412213740458,
|
|
"grad_norm": 0.5071890470760458,
|
|
"kl": 0.031951904296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0029,
|
|
"reward": 0.593750037252903,
|
|
"reward_std": 0.30577272549271584,
|
|
"rewards/accuracy_reward": 0.593750037252903,
|
|
"step": 268
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.30582427978516,
|
|
"epoch": 1.3715012722646311,
|
|
"grad_norm": 0.3804687691660587,
|
|
"kl": 0.02972412109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.573660746216774,
|
|
"reward_std": 0.2071095071732998,
|
|
"rewards/accuracy_reward": 0.573660746216774,
|
|
"step": 269
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.37278747558594,
|
|
"epoch": 1.376590330788804,
|
|
"grad_norm": 0.4090213962618039,
|
|
"kl": 0.0294189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0117,
|
|
"reward": 0.5446428954601288,
|
|
"reward_std": 0.1941103432327509,
|
|
"rewards/accuracy_reward": 0.5446428954601288,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.1897506713867,
|
|
"epoch": 1.381679389312977,
|
|
"grad_norm": 0.48042069051097636,
|
|
"kl": 0.0303955078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0069,
|
|
"reward": 0.5401785895228386,
|
|
"reward_std": 0.27686627209186554,
|
|
"rewards/accuracy_reward": 0.5401785895228386,
|
|
"step": 271
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.62501525878906,
|
|
"epoch": 1.38676844783715,
|
|
"grad_norm": 0.45262121383079323,
|
|
"kl": 0.02978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0141,
|
|
"reward": 0.5111607313156128,
|
|
"reward_std": 0.22661678865551949,
|
|
"rewards/accuracy_reward": 0.5111607313156128,
|
|
"step": 272
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 451.17635345458984,
|
|
"epoch": 1.3918575063613232,
|
|
"grad_norm": 0.4179025179046707,
|
|
"kl": 0.030609130859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.5758928805589676,
|
|
"reward_std": 0.19894640147686005,
|
|
"rewards/accuracy_reward": 0.5758928805589676,
|
|
"step": 273
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.39510345458984,
|
|
"epoch": 1.3969465648854962,
|
|
"grad_norm": 0.4180587963753563,
|
|
"kl": 0.028350830078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.5803571790456772,
|
|
"reward_std": 0.21554413810372353,
|
|
"rewards/accuracy_reward": 0.5803571790456772,
|
|
"step": 274
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.1718978881836,
|
|
"epoch": 1.4020356234096691,
|
|
"grad_norm": 0.4589957571854596,
|
|
"kl": 0.032562255859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0039,
|
|
"reward": 0.5870536044239998,
|
|
"reward_std": 0.2515076510608196,
|
|
"rewards/accuracy_reward": 0.5870536044239998,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.12947845458984,
|
|
"epoch": 1.4071246819338423,
|
|
"grad_norm": 0.4488105381852421,
|
|
"kl": 0.029541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.5870536118745804,
|
|
"reward_std": 0.23234201595187187,
|
|
"rewards/accuracy_reward": 0.5870536118745804,
|
|
"step": 276
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.8147506713867,
|
|
"epoch": 1.4122137404580153,
|
|
"grad_norm": 0.49935408777879425,
|
|
"kl": 0.0369873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.5290178805589676,
|
|
"reward_std": 0.25455041602253914,
|
|
"rewards/accuracy_reward": 0.5290178805589676,
|
|
"step": 277
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.8549270629883,
|
|
"epoch": 1.4173027989821882,
|
|
"grad_norm": 0.45934001217436377,
|
|
"kl": 0.03057861328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0141,
|
|
"reward": 0.589285746216774,
|
|
"reward_std": 0.2629491277039051,
|
|
"rewards/accuracy_reward": 0.589285746216774,
|
|
"step": 278
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.23885345458984,
|
|
"epoch": 1.4223918575063612,
|
|
"grad_norm": 0.3803307352328088,
|
|
"kl": 0.030548095703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0155,
|
|
"reward": 0.5200893059372902,
|
|
"reward_std": 0.22422028705477715,
|
|
"rewards/accuracy_reward": 0.5200893059372902,
|
|
"step": 279
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 374.8772506713867,
|
|
"epoch": 1.4274809160305344,
|
|
"grad_norm": 0.40325345382090017,
|
|
"kl": 0.03460693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.6540178954601288,
|
|
"reward_std": 0.15160241909325123,
|
|
"rewards/accuracy_reward": 0.6540178954601288,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.95984649658203,
|
|
"epoch": 1.4325699745547074,
|
|
"grad_norm": 0.47073666845442763,
|
|
"kl": 0.032440185546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0036,
|
|
"reward": 0.3883928805589676,
|
|
"reward_std": 0.2049722783267498,
|
|
"rewards/accuracy_reward": 0.3883928805589676,
|
|
"step": 281
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.3616256713867,
|
|
"epoch": 1.4376590330788805,
|
|
"grad_norm": 0.4470282122223272,
|
|
"kl": 0.02899169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.012,
|
|
"reward": 0.649553582072258,
|
|
"reward_std": 0.1916458122432232,
|
|
"rewards/accuracy_reward": 0.649553582072258,
|
|
"step": 282
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.8638610839844,
|
|
"epoch": 1.4427480916030535,
|
|
"grad_norm": 0.39981447818567717,
|
|
"kl": 0.03497314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0076,
|
|
"reward": 0.6093750223517418,
|
|
"reward_std": 0.17711340822279453,
|
|
"rewards/accuracy_reward": 0.6093750223517418,
|
|
"step": 283
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.58038330078125,
|
|
"epoch": 1.4478371501272265,
|
|
"grad_norm": 0.3820981361187511,
|
|
"kl": 0.03021240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.6205357387661934,
|
|
"reward_std": 0.16524812299758196,
|
|
"rewards/accuracy_reward": 0.6205357387661934,
|
|
"step": 284
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.3236846923828,
|
|
"epoch": 1.4529262086513994,
|
|
"grad_norm": 0.37936483564842083,
|
|
"kl": 0.0384521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.5379464626312256,
|
|
"reward_std": 0.1923013273626566,
|
|
"rewards/accuracy_reward": 0.5379464626312256,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.3660888671875,
|
|
"epoch": 1.4580152671755724,
|
|
"grad_norm": 0.33842922303195444,
|
|
"kl": 0.03204345703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0058,
|
|
"reward": 0.4308035895228386,
|
|
"reward_std": 0.1204148530960083,
|
|
"rewards/accuracy_reward": 0.4308035895228386,
|
|
"step": 286
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.7768096923828,
|
|
"epoch": 1.4631043256997456,
|
|
"grad_norm": 0.43404556961180785,
|
|
"kl": 0.029266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.6406250298023224,
|
|
"reward_std": 0.2344888113439083,
|
|
"rewards/accuracy_reward": 0.6406250298023224,
|
|
"step": 287
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.2835006713867,
|
|
"epoch": 1.4681933842239185,
|
|
"grad_norm": 0.3645532284578946,
|
|
"kl": 0.029327392578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.5334821678698063,
|
|
"reward_std": 0.17899974528700113,
|
|
"rewards/accuracy_reward": 0.5334821678698063,
|
|
"step": 288
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.8750305175781,
|
|
"epoch": 1.4732824427480917,
|
|
"grad_norm": 0.29925358250241385,
|
|
"kl": 0.03021240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0059,
|
|
"reward": 0.6763393133878708,
|
|
"reward_std": 0.12090125773102045,
|
|
"rewards/accuracy_reward": 0.6763393133878708,
|
|
"step": 289
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.1741256713867,
|
|
"epoch": 1.4783715012722647,
|
|
"grad_norm": 0.41734257737695496,
|
|
"kl": 0.03192138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.7410714477300644,
|
|
"reward_std": 0.17781273648142815,
|
|
"rewards/accuracy_reward": 0.7410714477300644,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.84376525878906,
|
|
"epoch": 1.4834605597964376,
|
|
"grad_norm": 0.4388068708751415,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0123,
|
|
"reward": 0.6450893208384514,
|
|
"reward_std": 0.2453245185315609,
|
|
"rewards/accuracy_reward": 0.6450893208384514,
|
|
"step": 291
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.1785888671875,
|
|
"epoch": 1.4885496183206106,
|
|
"grad_norm": 0.3374416338879907,
|
|
"kl": 0.027130126953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0131,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.15548420883715153,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 292
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.5960006713867,
|
|
"epoch": 1.4936386768447838,
|
|
"grad_norm": 0.3458868433278366,
|
|
"kl": 0.031585693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.5647321715950966,
|
|
"reward_std": 0.1358319204300642,
|
|
"rewards/accuracy_reward": 0.5647321715950966,
|
|
"step": 293
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.8526916503906,
|
|
"epoch": 1.4987277353689568,
|
|
"grad_norm": 0.3211074271463654,
|
|
"kl": 0.0347900390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.6562500447034836,
|
|
"reward_std": 0.13926138915121555,
|
|
"rewards/accuracy_reward": 0.6562500447034836,
|
|
"step": 294
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.2120666503906,
|
|
"epoch": 1.5038167938931297,
|
|
"grad_norm": 0.38219937811282745,
|
|
"kl": 0.0341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0079,
|
|
"reward": 0.6674107387661934,
|
|
"reward_std": 0.20987043622881174,
|
|
"rewards/accuracy_reward": 0.6674107387661934,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.3593978881836,
|
|
"epoch": 1.508905852417303,
|
|
"grad_norm": 0.43162745395979,
|
|
"kl": 0.0322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.5513393059372902,
|
|
"reward_std": 0.2469811663031578,
|
|
"rewards/accuracy_reward": 0.5513393059372902,
|
|
"step": 296
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.5714416503906,
|
|
"epoch": 1.5139949109414759,
|
|
"grad_norm": 0.4450368481965956,
|
|
"kl": 0.03717041015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.6250000298023224,
|
|
"reward_std": 0.2539636231958866,
|
|
"rewards/accuracy_reward": 0.6250000298023224,
|
|
"step": 297
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.12501525878906,
|
|
"epoch": 1.5190839694656488,
|
|
"grad_norm": 0.3218231617500364,
|
|
"kl": 0.033233642578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0039,
|
|
"reward": 0.4888393059372902,
|
|
"reward_std": 0.14116082433611155,
|
|
"rewards/accuracy_reward": 0.4888393059372902,
|
|
"step": 298
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.96653747558594,
|
|
"epoch": 1.5241730279898218,
|
|
"grad_norm": 0.41128833810114596,
|
|
"kl": 0.03143310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.517857164144516,
|
|
"reward_std": 0.20962579175829887,
|
|
"rewards/accuracy_reward": 0.517857164144516,
|
|
"step": 299
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.5268096923828,
|
|
"epoch": 1.5292620865139948,
|
|
"grad_norm": 0.3854823268565017,
|
|
"kl": 0.031005859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.4732143133878708,
|
|
"reward_std": 0.21179921738803387,
|
|
"rewards/accuracy_reward": 0.4732143133878708,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.1897506713867,
|
|
"epoch": 1.534351145038168,
|
|
"grad_norm": 0.3507539999839261,
|
|
"kl": 0.03533935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.7120536118745804,
|
|
"reward_std": 0.18208618462085724,
|
|
"rewards/accuracy_reward": 0.7120536118745804,
|
|
"step": 301
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.68082427978516,
|
|
"epoch": 1.5394402035623411,
|
|
"grad_norm": 0.4084442329727315,
|
|
"kl": 0.035064697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.5825893133878708,
|
|
"reward_std": 0.19158652145415545,
|
|
"rewards/accuracy_reward": 0.5825893133878708,
|
|
"step": 302
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 435.0580520629883,
|
|
"epoch": 1.544529262086514,
|
|
"grad_norm": 0.4381021420029986,
|
|
"kl": 0.034912109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0014,
|
|
"reward": 0.5357142984867096,
|
|
"reward_std": 0.21104642748832703,
|
|
"rewards/accuracy_reward": 0.5357142984867096,
|
|
"step": 303
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.93082427978516,
|
|
"epoch": 1.549618320610687,
|
|
"grad_norm": 0.4277718986497233,
|
|
"kl": 0.03472900390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0099,
|
|
"reward": 0.6406250298023224,
|
|
"reward_std": 0.19158761575818062,
|
|
"rewards/accuracy_reward": 0.6406250298023224,
|
|
"step": 304
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.15403747558594,
|
|
"epoch": 1.55470737913486,
|
|
"grad_norm": 0.3701582237907345,
|
|
"kl": 0.03363037109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.6205357536673546,
|
|
"reward_std": 0.1558861043304205,
|
|
"rewards/accuracy_reward": 0.6205357536673546,
|
|
"step": 305
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.51564025878906,
|
|
"epoch": 1.559796437659033,
|
|
"grad_norm": 0.3962037612138367,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0076,
|
|
"reward": 0.4642857387661934,
|
|
"reward_std": 0.19843707978725433,
|
|
"rewards/accuracy_reward": 0.4642857387661934,
|
|
"step": 306
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.88172149658203,
|
|
"epoch": 1.5648854961832062,
|
|
"grad_norm": 0.3602054384066848,
|
|
"kl": 0.031585693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.4866071566939354,
|
|
"reward_std": 0.17235196754336357,
|
|
"rewards/accuracy_reward": 0.4866071566939354,
|
|
"step": 307
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.9107360839844,
|
|
"epoch": 1.5699745547073791,
|
|
"grad_norm": 0.2773944269698253,
|
|
"kl": 0.0299072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.8147321790456772,
|
|
"reward_std": 0.09125630464404821,
|
|
"rewards/accuracy_reward": 0.8147321790456772,
|
|
"step": 308
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.0178756713867,
|
|
"epoch": 1.5750636132315523,
|
|
"grad_norm": 0.4004195588759506,
|
|
"kl": 0.031463623046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.4553571566939354,
|
|
"reward_std": 0.1763141043484211,
|
|
"rewards/accuracy_reward": 0.4553571566939354,
|
|
"step": 309
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.7723388671875,
|
|
"epoch": 1.5801526717557253,
|
|
"grad_norm": 0.3571687177934301,
|
|
"kl": 0.031951904296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.6205357313156128,
|
|
"reward_std": 0.17585763707756996,
|
|
"rewards/accuracy_reward": 0.6205357313156128,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.2812728881836,
|
|
"epoch": 1.5852417302798982,
|
|
"grad_norm": 0.3898645628909473,
|
|
"kl": 0.0343017578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0107,
|
|
"reward": 0.455357164144516,
|
|
"reward_std": 0.1789543218910694,
|
|
"rewards/accuracy_reward": 0.455357164144516,
|
|
"step": 311
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.9018020629883,
|
|
"epoch": 1.5903307888040712,
|
|
"grad_norm": 0.45925822888569257,
|
|
"kl": 0.0361328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0136,
|
|
"reward": 0.5401785895228386,
|
|
"reward_std": 0.27576345577836037,
|
|
"rewards/accuracy_reward": 0.5401785895228386,
|
|
"step": 312
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 376.39064025878906,
|
|
"epoch": 1.5954198473282442,
|
|
"grad_norm": 0.3952624210013506,
|
|
"kl": 0.03875732421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0172,
|
|
"reward": 0.502232164144516,
|
|
"reward_std": 0.128346499055624,
|
|
"rewards/accuracy_reward": 0.502232164144516,
|
|
"step": 313
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.2366256713867,
|
|
"epoch": 1.6005089058524173,
|
|
"grad_norm": 0.4302683829067842,
|
|
"kl": 0.03704833984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0154,
|
|
"reward": 0.4687500149011612,
|
|
"reward_std": 0.19160039350390434,
|
|
"rewards/accuracy_reward": 0.4687500149011612,
|
|
"step": 314
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.0759048461914,
|
|
"epoch": 1.6055979643765903,
|
|
"grad_norm": 0.3283633379052205,
|
|
"kl": 0.034576416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0065,
|
|
"reward": 0.6406250298023224,
|
|
"reward_std": 0.1151136914268136,
|
|
"rewards/accuracy_reward": 0.6406250298023224,
|
|
"step": 315
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.1585006713867,
|
|
"epoch": 1.6106870229007635,
|
|
"grad_norm": 0.41211305415430904,
|
|
"kl": 0.031707763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.4330357313156128,
|
|
"reward_std": 0.2064228653907776,
|
|
"rewards/accuracy_reward": 0.4330357313156128,
|
|
"step": 316
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.80804443359375,
|
|
"epoch": 1.6157760814249365,
|
|
"grad_norm": 0.4277863774804549,
|
|
"kl": 0.034759521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.562500037252903,
|
|
"reward_std": 0.2106967382133007,
|
|
"rewards/accuracy_reward": 0.562500037252903,
|
|
"step": 317
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.3906478881836,
|
|
"epoch": 1.6208651399491094,
|
|
"grad_norm": 0.42432018902000207,
|
|
"kl": 0.031585693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0022,
|
|
"reward": 0.5781250298023224,
|
|
"reward_std": 0.19926684722304344,
|
|
"rewards/accuracy_reward": 0.5781250298023224,
|
|
"step": 318
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.9263610839844,
|
|
"epoch": 1.6259541984732824,
|
|
"grad_norm": 0.41377970984118717,
|
|
"kl": 0.03594970703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0122,
|
|
"reward": 0.620535746216774,
|
|
"reward_std": 0.23870133608579636,
|
|
"rewards/accuracy_reward": 0.620535746216774,
|
|
"step": 319
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.1361770629883,
|
|
"epoch": 1.6310432569974553,
|
|
"grad_norm": 0.36151250654207095,
|
|
"kl": 0.03594970703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0074,
|
|
"reward": 0.5669643133878708,
|
|
"reward_std": 0.166221440769732,
|
|
"rewards/accuracy_reward": 0.5669643133878708,
|
|
"step": 320
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.2410888671875,
|
|
"epoch": 1.6361323155216285,
|
|
"grad_norm": 0.40267635839368887,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.4441964402794838,
|
|
"reward_std": 0.17330226674675941,
|
|
"rewards/accuracy_reward": 0.4441964402794838,
|
|
"step": 321
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.63841247558594,
|
|
"epoch": 1.6412213740458015,
|
|
"grad_norm": 0.33240541791531786,
|
|
"kl": 0.03179931640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0109,
|
|
"reward": 0.6049107387661934,
|
|
"reward_std": 0.1530414316803217,
|
|
"rewards/accuracy_reward": 0.6049107387661934,
|
|
"step": 322
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.9665298461914,
|
|
"epoch": 1.6463104325699747,
|
|
"grad_norm": 0.3621088793133356,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0053,
|
|
"reward": 0.6116071566939354,
|
|
"reward_std": 0.1585872322320938,
|
|
"rewards/accuracy_reward": 0.6116071566939354,
|
|
"step": 323
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.8973388671875,
|
|
"epoch": 1.6513994910941476,
|
|
"grad_norm": 0.4354893144018653,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0125,
|
|
"reward": 0.5736607313156128,
|
|
"reward_std": 0.2336120130494237,
|
|
"rewards/accuracy_reward": 0.5736607313156128,
|
|
"step": 324
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.6986846923828,
|
|
"epoch": 1.6564885496183206,
|
|
"grad_norm": 0.35881806451798626,
|
|
"kl": 0.0347900390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.4575892984867096,
|
|
"reward_std": 0.17096217907965183,
|
|
"rewards/accuracy_reward": 0.4575892984867096,
|
|
"step": 325
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.61163330078125,
|
|
"epoch": 1.6615776081424936,
|
|
"grad_norm": 0.4297659442878547,
|
|
"kl": 0.0390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0062,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.22144446894526482,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 326
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.7924270629883,
|
|
"epoch": 1.6666666666666665,
|
|
"grad_norm": 0.34862166048080134,
|
|
"kl": 0.033721923828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.4665178805589676,
|
|
"reward_std": 0.14675204921513796,
|
|
"rewards/accuracy_reward": 0.4665178805589676,
|
|
"step": 327
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.4107360839844,
|
|
"epoch": 1.6717557251908397,
|
|
"grad_norm": 0.4076142663981743,
|
|
"kl": 0.03118896484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0119,
|
|
"reward": 0.6339285969734192,
|
|
"reward_std": 0.22718551196157932,
|
|
"rewards/accuracy_reward": 0.6339285969734192,
|
|
"step": 328
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.5826110839844,
|
|
"epoch": 1.6768447837150129,
|
|
"grad_norm": 0.35303215684775435,
|
|
"kl": 0.03253173828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0017,
|
|
"reward": 0.620535746216774,
|
|
"reward_std": 0.1566992960870266,
|
|
"rewards/accuracy_reward": 0.620535746216774,
|
|
"step": 329
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.8147506713867,
|
|
"epoch": 1.6819338422391859,
|
|
"grad_norm": 0.3803328513367462,
|
|
"kl": 0.034515380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.604910746216774,
|
|
"reward_std": 0.19881199300289154,
|
|
"rewards/accuracy_reward": 0.604910746216774,
|
|
"step": 330
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.32591247558594,
|
|
"epoch": 1.6870229007633588,
|
|
"grad_norm": 0.4755914311292853,
|
|
"kl": 0.034027099609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.007,
|
|
"reward": 0.6205357387661934,
|
|
"reward_std": 0.24791720509529114,
|
|
"rewards/accuracy_reward": 0.6205357387661934,
|
|
"step": 331
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.4241256713867,
|
|
"epoch": 1.6921119592875318,
|
|
"grad_norm": 0.4104178979199164,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.6718750298023224,
|
|
"reward_std": 0.17258745804429054,
|
|
"rewards/accuracy_reward": 0.6718750298023224,
|
|
"step": 332
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.7500228881836,
|
|
"epoch": 1.6972010178117047,
|
|
"grad_norm": 0.39424631438568525,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.4709821604192257,
|
|
"reward_std": 0.1996939592063427,
|
|
"rewards/accuracy_reward": 0.4709821604192257,
|
|
"step": 333
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.5022430419922,
|
|
"epoch": 1.7022900763358777,
|
|
"grad_norm": 0.3814444494372407,
|
|
"kl": 0.03125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0105,
|
|
"reward": 0.5959821492433548,
|
|
"reward_std": 0.16038869880139828,
|
|
"rewards/accuracy_reward": 0.5959821492433548,
|
|
"step": 334
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.14733123779297,
|
|
"epoch": 1.7073791348600509,
|
|
"grad_norm": 0.41359950433869397,
|
|
"kl": 0.032470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0104,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.1900954283773899,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 335
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.3660888671875,
|
|
"epoch": 1.712468193384224,
|
|
"grad_norm": 0.40692091109165157,
|
|
"kl": 0.033294677734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.6004464700818062,
|
|
"reward_std": 0.16432827152311802,
|
|
"rewards/accuracy_reward": 0.6004464700818062,
|
|
"step": 336
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.2522506713867,
|
|
"epoch": 1.717557251908397,
|
|
"grad_norm": 0.4135208048065927,
|
|
"kl": 0.03399658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0152,
|
|
"reward": 0.5245535895228386,
|
|
"reward_std": 0.18983528763055801,
|
|
"rewards/accuracy_reward": 0.5245535895228386,
|
|
"step": 337
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.0692138671875,
|
|
"epoch": 1.72264631043257,
|
|
"grad_norm": 0.29376874707478684,
|
|
"kl": 0.03375244140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.5669643133878708,
|
|
"reward_std": 0.12192143499851227,
|
|
"rewards/accuracy_reward": 0.5669643133878708,
|
|
"step": 338
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.49778747558594,
|
|
"epoch": 1.727735368956743,
|
|
"grad_norm": 0.3968883357490178,
|
|
"kl": 0.032073974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0122,
|
|
"reward": 0.4910714626312256,
|
|
"reward_std": 0.1667480319738388,
|
|
"rewards/accuracy_reward": 0.4910714626312256,
|
|
"step": 339
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.55359649658203,
|
|
"epoch": 1.732824427480916,
|
|
"grad_norm": 0.4601899434981398,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0059,
|
|
"reward": 0.600446455180645,
|
|
"reward_std": 0.23292654752731323,
|
|
"rewards/accuracy_reward": 0.600446455180645,
|
|
"step": 340
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 332.2879638671875,
|
|
"epoch": 1.737913486005089,
|
|
"grad_norm": 0.5129007500319527,
|
|
"kl": 0.04302978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.5267857387661934,
|
|
"reward_std": 0.21882708743214607,
|
|
"rewards/accuracy_reward": 0.5267857387661934,
|
|
"step": 341
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.5268096923828,
|
|
"epoch": 1.743002544529262,
|
|
"grad_norm": 0.3902613328094504,
|
|
"kl": 0.034698486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0071,
|
|
"reward": 0.5223214477300644,
|
|
"reward_std": 0.1862612385302782,
|
|
"rewards/accuracy_reward": 0.5223214477300644,
|
|
"step": 342
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.68082427978516,
|
|
"epoch": 1.7480916030534353,
|
|
"grad_norm": 0.43948939716465657,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.573660746216774,
|
|
"reward_std": 0.21117087453603745,
|
|
"rewards/accuracy_reward": 0.573660746216774,
|
|
"step": 343
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.7165298461914,
|
|
"epoch": 1.7531806615776082,
|
|
"grad_norm": 0.4362451793291335,
|
|
"kl": 0.033477783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0086,
|
|
"reward": 0.5357143133878708,
|
|
"reward_std": 0.19573705457150936,
|
|
"rewards/accuracy_reward": 0.5357143133878708,
|
|
"step": 344
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.72769927978516,
|
|
"epoch": 1.7582697201017812,
|
|
"grad_norm": 0.41031786835866074,
|
|
"kl": 0.03466796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0072,
|
|
"reward": 0.5178571566939354,
|
|
"reward_std": 0.14876011572778225,
|
|
"rewards/accuracy_reward": 0.5178571566939354,
|
|
"step": 345
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 365.25447845458984,
|
|
"epoch": 1.7633587786259541,
|
|
"grad_norm": 0.46595037952621376,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0104,
|
|
"reward": 0.5781250223517418,
|
|
"reward_std": 0.18289937265217304,
|
|
"rewards/accuracy_reward": 0.5781250223517418,
|
|
"step": 346
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 374.93082427978516,
|
|
"epoch": 1.768447837150127,
|
|
"grad_norm": 0.45774566704542136,
|
|
"kl": 0.039154052734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.6049107536673546,
|
|
"reward_std": 0.18452896922826767,
|
|
"rewards/accuracy_reward": 0.6049107536673546,
|
|
"step": 347
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 367.59822845458984,
|
|
"epoch": 1.7735368956743003,
|
|
"grad_norm": 2.591658172875442,
|
|
"kl": 0.056793212890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0083,
|
|
"reward": 0.506696455180645,
|
|
"reward_std": 0.15466941893100739,
|
|
"rewards/accuracy_reward": 0.506696455180645,
|
|
"step": 348
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 374.86385345458984,
|
|
"epoch": 1.7786259541984732,
|
|
"grad_norm": 0.37643096212783456,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.7187500447034836,
|
|
"reward_std": 0.14254576992243528,
|
|
"rewards/accuracy_reward": 0.7187500447034836,
|
|
"step": 349
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.5736770629883,
|
|
"epoch": 1.7837150127226464,
|
|
"grad_norm": 0.3720669780083704,
|
|
"kl": 0.032257080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.6160714626312256,
|
|
"reward_std": 0.15607618167996407,
|
|
"rewards/accuracy_reward": 0.6160714626312256,
|
|
"step": 350
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.3705520629883,
|
|
"epoch": 1.7888040712468194,
|
|
"grad_norm": 0.36898135098898155,
|
|
"kl": 0.0330810546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.5424107313156128,
|
|
"reward_std": 0.1427915208041668,
|
|
"rewards/accuracy_reward": 0.5424107313156128,
|
|
"step": 351
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.2120666503906,
|
|
"epoch": 1.7938931297709924,
|
|
"grad_norm": 0.34968780393525445,
|
|
"kl": 0.03704833984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.5000000186264515,
|
|
"reward_std": 0.11996633186936378,
|
|
"rewards/accuracy_reward": 0.5000000186264515,
|
|
"step": 352
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.6160888671875,
|
|
"epoch": 1.7989821882951653,
|
|
"grad_norm": 2599.9910718457472,
|
|
"kl": 13.27606201171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.3824,
|
|
"reward": 0.5022321715950966,
|
|
"reward_std": 0.18969968520104885,
|
|
"rewards/accuracy_reward": 0.5022321715950966,
|
|
"step": 353
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.83260345458984,
|
|
"epoch": 1.8040712468193383,
|
|
"grad_norm": 0.34414783813611044,
|
|
"kl": 0.039794921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.6183035969734192,
|
|
"reward_std": 0.16032780893146992,
|
|
"rewards/accuracy_reward": 0.6183035969734192,
|
|
"step": 354
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.9084930419922,
|
|
"epoch": 1.8091603053435115,
|
|
"grad_norm": 0.3857649031928157,
|
|
"kl": 0.039215087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0128,
|
|
"reward": 0.5937500149011612,
|
|
"reward_std": 0.14901735074818134,
|
|
"rewards/accuracy_reward": 0.5937500149011612,
|
|
"step": 355
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.50672149658203,
|
|
"epoch": 1.8142493638676844,
|
|
"grad_norm": 0.349036494255612,
|
|
"kl": 0.033050537109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0025,
|
|
"reward": 0.580357164144516,
|
|
"reward_std": 0.1297006607055664,
|
|
"rewards/accuracy_reward": 0.580357164144516,
|
|
"step": 356
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.9174346923828,
|
|
"epoch": 1.8193384223918576,
|
|
"grad_norm": 0.27196929473992737,
|
|
"kl": 0.034759521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.5736607387661934,
|
|
"reward_std": 0.08670743089169264,
|
|
"rewards/accuracy_reward": 0.5736607387661934,
|
|
"step": 357
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.54466247558594,
|
|
"epoch": 1.8244274809160306,
|
|
"grad_norm": 0.35237072076561626,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.009,
|
|
"reward": 0.6785714626312256,
|
|
"reward_std": 0.11349314637482166,
|
|
"rewards/accuracy_reward": 0.6785714626312256,
|
|
"step": 358
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 370.69197845458984,
|
|
"epoch": 1.8295165394402035,
|
|
"grad_norm": 0.31748531000848407,
|
|
"kl": 0.0367431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.5424107387661934,
|
|
"reward_std": 0.10300773568451405,
|
|
"rewards/accuracy_reward": 0.5424107387661934,
|
|
"step": 359
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.5401916503906,
|
|
"epoch": 1.8346055979643765,
|
|
"grad_norm": 0.3810081949243458,
|
|
"kl": 0.0352783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0056,
|
|
"reward": 0.81026791036129,
|
|
"reward_std": 0.1514055710285902,
|
|
"rewards/accuracy_reward": 0.81026791036129,
|
|
"step": 360
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.8549270629883,
|
|
"epoch": 1.8396946564885495,
|
|
"grad_norm": 0.2877504412241006,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.424107164144516,
|
|
"reward_std": 0.09726398717612028,
|
|
"rewards/accuracy_reward": 0.424107164144516,
|
|
"step": 361
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.46207427978516,
|
|
"epoch": 1.8447837150127226,
|
|
"grad_norm": 0.3646209313520994,
|
|
"kl": 0.04241943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.6093750074505806,
|
|
"reward_std": 0.1521219816058874,
|
|
"rewards/accuracy_reward": 0.6093750074505806,
|
|
"step": 362
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 458.89734649658203,
|
|
"epoch": 1.8498727735368958,
|
|
"grad_norm": 0.4243928780122091,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0187,
|
|
"reward": 0.4665178880095482,
|
|
"reward_std": 0.20290330052375793,
|
|
"rewards/accuracy_reward": 0.4665178880095482,
|
|
"step": 363
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.5446548461914,
|
|
"epoch": 1.8549618320610688,
|
|
"grad_norm": 0.31994331064348525,
|
|
"kl": 0.0322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0074,
|
|
"reward": 0.6450893133878708,
|
|
"reward_std": 0.12879652343690395,
|
|
"rewards/accuracy_reward": 0.6450893133878708,
|
|
"step": 364
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.03797149658203,
|
|
"epoch": 1.8600508905852418,
|
|
"grad_norm": 0.40152860734111306,
|
|
"kl": 0.037322998046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.6004464775323868,
|
|
"reward_std": 0.13713151961565018,
|
|
"rewards/accuracy_reward": 0.6004464775323868,
|
|
"step": 365
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.9241180419922,
|
|
"epoch": 1.8651399491094147,
|
|
"grad_norm": 0.2987453957740081,
|
|
"kl": 0.029693603515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0098,
|
|
"reward": 0.589285746216774,
|
|
"reward_std": 0.12920781783759594,
|
|
"rewards/accuracy_reward": 0.589285746216774,
|
|
"step": 366
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.2991256713867,
|
|
"epoch": 1.8702290076335877,
|
|
"grad_norm": 0.554465600930555,
|
|
"kl": 0.0467529296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.6004464477300644,
|
|
"reward_std": 0.2248956747353077,
|
|
"rewards/accuracy_reward": 0.6004464477300644,
|
|
"step": 367
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.3013610839844,
|
|
"epoch": 1.8753180661577609,
|
|
"grad_norm": 0.31516645881936106,
|
|
"kl": 0.035675048828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.6071428954601288,
|
|
"reward_std": 0.11709805019199848,
|
|
"rewards/accuracy_reward": 0.6071428954601288,
|
|
"step": 368
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.3504638671875,
|
|
"epoch": 1.8804071246819338,
|
|
"grad_norm": 0.29945925934244355,
|
|
"kl": 0.043212890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0081,
|
|
"reward": 0.6138393208384514,
|
|
"reward_std": 0.1219743937253952,
|
|
"rewards/accuracy_reward": 0.6138393208384514,
|
|
"step": 369
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.60717010498047,
|
|
"epoch": 1.885496183206107,
|
|
"grad_norm": 0.400533724930988,
|
|
"kl": 0.05548095703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0093,
|
|
"reward": 0.5736607387661934,
|
|
"reward_std": 0.15631252340972424,
|
|
"rewards/accuracy_reward": 0.5736607387661934,
|
|
"step": 370
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.2455520629883,
|
|
"epoch": 1.89058524173028,
|
|
"grad_norm": 0.3670173912293237,
|
|
"kl": 0.0465087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0021,
|
|
"reward": 0.6183035895228386,
|
|
"reward_std": 0.13964036107063293,
|
|
"rewards/accuracy_reward": 0.6183035895228386,
|
|
"step": 371
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 439.30359649658203,
|
|
"epoch": 1.895674300254453,
|
|
"grad_norm": 0.4588507404224872,
|
|
"kl": 0.04742431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.502232164144516,
|
|
"reward_std": 0.1481290515512228,
|
|
"rewards/accuracy_reward": 0.502232164144516,
|
|
"step": 372
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.11385345458984,
|
|
"epoch": 1.900763358778626,
|
|
"grad_norm": 39.86349118531167,
|
|
"kl": 0.4259033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0122,
|
|
"reward": 0.6093750149011612,
|
|
"reward_std": 0.16022942773997784,
|
|
"rewards/accuracy_reward": 0.6093750149011612,
|
|
"step": 373
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.35939025878906,
|
|
"epoch": 1.9058524173027989,
|
|
"grad_norm": 0.33153842654680793,
|
|
"kl": 0.0458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0035,
|
|
"reward": 0.627232164144516,
|
|
"reward_std": 0.14701843727380037,
|
|
"rewards/accuracy_reward": 0.627232164144516,
|
|
"step": 374
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.33707427978516,
|
|
"epoch": 1.910941475826972,
|
|
"grad_norm": 0.33889936958064354,
|
|
"kl": 0.044921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.5245535969734192,
|
|
"reward_std": 0.14769601821899414,
|
|
"rewards/accuracy_reward": 0.5245535969734192,
|
|
"step": 375
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.78350830078125,
|
|
"epoch": 1.916030534351145,
|
|
"grad_norm": 0.4102871349992259,
|
|
"kl": 0.04376220703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0196,
|
|
"reward": 0.3928571566939354,
|
|
"reward_std": 0.21786130033433437,
|
|
"rewards/accuracy_reward": 0.3928571566939354,
|
|
"step": 376
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.90403747558594,
|
|
"epoch": 1.9211195928753182,
|
|
"grad_norm": 0.33711468718878645,
|
|
"kl": 0.044708251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.4464285969734192,
|
|
"reward_std": 0.14733134396374226,
|
|
"rewards/accuracy_reward": 0.4464285969734192,
|
|
"step": 377
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.5491256713867,
|
|
"epoch": 1.9262086513994912,
|
|
"grad_norm": 0.35638819766037244,
|
|
"kl": 0.04266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.5156250223517418,
|
|
"reward_std": 0.14308641105890274,
|
|
"rewards/accuracy_reward": 0.5156250223517418,
|
|
"step": 378
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.14064025878906,
|
|
"epoch": 1.9312977099236641,
|
|
"grad_norm": 0.35502262918683586,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0089,
|
|
"reward": 0.5758928880095482,
|
|
"reward_std": 0.09469473734498024,
|
|
"rewards/accuracy_reward": 0.5758928880095482,
|
|
"step": 379
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 455.4799270629883,
|
|
"epoch": 1.936386768447837,
|
|
"grad_norm": 0.330973980295281,
|
|
"kl": 0.03887939453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0061,
|
|
"reward": 0.5691964477300644,
|
|
"reward_std": 0.15365827456116676,
|
|
"rewards/accuracy_reward": 0.5691964477300644,
|
|
"step": 380
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.4643020629883,
|
|
"epoch": 1.94147582697201,
|
|
"grad_norm": 0.43447773535161155,
|
|
"kl": 0.039306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0066,
|
|
"reward": 0.537946455180645,
|
|
"reward_std": 0.20215855166316032,
|
|
"rewards/accuracy_reward": 0.537946455180645,
|
|
"step": 381
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.75001525878906,
|
|
"epoch": 1.9465648854961832,
|
|
"grad_norm": 0.3709496721140309,
|
|
"kl": 0.04852294921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0046,
|
|
"reward": 0.647321455180645,
|
|
"reward_std": 0.19486423209309578,
|
|
"rewards/accuracy_reward": 0.647321455180645,
|
|
"step": 382
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.32144927978516,
|
|
"epoch": 1.9516539440203562,
|
|
"grad_norm": 0.33983340450443467,
|
|
"kl": 0.04833984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.642857164144516,
|
|
"reward_std": 0.13795265182852745,
|
|
"rewards/accuracy_reward": 0.642857164144516,
|
|
"step": 383
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.7812728881836,
|
|
"epoch": 1.9567430025445294,
|
|
"grad_norm": 0.3542287261755593,
|
|
"kl": 0.0501708984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.016,
|
|
"reward": 0.6026786044239998,
|
|
"reward_std": 0.130963871255517,
|
|
"rewards/accuracy_reward": 0.6026786044239998,
|
|
"step": 384
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.20314025878906,
|
|
"epoch": 1.9618320610687023,
|
|
"grad_norm": 0.4141484086187222,
|
|
"kl": 0.0703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0165,
|
|
"reward": 0.5111607387661934,
|
|
"reward_std": 0.17906580865383148,
|
|
"rewards/accuracy_reward": 0.5111607387661934,
|
|
"step": 385
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.6138610839844,
|
|
"epoch": 1.9669211195928753,
|
|
"grad_norm": 0.5824860644956747,
|
|
"kl": 0.067626953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0072,
|
|
"reward": 0.4977678805589676,
|
|
"reward_std": 0.18197993654757738,
|
|
"rewards/accuracy_reward": 0.4977678805589676,
|
|
"step": 386
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.7143020629883,
|
|
"epoch": 1.9720101781170483,
|
|
"grad_norm": 0.4942465240724718,
|
|
"kl": 0.062255859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0053,
|
|
"reward": 0.4575893133878708,
|
|
"reward_std": 0.13631832972168922,
|
|
"rewards/accuracy_reward": 0.4575893133878708,
|
|
"step": 387
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.1785888671875,
|
|
"epoch": 1.9770992366412212,
|
|
"grad_norm": 0.3869605632287253,
|
|
"kl": 0.04345703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0074,
|
|
"reward": 0.5825893208384514,
|
|
"reward_std": 0.16959000378847122,
|
|
"rewards/accuracy_reward": 0.5825893208384514,
|
|
"step": 388
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.16072845458984,
|
|
"epoch": 1.9821882951653944,
|
|
"grad_norm": 0.37786287346057257,
|
|
"kl": 0.03607177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0031,
|
|
"reward": 0.595982164144516,
|
|
"reward_std": 0.14545245841145515,
|
|
"rewards/accuracy_reward": 0.595982164144516,
|
|
"step": 389
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.7143096923828,
|
|
"epoch": 1.9872773536895676,
|
|
"grad_norm": 0.39575592064903486,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0182,
|
|
"reward": 0.6718750149011612,
|
|
"reward_std": 0.18826821073889732,
|
|
"rewards/accuracy_reward": 0.6718750149011612,
|
|
"step": 390
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.8772430419922,
|
|
"epoch": 1.9923664122137406,
|
|
"grad_norm": 0.4651636579207121,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0161,
|
|
"reward": 0.5625000298023224,
|
|
"reward_std": 0.24423056095838547,
|
|
"rewards/accuracy_reward": 0.5625000298023224,
|
|
"step": 391
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.13463592529297,
|
|
"epoch": 1.9974554707379135,
|
|
"grad_norm": 0.44450093547546876,
|
|
"kl": 0.036407470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0022,
|
|
"reward": 0.6250000298023224,
|
|
"reward_std": 0.2150175478309393,
|
|
"rewards/accuracy_reward": 0.6250000298023224,
|
|
"step": 392
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.3460006713867,
|
|
"epoch": 2.005089058524173,
|
|
"grad_norm": 0.2742009983878323,
|
|
"kl": 0.041015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.754464328289032,
|
|
"reward_std": 0.1010371521115303,
|
|
"rewards/accuracy_reward": 0.754464328289032,
|
|
"step": 393
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.9598388671875,
|
|
"epoch": 2.010178117048346,
|
|
"grad_norm": 0.3472987389060835,
|
|
"kl": 0.037567138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.5111607313156128,
|
|
"reward_std": 0.10763046145439148,
|
|
"rewards/accuracy_reward": 0.5111607313156128,
|
|
"step": 394
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 370.2410888671875,
|
|
"epoch": 2.015267175572519,
|
|
"grad_norm": 0.39088139408455136,
|
|
"kl": 0.0390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0172,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.14116082713007927,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 395
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.7745666503906,
|
|
"epoch": 2.0203562340966923,
|
|
"grad_norm": 0.34232536414655745,
|
|
"kl": 0.035797119140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0124,
|
|
"reward": 0.8482142984867096,
|
|
"reward_std": 0.12868122477084398,
|
|
"rewards/accuracy_reward": 0.8482142984867096,
|
|
"step": 396
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 357.08484649658203,
|
|
"epoch": 2.0254452926208653,
|
|
"grad_norm": 0.399584855628817,
|
|
"kl": 0.039398193359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.6941964477300644,
|
|
"reward_std": 0.14294251147657633,
|
|
"rewards/accuracy_reward": 0.6941964477300644,
|
|
"step": 397
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.6897506713867,
|
|
"epoch": 2.030534351145038,
|
|
"grad_norm": 130.25491711312384,
|
|
"kl": 4.02850341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.1802,
|
|
"reward": 0.7656250447034836,
|
|
"reward_std": 0.11223897151648998,
|
|
"rewards/accuracy_reward": 0.7656250447034836,
|
|
"step": 398
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.0736846923828,
|
|
"epoch": 2.035623409669211,
|
|
"grad_norm": 0.46588574160831064,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.01,
|
|
"reward": 0.7008928805589676,
|
|
"reward_std": 0.20006506331264973,
|
|
"rewards/accuracy_reward": 0.7008928805589676,
|
|
"step": 399
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.6339416503906,
|
|
"epoch": 2.040712468193384,
|
|
"grad_norm": 0.373123724241272,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0058,
|
|
"reward": 0.7321428954601288,
|
|
"reward_std": 0.13184583187103271,
|
|
"rewards/accuracy_reward": 0.7321428954601288,
|
|
"step": 400
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.9107360839844,
|
|
"epoch": 2.045801526717557,
|
|
"grad_norm": 0.31806823628981257,
|
|
"kl": 0.04083251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0053,
|
|
"reward": 0.7767857313156128,
|
|
"reward_std": 0.09784962050616741,
|
|
"rewards/accuracy_reward": 0.7767857313156128,
|
|
"step": 401
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.2254638671875,
|
|
"epoch": 2.05089058524173,
|
|
"grad_norm": 0.42290855068697064,
|
|
"kl": 0.0394287109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0061,
|
|
"reward": 0.6562500298023224,
|
|
"reward_std": 0.19378305226564407,
|
|
"rewards/accuracy_reward": 0.6562500298023224,
|
|
"step": 402
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 426.7433319091797,
|
|
"epoch": 2.0559796437659035,
|
|
"grad_norm": 0.2605804750219208,
|
|
"kl": 0.037017822265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0076,
|
|
"reward": 0.8035714626312256,
|
|
"reward_std": 0.06577018182724714,
|
|
"rewards/accuracy_reward": 0.8035714626312256,
|
|
"step": 403
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.4776916503906,
|
|
"epoch": 2.0610687022900764,
|
|
"grad_norm": 0.280147550531787,
|
|
"kl": 0.035491943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.7455357611179352,
|
|
"reward_std": 0.07104842457920313,
|
|
"rewards/accuracy_reward": 0.7455357611179352,
|
|
"step": 404
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.09822845458984,
|
|
"epoch": 2.0661577608142494,
|
|
"grad_norm": 0.3281689744499087,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.7611607611179352,
|
|
"reward_std": 0.1360610779374838,
|
|
"rewards/accuracy_reward": 0.7611607611179352,
|
|
"step": 405
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.3593978881836,
|
|
"epoch": 2.0712468193384224,
|
|
"grad_norm": 0.31823988820677146,
|
|
"kl": 0.0439453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.12937422282993793,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 406
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.81251525878906,
|
|
"epoch": 2.0763358778625953,
|
|
"grad_norm": 0.3993553492680455,
|
|
"kl": 0.04681396484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0066,
|
|
"reward": 0.7232143133878708,
|
|
"reward_std": 0.09746193699538708,
|
|
"rewards/accuracy_reward": 0.7232143133878708,
|
|
"step": 407
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 438.1026916503906,
|
|
"epoch": 2.0814249363867683,
|
|
"grad_norm": 0.26287455186122644,
|
|
"kl": 0.0369873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0089,
|
|
"reward": 0.6250000298023224,
|
|
"reward_std": 0.08987135346978903,
|
|
"rewards/accuracy_reward": 0.6250000298023224,
|
|
"step": 408
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.8459930419922,
|
|
"epoch": 2.0865139949109412,
|
|
"grad_norm": 0.3305582164075515,
|
|
"kl": 0.04248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0098,
|
|
"reward": 0.6964285969734192,
|
|
"reward_std": 0.09424471203237772,
|
|
"rewards/accuracy_reward": 0.6964285969734192,
|
|
"step": 409
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.44644927978516,
|
|
"epoch": 2.0916030534351147,
|
|
"grad_norm": 0.3452528519941532,
|
|
"kl": 0.0430908203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0109,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.13682775013148785,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 410
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.33707427978516,
|
|
"epoch": 2.0966921119592876,
|
|
"grad_norm": 0.3699552318627565,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.7433035969734192,
|
|
"reward_std": 0.10905144643038511,
|
|
"rewards/accuracy_reward": 0.7433035969734192,
|
|
"step": 411
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.36832427978516,
|
|
"epoch": 2.1017811704834606,
|
|
"grad_norm": 0.44646836528989053,
|
|
"kl": 0.04669189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0074,
|
|
"reward": 0.6473214626312256,
|
|
"reward_std": 0.16127971559762955,
|
|
"rewards/accuracy_reward": 0.6473214626312256,
|
|
"step": 412
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.11385345458984,
|
|
"epoch": 2.1068702290076335,
|
|
"grad_norm": 0.2947258485589255,
|
|
"kl": 0.03851318359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0006,
|
|
"reward": 0.7008928954601288,
|
|
"reward_std": 0.10946544446051121,
|
|
"rewards/accuracy_reward": 0.7008928954601288,
|
|
"step": 413
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.6986770629883,
|
|
"epoch": 2.1119592875318065,
|
|
"grad_norm": 0.3735043766143211,
|
|
"kl": 0.0386962890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0106,
|
|
"reward": 0.7656250298023224,
|
|
"reward_std": 0.13605837430804968,
|
|
"rewards/accuracy_reward": 0.7656250298023224,
|
|
"step": 414
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.44644927978516,
|
|
"epoch": 2.1170483460559795,
|
|
"grad_norm": 0.33044634292850333,
|
|
"kl": 0.03399658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.6941964626312256,
|
|
"reward_std": 0.1353988889604807,
|
|
"rewards/accuracy_reward": 0.6941964626312256,
|
|
"step": 415
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.8504638671875,
|
|
"epoch": 2.122137404580153,
|
|
"grad_norm": 0.3482109526429795,
|
|
"kl": 0.038238525390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0099,
|
|
"reward": 0.6651786118745804,
|
|
"reward_std": 0.09537875186651945,
|
|
"rewards/accuracy_reward": 0.6651786118745804,
|
|
"step": 416
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.33707427978516,
|
|
"epoch": 2.127226463104326,
|
|
"grad_norm": 0.23540557666160633,
|
|
"kl": 0.035552978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.6473214477300644,
|
|
"reward_std": 0.06424972135573626,
|
|
"rewards/accuracy_reward": 0.6473214477300644,
|
|
"step": 417
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.34153747558594,
|
|
"epoch": 2.132315521628499,
|
|
"grad_norm": 0.4010454738516299,
|
|
"kl": 0.0411376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.6049107313156128,
|
|
"reward_std": 0.15023404452949762,
|
|
"rewards/accuracy_reward": 0.6049107313156128,
|
|
"step": 418
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.90403747558594,
|
|
"epoch": 2.1374045801526718,
|
|
"grad_norm": 0.3806691644319369,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0049,
|
|
"reward": 0.6383928954601288,
|
|
"reward_std": 0.10896879248321056,
|
|
"rewards/accuracy_reward": 0.6383928954601288,
|
|
"step": 419
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.0044860839844,
|
|
"epoch": 2.1424936386768447,
|
|
"grad_norm": 0.3365895191854068,
|
|
"kl": 0.0411376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.6785714700818062,
|
|
"reward_std": 0.10474831517785788,
|
|
"rewards/accuracy_reward": 0.6785714700818062,
|
|
"step": 420
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 374.0714416503906,
|
|
"epoch": 2.1475826972010177,
|
|
"grad_norm": 0.3699404951725154,
|
|
"kl": 0.037261962890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.6406250149011612,
|
|
"reward_std": 0.1640634946525097,
|
|
"rewards/accuracy_reward": 0.6406250149011612,
|
|
"step": 421
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 361.0401916503906,
|
|
"epoch": 2.1526717557251906,
|
|
"grad_norm": 0.3745712794228524,
|
|
"kl": 0.04376220703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0056,
|
|
"reward": 0.7477678954601288,
|
|
"reward_std": 0.14180846884846687,
|
|
"rewards/accuracy_reward": 0.7477678954601288,
|
|
"step": 422
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.10716247558594,
|
|
"epoch": 2.157760814249364,
|
|
"grad_norm": 0.23307581680227343,
|
|
"kl": 0.03656005859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0016,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.06262173783034086,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 423
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.1875228881836,
|
|
"epoch": 2.162849872773537,
|
|
"grad_norm": 0.38264157640860647,
|
|
"kl": 0.04461669921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0076,
|
|
"reward": 0.6383928805589676,
|
|
"reward_std": 0.15253328159451485,
|
|
"rewards/accuracy_reward": 0.6383928805589676,
|
|
"step": 424
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.47322845458984,
|
|
"epoch": 2.16793893129771,
|
|
"grad_norm": 0.30949443469371074,
|
|
"kl": 0.034393310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0079,
|
|
"reward": 0.6406250260770321,
|
|
"reward_std": 0.10763046704232693,
|
|
"rewards/accuracy_reward": 0.6406250260770321,
|
|
"step": 425
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.3437728881836,
|
|
"epoch": 2.173027989821883,
|
|
"grad_norm": 0.34019617897471943,
|
|
"kl": 0.0382080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.5825893059372902,
|
|
"reward_std": 0.10456457268446684,
|
|
"rewards/accuracy_reward": 0.5825893059372902,
|
|
"step": 426
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 378.52010345458984,
|
|
"epoch": 2.178117048346056,
|
|
"grad_norm": 0.252142055366362,
|
|
"kl": 0.0340576171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.6808036118745804,
|
|
"reward_std": 0.07311739027500153,
|
|
"rewards/accuracy_reward": 0.6808036118745804,
|
|
"step": 427
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.3526916503906,
|
|
"epoch": 2.183206106870229,
|
|
"grad_norm": 0.2591769614280603,
|
|
"kl": 0.0418701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.761160746216774,
|
|
"reward_std": 0.09067067224532366,
|
|
"rewards/accuracy_reward": 0.761160746216774,
|
|
"step": 428
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.3326110839844,
|
|
"epoch": 2.188295165394402,
|
|
"grad_norm": 0.3599011585237677,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.6785714477300644,
|
|
"reward_std": 0.13360299356281757,
|
|
"rewards/accuracy_reward": 0.6785714477300644,
|
|
"step": 429
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 368.3281478881836,
|
|
"epoch": 2.1933842239185752,
|
|
"grad_norm": 0.3769927627686171,
|
|
"kl": 0.04144287109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.5781250298023224,
|
|
"reward_std": 0.09783685021102428,
|
|
"rewards/accuracy_reward": 0.5781250298023224,
|
|
"step": 430
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.1473388671875,
|
|
"epoch": 2.198473282442748,
|
|
"grad_norm": 0.2943189171062689,
|
|
"kl": 0.035247802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0046,
|
|
"reward": 0.6540178880095482,
|
|
"reward_std": 0.10128805786371231,
|
|
"rewards/accuracy_reward": 0.6540178880095482,
|
|
"step": 431
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 379.0513610839844,
|
|
"epoch": 2.203562340966921,
|
|
"grad_norm": 0.34144583657086436,
|
|
"kl": 0.03643798828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.7767857313156128,
|
|
"reward_std": 0.12260545510798693,
|
|
"rewards/accuracy_reward": 0.7767857313156128,
|
|
"step": 432
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.0870819091797,
|
|
"epoch": 2.208651399491094,
|
|
"grad_norm": 0.3126396945263014,
|
|
"kl": 0.0384521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.0747014582157135,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 433
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.0446548461914,
|
|
"epoch": 2.213740458015267,
|
|
"grad_norm": 0.3709590910879604,
|
|
"kl": 0.03729248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0125,
|
|
"reward": 0.8102678954601288,
|
|
"reward_std": 0.16197794489562511,
|
|
"rewards/accuracy_reward": 0.8102678954601288,
|
|
"step": 434
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.2076110839844,
|
|
"epoch": 2.21882951653944,
|
|
"grad_norm": 0.28491003800984027,
|
|
"kl": 0.0362548828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0095,
|
|
"reward": 0.6071428805589676,
|
|
"reward_std": 0.08733848761767149,
|
|
"rewards/accuracy_reward": 0.6071428805589676,
|
|
"step": 435
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.9710006713867,
|
|
"epoch": 2.223918575063613,
|
|
"grad_norm": 0.2349347321988743,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.689732164144516,
|
|
"reward_std": 0.06589301396161318,
|
|
"rewards/accuracy_reward": 0.689732164144516,
|
|
"step": 436
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.72769927978516,
|
|
"epoch": 2.2290076335877864,
|
|
"grad_norm": 0.29500242270003374,
|
|
"kl": 0.03863525390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0115,
|
|
"reward": 0.6294643133878708,
|
|
"reward_std": 0.07482158578932285,
|
|
"rewards/accuracy_reward": 0.6294643133878708,
|
|
"step": 437
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.10716247558594,
|
|
"epoch": 2.2340966921119594,
|
|
"grad_norm": 0.26301039338953314,
|
|
"kl": 0.039031982421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0066,
|
|
"reward": 0.7611607611179352,
|
|
"reward_std": 0.07249427866190672,
|
|
"rewards/accuracy_reward": 0.7611607611179352,
|
|
"step": 438
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.20538330078125,
|
|
"epoch": 2.2391857506361323,
|
|
"grad_norm": 0.3019924103632674,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.09883901290595531,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 439
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.13172149658203,
|
|
"epoch": 2.2442748091603053,
|
|
"grad_norm": 0.17822494728690552,
|
|
"kl": 0.04278564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.6986607611179352,
|
|
"reward_std": 0.0267857164144516,
|
|
"rewards/accuracy_reward": 0.6986607611179352,
|
|
"step": 440
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.02233123779297,
|
|
"epoch": 2.2493638676844783,
|
|
"grad_norm": 0.21469670399697338,
|
|
"kl": 0.036468505859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0006,
|
|
"reward": 0.7254464626312256,
|
|
"reward_std": 0.03772235009819269,
|
|
"rewards/accuracy_reward": 0.7254464626312256,
|
|
"step": 441
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.6116180419922,
|
|
"epoch": 2.2544529262086512,
|
|
"grad_norm": 0.16764568530225557,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0082,
|
|
"reward": 0.6250000149011612,
|
|
"reward_std": 0.04935094714164734,
|
|
"rewards/accuracy_reward": 0.6250000149011612,
|
|
"step": 442
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.14288330078125,
|
|
"epoch": 2.2595419847328246,
|
|
"grad_norm": 0.21281068247363297,
|
|
"kl": 0.036956787109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.7098214626312256,
|
|
"reward_std": 0.05728259403258562,
|
|
"rewards/accuracy_reward": 0.7098214626312256,
|
|
"step": 443
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.5178756713867,
|
|
"epoch": 2.2646310432569976,
|
|
"grad_norm": 0.9965814898124155,
|
|
"kl": 0.0435791015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.5111607350409031,
|
|
"reward_std": 0.0895366258919239,
|
|
"rewards/accuracy_reward": 0.5111607350409031,
|
|
"step": 444
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.6986770629883,
|
|
"epoch": 2.2697201017811706,
|
|
"grad_norm": 0.3309671635916835,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.7209821939468384,
|
|
"reward_std": 0.10700734984129667,
|
|
"rewards/accuracy_reward": 0.7209821939468384,
|
|
"step": 445
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.07144927978516,
|
|
"epoch": 2.2748091603053435,
|
|
"grad_norm": 0.4272193039446874,
|
|
"kl": 0.04412841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.017,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.08104903623461723,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 446
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.18082427978516,
|
|
"epoch": 2.2798982188295165,
|
|
"grad_norm": 0.3000200900552537,
|
|
"kl": 0.037628173828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.6294643208384514,
|
|
"reward_std": 0.07771168742328882,
|
|
"rewards/accuracy_reward": 0.6294643208384514,
|
|
"step": 447
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.3526916503906,
|
|
"epoch": 2.2849872773536894,
|
|
"grad_norm": 1.937837118819002,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.5558036044239998,
|
|
"reward_std": 0.1322879195213318,
|
|
"rewards/accuracy_reward": 0.5558036044239998,
|
|
"step": 448
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.02903747558594,
|
|
"epoch": 2.2900763358778624,
|
|
"grad_norm": 0.7869042879630593,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0101,
|
|
"reward": 0.7723214626312256,
|
|
"reward_std": 0.08922641910612583,
|
|
"rewards/accuracy_reward": 0.7723214626312256,
|
|
"step": 449
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.6027069091797,
|
|
"epoch": 2.2951653944020354,
|
|
"grad_norm": 0.2914751051443095,
|
|
"kl": 0.03363037109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0058,
|
|
"reward": 0.7678571790456772,
|
|
"reward_std": 0.10282399691641331,
|
|
"rewards/accuracy_reward": 0.7678571790456772,
|
|
"step": 450
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.5424270629883,
|
|
"epoch": 2.300254452926209,
|
|
"grad_norm": 0.35247194342747756,
|
|
"kl": 0.03515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0068,
|
|
"reward": 0.689732164144516,
|
|
"reward_std": 0.1558098942041397,
|
|
"rewards/accuracy_reward": 0.689732164144516,
|
|
"step": 451
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.58260345458984,
|
|
"epoch": 2.3053435114503817,
|
|
"grad_norm": 0.31912449668655113,
|
|
"kl": 0.034912109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.7678571939468384,
|
|
"reward_std": 0.12644578889012337,
|
|
"rewards/accuracy_reward": 0.7678571939468384,
|
|
"step": 452
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.71653747558594,
|
|
"epoch": 2.3104325699745547,
|
|
"grad_norm": 0.6029598774597014,
|
|
"kl": 0.033966064453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0036,
|
|
"reward": 0.756696455180645,
|
|
"reward_std": 0.12330368347465992,
|
|
"rewards/accuracy_reward": 0.756696455180645,
|
|
"step": 453
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.4620666503906,
|
|
"epoch": 2.3155216284987277,
|
|
"grad_norm": 0.3081410698912104,
|
|
"kl": 0.03985595703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.683035746216774,
|
|
"reward_std": 0.11600309796631336,
|
|
"rewards/accuracy_reward": 0.683035746216774,
|
|
"step": 454
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.7879638671875,
|
|
"epoch": 2.3206106870229006,
|
|
"grad_norm": 0.30355954965410276,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.11273291427642107,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 455
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.8995666503906,
|
|
"epoch": 2.325699745547074,
|
|
"grad_norm": 0.2773728066737305,
|
|
"kl": 0.03631591796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0061,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.11543403938412666,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 456
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.0960006713867,
|
|
"epoch": 2.330788804071247,
|
|
"grad_norm": 0.2871216023789764,
|
|
"kl": 0.032318115234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0005,
|
|
"reward": 0.6250000298023224,
|
|
"reward_std": 0.09111165534704924,
|
|
"rewards/accuracy_reward": 0.6250000298023224,
|
|
"step": 457
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.71876525878906,
|
|
"epoch": 2.33587786259542,
|
|
"grad_norm": 0.34728748119133596,
|
|
"kl": 0.033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.16387071087956429,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 458
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.1629638671875,
|
|
"epoch": 2.340966921119593,
|
|
"grad_norm": 0.2710432422591856,
|
|
"kl": 0.03680419921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0041,
|
|
"reward": 0.6450893133878708,
|
|
"reward_std": 0.08534066285938025,
|
|
"rewards/accuracy_reward": 0.6450893133878708,
|
|
"step": 459
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 453.3393020629883,
|
|
"epoch": 2.346055979643766,
|
|
"grad_norm": 0.2531748418736068,
|
|
"kl": 0.03033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0071,
|
|
"reward": 0.6428571939468384,
|
|
"reward_std": 0.06011178996413946,
|
|
"rewards/accuracy_reward": 0.6428571939468384,
|
|
"step": 460
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.7924346923828,
|
|
"epoch": 2.351145038167939,
|
|
"grad_norm": 0.29212336637874164,
|
|
"kl": 0.033905029296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0043,
|
|
"reward": 0.7901786118745804,
|
|
"reward_std": 0.06558010634034872,
|
|
"rewards/accuracy_reward": 0.7901786118745804,
|
|
"step": 461
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.77457427978516,
|
|
"epoch": 2.356234096692112,
|
|
"grad_norm": 0.2529114798782207,
|
|
"kl": 0.031982421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0119,
|
|
"reward": 0.6607143059372902,
|
|
"reward_std": 0.09752393420785666,
|
|
"rewards/accuracy_reward": 0.6607143059372902,
|
|
"step": 462
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.6183166503906,
|
|
"epoch": 2.3613231552162848,
|
|
"grad_norm": 0.28781993386561655,
|
|
"kl": 0.036651611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.7500000447034836,
|
|
"reward_std": 0.0739385262131691,
|
|
"rewards/accuracy_reward": 0.7500000447034836,
|
|
"step": 463
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.93975830078125,
|
|
"epoch": 2.366412213740458,
|
|
"grad_norm": 0.25389227469639425,
|
|
"kl": 0.0308837890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.05684161093086004,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 464
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.21653747558594,
|
|
"epoch": 2.371501272264631,
|
|
"grad_norm": 0.4598666607693008,
|
|
"kl": 0.03839111328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0082,
|
|
"reward": 0.620535746216774,
|
|
"reward_std": 0.17757731303572655,
|
|
"rewards/accuracy_reward": 0.620535746216774,
|
|
"step": 465
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.4241256713867,
|
|
"epoch": 2.376590330788804,
|
|
"grad_norm": 0.24305152529088678,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.005,
|
|
"reward": 0.770089328289032,
|
|
"reward_std": 0.04526757914572954,
|
|
"rewards/accuracy_reward": 0.770089328289032,
|
|
"step": 466
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.6741256713867,
|
|
"epoch": 2.381679389312977,
|
|
"grad_norm": 0.25200191074624484,
|
|
"kl": 0.035003662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0083,
|
|
"reward": 0.6227678805589676,
|
|
"reward_std": 0.05827951803803444,
|
|
"rewards/accuracy_reward": 0.6227678805589676,
|
|
"step": 467
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.37501525878906,
|
|
"epoch": 2.38676844783715,
|
|
"grad_norm": 0.2410195169582811,
|
|
"kl": 0.033538818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.7790178954601288,
|
|
"reward_std": 0.0782818403095007,
|
|
"rewards/accuracy_reward": 0.7790178954601288,
|
|
"step": 468
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.60939025878906,
|
|
"epoch": 2.391857506361323,
|
|
"grad_norm": 0.2660262521334539,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0117,
|
|
"reward": 0.7589286118745804,
|
|
"reward_std": 0.07602170202881098,
|
|
"rewards/accuracy_reward": 0.7589286118745804,
|
|
"step": 469
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.2210006713867,
|
|
"epoch": 2.3969465648854964,
|
|
"grad_norm": 0.38122609024199494,
|
|
"kl": 0.034332275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.7433035969734192,
|
|
"reward_std": 0.14660858921706676,
|
|
"rewards/accuracy_reward": 0.7433035969734192,
|
|
"step": 470
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 435.5268096923828,
|
|
"epoch": 2.4020356234096694,
|
|
"grad_norm": 0.23998608078259098,
|
|
"kl": 0.035552978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.6540178805589676,
|
|
"reward_std": 0.0509259719401598,
|
|
"rewards/accuracy_reward": 0.6540178805589676,
|
|
"step": 471
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.6919860839844,
|
|
"epoch": 2.4071246819338423,
|
|
"grad_norm": 0.3133690114987392,
|
|
"kl": 0.03448486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0145,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.13319170102477074,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 472
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.46207427978516,
|
|
"epoch": 2.4122137404580153,
|
|
"grad_norm": 0.2293450539149541,
|
|
"kl": 0.03485107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0026,
|
|
"reward": 0.848214328289032,
|
|
"reward_std": 0.05338135547935963,
|
|
"rewards/accuracy_reward": 0.848214328289032,
|
|
"step": 473
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 378.0669860839844,
|
|
"epoch": 2.4173027989821882,
|
|
"grad_norm": 0.28646665317742065,
|
|
"kl": 0.04327392578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.6473214477300644,
|
|
"reward_std": 0.09079350251704454,
|
|
"rewards/accuracy_reward": 0.6473214477300644,
|
|
"step": 474
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.1138610839844,
|
|
"epoch": 2.422391857506361,
|
|
"grad_norm": 0.24344233501139234,
|
|
"kl": 0.035247802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0095,
|
|
"reward": 0.8816964626312256,
|
|
"reward_std": 0.07695145718753338,
|
|
"rewards/accuracy_reward": 0.8816964626312256,
|
|
"step": 475
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.2812728881836,
|
|
"epoch": 2.427480916030534,
|
|
"grad_norm": 0.19091344253106,
|
|
"kl": 0.0390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.823660746216774,
|
|
"reward_std": 0.04893965553492308,
|
|
"rewards/accuracy_reward": 0.823660746216774,
|
|
"step": 476
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.5424270629883,
|
|
"epoch": 2.432569974554707,
|
|
"grad_norm": 0.2162042112233782,
|
|
"kl": 0.034881591796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0058,
|
|
"reward": 0.7656250298023224,
|
|
"reward_std": 0.05684161186218262,
|
|
"rewards/accuracy_reward": 0.7656250298023224,
|
|
"step": 477
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.49778747558594,
|
|
"epoch": 2.4376590330788805,
|
|
"grad_norm": 0.2279534507206969,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0038,
|
|
"reward": 0.7232143133878708,
|
|
"reward_std": 0.07123849913477898,
|
|
"rewards/accuracy_reward": 0.7232143133878708,
|
|
"step": 478
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.0960006713867,
|
|
"epoch": 2.4427480916030535,
|
|
"grad_norm": 0.3931866378264231,
|
|
"kl": 0.041046142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.6450893133878708,
|
|
"reward_std": 0.08632101211696863,
|
|
"rewards/accuracy_reward": 0.6450893133878708,
|
|
"step": 479
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.54466247558594,
|
|
"epoch": 2.4478371501272265,
|
|
"grad_norm": 0.3263765017270065,
|
|
"kl": 0.037872314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.7299107611179352,
|
|
"reward_std": 0.09125630278140306,
|
|
"rewards/accuracy_reward": 0.7299107611179352,
|
|
"step": 480
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.89064025878906,
|
|
"epoch": 2.4529262086513994,
|
|
"grad_norm": 0.37816173334724634,
|
|
"kl": 0.03729248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.5312500223517418,
|
|
"reward_std": 0.18900035321712494,
|
|
"rewards/accuracy_reward": 0.5312500223517418,
|
|
"step": 481
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.5089569091797,
|
|
"epoch": 2.4580152671755724,
|
|
"grad_norm": 0.31647115041719376,
|
|
"kl": 0.0355224609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.6406250298023224,
|
|
"reward_std": 0.13283337000757456,
|
|
"rewards/accuracy_reward": 0.6406250298023224,
|
|
"step": 482
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.91297149658203,
|
|
"epoch": 2.4631043256997454,
|
|
"grad_norm": 0.44899756852157036,
|
|
"kl": 0.0477294921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0079,
|
|
"reward": 0.8258928805589676,
|
|
"reward_std": 0.09242038615047932,
|
|
"rewards/accuracy_reward": 0.8258928805589676,
|
|
"step": 483
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 456.3527069091797,
|
|
"epoch": 2.4681933842239188,
|
|
"grad_norm": 0.3007865981492443,
|
|
"kl": 0.03271484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0084,
|
|
"reward": 0.776785746216774,
|
|
"reward_std": 0.1100874561816454,
|
|
"rewards/accuracy_reward": 0.776785746216774,
|
|
"step": 484
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.5625228881836,
|
|
"epoch": 2.4732824427480917,
|
|
"grad_norm": 0.3841981197397463,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.7053571715950966,
|
|
"reward_std": 0.14869811479002237,
|
|
"rewards/accuracy_reward": 0.7053571715950966,
|
|
"step": 485
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.3526916503906,
|
|
"epoch": 2.4783715012722647,
|
|
"grad_norm": 0.21849881479643907,
|
|
"kl": 0.0345458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0036,
|
|
"reward": 0.6629464626312256,
|
|
"reward_std": 0.0834527313709259,
|
|
"rewards/accuracy_reward": 0.6629464626312256,
|
|
"step": 486
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 371.7410888671875,
|
|
"epoch": 2.4834605597964376,
|
|
"grad_norm": 0.320815641821296,
|
|
"kl": 0.04010009765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.017,
|
|
"reward": 0.7187500447034836,
|
|
"reward_std": 0.12377672828733921,
|
|
"rewards/accuracy_reward": 0.7187500447034836,
|
|
"step": 487
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.64733123779297,
|
|
"epoch": 2.4885496183206106,
|
|
"grad_norm": 0.295315154149744,
|
|
"kl": 0.038330078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.5892857387661934,
|
|
"reward_std": 0.1196470819413662,
|
|
"rewards/accuracy_reward": 0.5892857387661934,
|
|
"step": 488
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.5044860839844,
|
|
"epoch": 2.4936386768447836,
|
|
"grad_norm": 0.30493045133475927,
|
|
"kl": 0.0389404296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.707589328289032,
|
|
"reward_std": 0.09578065201640129,
|
|
"rewards/accuracy_reward": 0.707589328289032,
|
|
"step": 489
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.64288330078125,
|
|
"epoch": 2.4987277353689565,
|
|
"grad_norm": 0.36220255206982693,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0107,
|
|
"reward": 0.593750037252903,
|
|
"reward_std": 0.10374343954026699,
|
|
"rewards/accuracy_reward": 0.593750037252903,
|
|
"step": 490
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.23439025878906,
|
|
"epoch": 2.5038167938931295,
|
|
"grad_norm": 0.2994940841618887,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0056,
|
|
"reward": 0.7455357611179352,
|
|
"reward_std": 0.12250707671046257,
|
|
"rewards/accuracy_reward": 0.7455357611179352,
|
|
"step": 491
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.5736846923828,
|
|
"epoch": 2.508905852417303,
|
|
"grad_norm": 0.3568713253372041,
|
|
"kl": 0.0406494140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0119,
|
|
"reward": 0.553571455180645,
|
|
"reward_std": 0.14193019922822714,
|
|
"rewards/accuracy_reward": 0.553571455180645,
|
|
"step": 492
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 438.1428680419922,
|
|
"epoch": 2.513994910941476,
|
|
"grad_norm": 0.31931457684348474,
|
|
"kl": 0.03009033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0032,
|
|
"reward": 0.6830357313156128,
|
|
"reward_std": 0.12449068203568459,
|
|
"rewards/accuracy_reward": 0.6830357313156128,
|
|
"step": 493
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.0402069091797,
|
|
"epoch": 2.519083969465649,
|
|
"grad_norm": 0.3240174542764656,
|
|
"kl": 0.03570556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.7522321790456772,
|
|
"reward_std": 0.12789164297282696,
|
|
"rewards/accuracy_reward": 0.7522321790456772,
|
|
"step": 494
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.61385345458984,
|
|
"epoch": 2.524173027989822,
|
|
"grad_norm": 0.3173437085118972,
|
|
"kl": 0.04254150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0039,
|
|
"reward": 0.7232143133878708,
|
|
"reward_std": 0.10991919878870249,
|
|
"rewards/accuracy_reward": 0.7232143133878708,
|
|
"step": 495
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.8303756713867,
|
|
"epoch": 2.5292620865139948,
|
|
"grad_norm": 0.36127386983772714,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0017,
|
|
"reward": 0.7098214700818062,
|
|
"reward_std": 0.12185426987707615,
|
|
"rewards/accuracy_reward": 0.7098214700818062,
|
|
"step": 496
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.06697845458984,
|
|
"epoch": 2.534351145038168,
|
|
"grad_norm": 0.36061772993304764,
|
|
"kl": 0.034820556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0056,
|
|
"reward": 0.754464328289032,
|
|
"reward_std": 0.15109536796808243,
|
|
"rewards/accuracy_reward": 0.754464328289032,
|
|
"step": 497
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.91966247558594,
|
|
"epoch": 2.539440203562341,
|
|
"grad_norm": 0.39722756084611466,
|
|
"kl": 0.0340576171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.002,
|
|
"reward": 0.6562500447034836,
|
|
"reward_std": 0.1960136517882347,
|
|
"rewards/accuracy_reward": 0.6562500447034836,
|
|
"step": 498
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.82592010498047,
|
|
"epoch": 2.544529262086514,
|
|
"grad_norm": 0.31069158155189774,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.6093750298023224,
|
|
"reward_std": 0.10550853796303272,
|
|
"rewards/accuracy_reward": 0.6093750298023224,
|
|
"step": 499
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.2321548461914,
|
|
"epoch": 2.549618320610687,
|
|
"grad_norm": 0.4584225175807347,
|
|
"kl": 0.045654296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.7433036118745804,
|
|
"reward_std": 0.12795093655586243,
|
|
"rewards/accuracy_reward": 0.7433036118745804,
|
|
"step": 500
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.7857360839844,
|
|
"epoch": 2.55470737913486,
|
|
"grad_norm": 0.28617880733955425,
|
|
"kl": 0.033416748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0143,
|
|
"reward": 0.6674107387661934,
|
|
"reward_std": 0.11140253208577633,
|
|
"rewards/accuracy_reward": 0.6674107387661934,
|
|
"step": 501
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.3281478881836,
|
|
"epoch": 2.559796437659033,
|
|
"grad_norm": 0.3015933954740273,
|
|
"kl": 0.032470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0071,
|
|
"reward": 0.7254464626312256,
|
|
"reward_std": 0.13475124537944794,
|
|
"rewards/accuracy_reward": 0.7254464626312256,
|
|
"step": 502
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 378.11609649658203,
|
|
"epoch": 2.564885496183206,
|
|
"grad_norm": 0.26912659206068457,
|
|
"kl": 0.033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.7321428954601288,
|
|
"reward_std": 0.07387762051075697,
|
|
"rewards/accuracy_reward": 0.7321428954601288,
|
|
"step": 503
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.60939025878906,
|
|
"epoch": 2.569974554707379,
|
|
"grad_norm": 0.27967517952321175,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.7857143133878708,
|
|
"reward_std": 0.09173476323485374,
|
|
"rewards/accuracy_reward": 0.7857143133878708,
|
|
"step": 504
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.6250228881836,
|
|
"epoch": 2.5750636132315523,
|
|
"grad_norm": 0.22205355273321656,
|
|
"kl": 0.031585693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.7343750298023224,
|
|
"reward_std": 0.08394023403525352,
|
|
"rewards/accuracy_reward": 0.7343750298023224,
|
|
"step": 505
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.39510345458984,
|
|
"epoch": 2.5801526717557253,
|
|
"grad_norm": 0.2898121744066452,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.620535746216774,
|
|
"reward_std": 0.10770668089389801,
|
|
"rewards/accuracy_reward": 0.620535746216774,
|
|
"step": 506
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.64510345458984,
|
|
"epoch": 2.5852417302798982,
|
|
"grad_norm": 0.3362396947209023,
|
|
"kl": 0.036376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0043,
|
|
"reward": 0.7968750298023224,
|
|
"reward_std": 0.16273310966789722,
|
|
"rewards/accuracy_reward": 0.7968750298023224,
|
|
"step": 507
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.9218978881836,
|
|
"epoch": 2.590330788804071,
|
|
"grad_norm": 0.23902970790337014,
|
|
"kl": 0.030181884765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0054,
|
|
"reward": 0.7366071790456772,
|
|
"reward_std": 0.07940684352070093,
|
|
"rewards/accuracy_reward": 0.7366071790456772,
|
|
"step": 508
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.7544860839844,
|
|
"epoch": 2.595419847328244,
|
|
"grad_norm": 0.29514999944220416,
|
|
"kl": 0.03887939453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0033,
|
|
"reward": 0.6584821715950966,
|
|
"reward_std": 0.07890495471656322,
|
|
"rewards/accuracy_reward": 0.6584821715950966,
|
|
"step": 509
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.8660888671875,
|
|
"epoch": 2.6005089058524176,
|
|
"grad_norm": 0.3798953413187013,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0122,
|
|
"reward": 0.6696428805589676,
|
|
"reward_std": 0.19469597935676575,
|
|
"rewards/accuracy_reward": 0.6696428805589676,
|
|
"step": 510
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.5000228881836,
|
|
"epoch": 2.6055979643765905,
|
|
"grad_norm": 0.33955130967501823,
|
|
"kl": 0.03558349609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0025,
|
|
"reward": 0.5491071715950966,
|
|
"reward_std": 0.12097746971994638,
|
|
"rewards/accuracy_reward": 0.5491071715950966,
|
|
"step": 511
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.5692138671875,
|
|
"epoch": 2.6106870229007635,
|
|
"grad_norm": 0.24669263696065316,
|
|
"kl": 0.0352783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.6674107387661934,
|
|
"reward_std": 0.08934654574841261,
|
|
"rewards/accuracy_reward": 0.6674107387661934,
|
|
"step": 512
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.66519927978516,
|
|
"epoch": 2.6157760814249365,
|
|
"grad_norm": 0.2859664489787216,
|
|
"kl": 0.0347900390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0009,
|
|
"reward": 0.7209821790456772,
|
|
"reward_std": 0.10619255620986223,
|
|
"rewards/accuracy_reward": 0.7209821790456772,
|
|
"step": 513
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.8549270629883,
|
|
"epoch": 2.6208651399491094,
|
|
"grad_norm": 0.2572719508539601,
|
|
"kl": 0.03143310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.7790178954601288,
|
|
"reward_std": 0.12461351975798607,
|
|
"rewards/accuracy_reward": 0.7790178954601288,
|
|
"step": 514
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 388.76341247558594,
|
|
"epoch": 2.6259541984732824,
|
|
"grad_norm": 0.3449533451220129,
|
|
"kl": 0.03955078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.7544643133878708,
|
|
"reward_std": 0.10766920261085033,
|
|
"rewards/accuracy_reward": 0.7544643133878708,
|
|
"step": 515
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.7835006713867,
|
|
"epoch": 2.6310432569974553,
|
|
"grad_norm": 0.632385436314343,
|
|
"kl": 0.032257080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0098,
|
|
"reward": 0.8058035969734192,
|
|
"reward_std": 0.13097807671874762,
|
|
"rewards/accuracy_reward": 0.8058035969734192,
|
|
"step": 516
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.59154510498047,
|
|
"epoch": 2.6361323155216283,
|
|
"grad_norm": 0.33196555497200964,
|
|
"kl": 0.0389404296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0069,
|
|
"reward": 0.587053582072258,
|
|
"reward_std": 0.11103776469826698,
|
|
"rewards/accuracy_reward": 0.587053582072258,
|
|
"step": 517
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.5513610839844,
|
|
"epoch": 2.6412213740458013,
|
|
"grad_norm": 0.3451468883461782,
|
|
"kl": 0.03460693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0149,
|
|
"reward": 0.5937500149011612,
|
|
"reward_std": 0.14383834693580866,
|
|
"rewards/accuracy_reward": 0.5937500149011612,
|
|
"step": 518
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 451.51788330078125,
|
|
"epoch": 2.6463104325699747,
|
|
"grad_norm": 0.28800529153503623,
|
|
"kl": 0.034454345703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.5959821790456772,
|
|
"reward_std": 0.12700967863202095,
|
|
"rewards/accuracy_reward": 0.5959821790456772,
|
|
"step": 519
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.31251525878906,
|
|
"epoch": 2.6513994910941476,
|
|
"grad_norm": 0.25559387172202164,
|
|
"kl": 0.033294677734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.6339285969734192,
|
|
"reward_std": 0.11485719867050648,
|
|
"rewards/accuracy_reward": 0.6339285969734192,
|
|
"step": 520
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.74109649658203,
|
|
"epoch": 2.6564885496183206,
|
|
"grad_norm": 0.2536722799441574,
|
|
"kl": 0.03643798828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0132,
|
|
"reward": 0.4910714328289032,
|
|
"reward_std": 0.08318000193685293,
|
|
"rewards/accuracy_reward": 0.4910714328289032,
|
|
"step": 521
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.70983123779297,
|
|
"epoch": 2.6615776081424936,
|
|
"grad_norm": 0.2221512357164997,
|
|
"kl": 0.03692626953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.683035746216774,
|
|
"reward_std": 0.07241806481033564,
|
|
"rewards/accuracy_reward": 0.683035746216774,
|
|
"step": 522
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.91297149658203,
|
|
"epoch": 2.6666666666666665,
|
|
"grad_norm": 0.3046176456837757,
|
|
"kl": 0.033477783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0055,
|
|
"reward": 0.7321428805589676,
|
|
"reward_std": 0.0882824556902051,
|
|
"rewards/accuracy_reward": 0.7321428805589676,
|
|
"step": 523
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 372.9174270629883,
|
|
"epoch": 2.67175572519084,
|
|
"grad_norm": 0.2409606452831004,
|
|
"kl": 0.04193115234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.004,
|
|
"reward": 0.7098214477300644,
|
|
"reward_std": 0.04791303910315037,
|
|
"rewards/accuracy_reward": 0.7098214477300644,
|
|
"step": 524
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.5781478881836,
|
|
"epoch": 2.676844783715013,
|
|
"grad_norm": 0.27818205005551117,
|
|
"kl": 0.03411865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0025,
|
|
"reward": 0.6294643133878708,
|
|
"reward_std": 0.07959691435098648,
|
|
"rewards/accuracy_reward": 0.6294643133878708,
|
|
"step": 525
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.12278747558594,
|
|
"epoch": 2.681933842239186,
|
|
"grad_norm": 0.3064833397677243,
|
|
"kl": 0.04248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.595982164144516,
|
|
"reward_std": 0.08645018469542265,
|
|
"rewards/accuracy_reward": 0.595982164144516,
|
|
"step": 526
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.58707427978516,
|
|
"epoch": 2.687022900763359,
|
|
"grad_norm": 0.1439587126531661,
|
|
"kl": 0.037139892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0036,
|
|
"reward": 0.6718750298023224,
|
|
"reward_std": 0.029295665211975574,
|
|
"rewards/accuracy_reward": 0.6718750298023224,
|
|
"step": 527
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.4710006713867,
|
|
"epoch": 2.6921119592875318,
|
|
"grad_norm": 0.19355861305380997,
|
|
"kl": 0.036376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.008,
|
|
"reward": 0.6785714700818062,
|
|
"reward_std": 0.05042298696935177,
|
|
"rewards/accuracy_reward": 0.6785714700818062,
|
|
"step": 528
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.09153747558594,
|
|
"epoch": 2.6972010178117047,
|
|
"grad_norm": 0.2527678447303911,
|
|
"kl": 0.03192138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.7053571939468384,
|
|
"reward_std": 0.06765811704099178,
|
|
"rewards/accuracy_reward": 0.7053571939468384,
|
|
"step": 529
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.0491256713867,
|
|
"epoch": 2.7022900763358777,
|
|
"grad_norm": 0.331200966018847,
|
|
"kl": 0.03302001953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.6852678880095482,
|
|
"reward_std": 0.09613898955285549,
|
|
"rewards/accuracy_reward": 0.6852678880095482,
|
|
"step": 530
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.2701110839844,
|
|
"epoch": 2.7073791348600507,
|
|
"grad_norm": 0.29291114397156326,
|
|
"kl": 0.03558349609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.6071428954601288,
|
|
"reward_std": 0.11480188090354204,
|
|
"rewards/accuracy_reward": 0.6071428954601288,
|
|
"step": 531
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.6607360839844,
|
|
"epoch": 2.712468193384224,
|
|
"grad_norm": 0.31804500261647317,
|
|
"kl": 0.03912353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.009,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.1205065455287695,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 532
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.7589416503906,
|
|
"epoch": 2.717557251908397,
|
|
"grad_norm": 0.26914663178766046,
|
|
"kl": 0.040771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0155,
|
|
"reward": 0.676339328289032,
|
|
"reward_std": 0.07469875272363424,
|
|
"rewards/accuracy_reward": 0.676339328289032,
|
|
"step": 533
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.51563262939453,
|
|
"epoch": 2.72264631043257,
|
|
"grad_norm": 0.36988046687006165,
|
|
"kl": 0.04022216796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0112,
|
|
"reward": 0.6517857387661934,
|
|
"reward_std": 0.14681287854909897,
|
|
"rewards/accuracy_reward": 0.6517857387661934,
|
|
"step": 534
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.5245666503906,
|
|
"epoch": 2.727735368956743,
|
|
"grad_norm": 0.3936767345121069,
|
|
"kl": 0.03948974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.009,
|
|
"reward": 0.7544643133878708,
|
|
"reward_std": 0.1561136618256569,
|
|
"rewards/accuracy_reward": 0.7544643133878708,
|
|
"step": 535
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.8482360839844,
|
|
"epoch": 2.732824427480916,
|
|
"grad_norm": 0.24488324139826242,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.013,
|
|
"reward": 0.6562500223517418,
|
|
"reward_std": 0.07999247591942549,
|
|
"rewards/accuracy_reward": 0.6562500223517418,
|
|
"step": 536
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.2924270629883,
|
|
"epoch": 2.7379134860050893,
|
|
"grad_norm": 0.09820479925311285,
|
|
"kl": 0.034423828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.8125000298023224,
|
|
"reward_std": 0.015971913933753967,
|
|
"rewards/accuracy_reward": 0.8125000298023224,
|
|
"step": 537
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.64064025878906,
|
|
"epoch": 2.7430025445292623,
|
|
"grad_norm": 0.286300366081529,
|
|
"kl": 0.03802490234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.816964328289032,
|
|
"reward_std": 0.09291432984173298,
|
|
"rewards/accuracy_reward": 0.816964328289032,
|
|
"step": 538
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.6518020629883,
|
|
"epoch": 2.7480916030534353,
|
|
"grad_norm": 0.3399892125694863,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0024,
|
|
"reward": 0.636160746216774,
|
|
"reward_std": 0.11417082604020834,
|
|
"rewards/accuracy_reward": 0.636160746216774,
|
|
"step": 539
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.64510345458984,
|
|
"epoch": 2.753180661577608,
|
|
"grad_norm": 0.2047451118377167,
|
|
"kl": 0.0401611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0034,
|
|
"reward": 0.7566964402794838,
|
|
"reward_std": 0.07594658620655537,
|
|
"rewards/accuracy_reward": 0.7566964402794838,
|
|
"step": 540
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.5424270629883,
|
|
"epoch": 2.758269720101781,
|
|
"grad_norm": 0.20144452404013666,
|
|
"kl": 0.0401611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0043,
|
|
"reward": 0.645089328289032,
|
|
"reward_std": 0.058865154162049294,
|
|
"rewards/accuracy_reward": 0.645089328289032,
|
|
"step": 541
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.56029510498047,
|
|
"epoch": 2.763358778625954,
|
|
"grad_norm": 0.2343635165306646,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.6651786118745804,
|
|
"reward_std": 0.08738510962575674,
|
|
"rewards/accuracy_reward": 0.6651786118745804,
|
|
"step": 542
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.4085006713867,
|
|
"epoch": 2.768447837150127,
|
|
"grad_norm": 0.3373893144181298,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.6584821790456772,
|
|
"reward_std": 0.13745236676186323,
|
|
"rewards/accuracy_reward": 0.6584821790456772,
|
|
"step": 543
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.9843978881836,
|
|
"epoch": 2.7735368956743,
|
|
"grad_norm": 0.3204217362971684,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0091,
|
|
"reward": 0.660714328289032,
|
|
"reward_std": 0.13499699160456657,
|
|
"rewards/accuracy_reward": 0.660714328289032,
|
|
"step": 544
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.93751525878906,
|
|
"epoch": 2.778625954198473,
|
|
"grad_norm": 0.3300048793679498,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.6830357611179352,
|
|
"reward_std": 0.14531533233821392,
|
|
"rewards/accuracy_reward": 0.6830357611179352,
|
|
"step": 545
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.7366256713867,
|
|
"epoch": 2.7837150127226464,
|
|
"grad_norm": 0.2780528336605366,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.6517857536673546,
|
|
"reward_std": 0.07582645677030087,
|
|
"rewards/accuracy_reward": 0.6517857536673546,
|
|
"step": 546
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.80582427978516,
|
|
"epoch": 2.7888040712468194,
|
|
"grad_norm": 0.21568216254912498,
|
|
"kl": 0.033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.05859133042395115,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 547
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.2299270629883,
|
|
"epoch": 2.7938931297709924,
|
|
"grad_norm": 0.3650557249780628,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.8370536118745804,
|
|
"reward_std": 0.1332291793078184,
|
|
"rewards/accuracy_reward": 0.8370536118745804,
|
|
"step": 548
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.74332427978516,
|
|
"epoch": 2.7989821882951653,
|
|
"grad_norm": 0.284772071971351,
|
|
"kl": 0.03265380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0075,
|
|
"reward": 0.7745535969734192,
|
|
"reward_std": 0.07155140861868858,
|
|
"rewards/accuracy_reward": 0.7745535969734192,
|
|
"step": 549
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.8884048461914,
|
|
"epoch": 2.8040712468193383,
|
|
"grad_norm": 0.377128397877783,
|
|
"kl": 0.034881591796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0067,
|
|
"reward": 0.6540178805589676,
|
|
"reward_std": 0.1443558856844902,
|
|
"rewards/accuracy_reward": 0.6540178805589676,
|
|
"step": 550
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.68975830078125,
|
|
"epoch": 2.8091603053435117,
|
|
"grad_norm": 0.2440057694514428,
|
|
"kl": 0.04638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0098,
|
|
"reward": 0.7723214477300644,
|
|
"reward_std": 0.06154969800263643,
|
|
"rewards/accuracy_reward": 0.7723214477300644,
|
|
"step": 551
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.14957427978516,
|
|
"epoch": 2.8142493638676847,
|
|
"grad_norm": 0.30153622558736576,
|
|
"kl": 0.035064697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.6852678954601288,
|
|
"reward_std": 0.08264064136892557,
|
|
"rewards/accuracy_reward": 0.6852678954601288,
|
|
"step": 552
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 379.56028747558594,
|
|
"epoch": 2.8193384223918576,
|
|
"grad_norm": 0.337655409367331,
|
|
"kl": 0.04473876953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.5915178954601288,
|
|
"reward_std": 0.10053416807204485,
|
|
"rewards/accuracy_reward": 0.5915178954601288,
|
|
"step": 553
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.39510345458984,
|
|
"epoch": 2.8244274809160306,
|
|
"grad_norm": 0.30254905149794203,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.7834821939468384,
|
|
"reward_std": 0.10977564752101898,
|
|
"rewards/accuracy_reward": 0.7834821939468384,
|
|
"step": 554
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.9397506713867,
|
|
"epoch": 2.8295165394402035,
|
|
"grad_norm": 0.2980175473217978,
|
|
"kl": 0.033172607421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.6026786118745804,
|
|
"reward_std": 0.10644346475601196,
|
|
"rewards/accuracy_reward": 0.6026786118745804,
|
|
"step": 555
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.2656478881836,
|
|
"epoch": 2.8346055979643765,
|
|
"grad_norm": 0.30986857343738305,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.651785746216774,
|
|
"reward_std": 0.11615678668022156,
|
|
"rewards/accuracy_reward": 0.651785746216774,
|
|
"step": 556
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.85491943359375,
|
|
"epoch": 2.8396946564885495,
|
|
"grad_norm": 0.4046743261194774,
|
|
"kl": 0.03515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0084,
|
|
"reward": 0.6383928880095482,
|
|
"reward_std": 0.16278479434549809,
|
|
"rewards/accuracy_reward": 0.6383928880095482,
|
|
"step": 557
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.50447845458984,
|
|
"epoch": 2.8447837150127224,
|
|
"grad_norm": 0.33955198158693556,
|
|
"kl": 0.031982421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.6406250223517418,
|
|
"reward_std": 0.14977919682860374,
|
|
"rewards/accuracy_reward": 0.6406250223517418,
|
|
"step": 558
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 445.9486770629883,
|
|
"epoch": 2.849872773536896,
|
|
"grad_norm": 0.5235292122720531,
|
|
"kl": 0.04193115234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.005,
|
|
"reward": 0.6785714477300644,
|
|
"reward_std": 0.13369468972086906,
|
|
"rewards/accuracy_reward": 0.6785714477300644,
|
|
"step": 559
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.21653747558594,
|
|
"epoch": 2.854961832061069,
|
|
"grad_norm": 0.3647830766982642,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.5580357313156128,
|
|
"reward_std": 0.09375077206641436,
|
|
"rewards/accuracy_reward": 0.5580357313156128,
|
|
"step": 560
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.5781478881836,
|
|
"epoch": 2.8600508905852418,
|
|
"grad_norm": 0.2915846428107292,
|
|
"kl": 0.03924560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0101,
|
|
"reward": 0.7500000447034836,
|
|
"reward_std": 0.09638989623636007,
|
|
"rewards/accuracy_reward": 0.7500000447034836,
|
|
"step": 561
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.76341247558594,
|
|
"epoch": 2.8651399491094147,
|
|
"grad_norm": 2.5118673372186335,
|
|
"kl": 0.06280517578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.74776791036129,
|
|
"reward_std": 0.14617369696497917,
|
|
"rewards/accuracy_reward": 0.74776791036129,
|
|
"step": 562
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.06922149658203,
|
|
"epoch": 2.8702290076335877,
|
|
"grad_norm": 0.3476480288065041,
|
|
"kl": 0.031097412109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.631696455180645,
|
|
"reward_std": 0.1722291260957718,
|
|
"rewards/accuracy_reward": 0.631696455180645,
|
|
"step": 563
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.5580520629883,
|
|
"epoch": 2.875318066157761,
|
|
"grad_norm": 0.317993366086719,
|
|
"kl": 0.04376220703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0116,
|
|
"reward": 0.5178571715950966,
|
|
"reward_std": 0.09537875838577747,
|
|
"rewards/accuracy_reward": 0.5178571715950966,
|
|
"step": 564
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.98216247558594,
|
|
"epoch": 2.880407124681934,
|
|
"grad_norm": 0.2902885029430109,
|
|
"kl": 0.036529541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0118,
|
|
"reward": 0.6250000298023224,
|
|
"reward_std": 0.07891290169209242,
|
|
"rewards/accuracy_reward": 0.6250000298023224,
|
|
"step": 565
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.84376525878906,
|
|
"epoch": 2.885496183206107,
|
|
"grad_norm": 0.32453080996916994,
|
|
"kl": 0.03338623046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0128,
|
|
"reward": 0.7053571715950966,
|
|
"reward_std": 0.12533627077937126,
|
|
"rewards/accuracy_reward": 0.7053571715950966,
|
|
"step": 566
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.2991256713867,
|
|
"epoch": 2.89058524173028,
|
|
"grad_norm": 0.3087751087174545,
|
|
"kl": 0.041015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0077,
|
|
"reward": 0.5691964477300644,
|
|
"reward_std": 0.11318294238299131,
|
|
"rewards/accuracy_reward": 0.5691964477300644,
|
|
"step": 567
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.6428680419922,
|
|
"epoch": 2.895674300254453,
|
|
"grad_norm": 0.4055341932266634,
|
|
"kl": 0.04046630859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.017,
|
|
"reward": 0.6294643208384514,
|
|
"reward_std": 0.1625868445262313,
|
|
"rewards/accuracy_reward": 0.6294643208384514,
|
|
"step": 568
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.8750228881836,
|
|
"epoch": 2.900763358778626,
|
|
"grad_norm": 0.3847610873766707,
|
|
"kl": 0.036651611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0055,
|
|
"reward": 0.8102678805589676,
|
|
"reward_std": 0.2118864431977272,
|
|
"rewards/accuracy_reward": 0.8102678805589676,
|
|
"step": 569
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.62947845458984,
|
|
"epoch": 2.905852417302799,
|
|
"grad_norm": 0.41563935712517536,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.6964285969734192,
|
|
"reward_std": 0.1858063880354166,
|
|
"rewards/accuracy_reward": 0.6964285969734192,
|
|
"step": 570
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.64957427978516,
|
|
"epoch": 2.910941475826972,
|
|
"grad_norm": 0.44069690199594175,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0047,
|
|
"reward": 0.6897321790456772,
|
|
"reward_std": 0.20478853583335876,
|
|
"rewards/accuracy_reward": 0.6897321790456772,
|
|
"step": 571
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 373.8035888671875,
|
|
"epoch": 2.916030534351145,
|
|
"grad_norm": 0.3411320971221695,
|
|
"kl": 0.035247802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0237,
|
|
"reward": 0.7745535969734192,
|
|
"reward_std": 0.12431609351187944,
|
|
"rewards/accuracy_reward": 0.7745535969734192,
|
|
"step": 572
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.82814025878906,
|
|
"epoch": 2.921119592875318,
|
|
"grad_norm": 0.46650189630733896,
|
|
"kl": 0.04046630859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.6852678805589676,
|
|
"reward_std": 0.2135436087846756,
|
|
"rewards/accuracy_reward": 0.6852678805589676,
|
|
"step": 573
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.54689025878906,
|
|
"epoch": 2.926208651399491,
|
|
"grad_norm": 0.3633625043570078,
|
|
"kl": 0.0404052734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.6651785969734192,
|
|
"reward_std": 0.13997509516775608,
|
|
"rewards/accuracy_reward": 0.6651785969734192,
|
|
"step": 574
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.51342010498047,
|
|
"epoch": 2.931297709923664,
|
|
"grad_norm": 0.3486662295054925,
|
|
"kl": 0.03570556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.6651786118745804,
|
|
"reward_std": 0.1588444784283638,
|
|
"rewards/accuracy_reward": 0.6651786118745804,
|
|
"step": 575
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.35939025878906,
|
|
"epoch": 2.936386768447837,
|
|
"grad_norm": 0.2602516130746799,
|
|
"kl": 0.03900146484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.6875000298023224,
|
|
"reward_std": 0.0915446886792779,
|
|
"rewards/accuracy_reward": 0.6875000298023224,
|
|
"step": 576
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 444.9620666503906,
|
|
"epoch": 2.94147582697201,
|
|
"grad_norm": 0.35960628442279924,
|
|
"kl": 0.034393310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.7388393133878708,
|
|
"reward_std": 0.1947804857045412,
|
|
"rewards/accuracy_reward": 0.7388393133878708,
|
|
"step": 577
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.1986770629883,
|
|
"epoch": 2.9465648854961835,
|
|
"grad_norm": 0.374525516444503,
|
|
"kl": 0.030975341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.6227678880095482,
|
|
"reward_std": 0.17119617387652397,
|
|
"rewards/accuracy_reward": 0.6227678880095482,
|
|
"step": 578
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.1183319091797,
|
|
"epoch": 2.9516539440203564,
|
|
"grad_norm": 0.2972914056458539,
|
|
"kl": 0.03167724609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.6495536118745804,
|
|
"reward_std": 0.10004776064306498,
|
|
"rewards/accuracy_reward": 0.6495536118745804,
|
|
"step": 579
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.02234649658203,
|
|
"epoch": 2.9567430025445294,
|
|
"grad_norm": 0.2602697339274284,
|
|
"kl": 0.0377197265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0121,
|
|
"reward": 0.5669643208384514,
|
|
"reward_std": 0.08922642096877098,
|
|
"rewards/accuracy_reward": 0.5669643208384514,
|
|
"step": 580
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.30358123779297,
|
|
"epoch": 2.9618320610687023,
|
|
"grad_norm": 0.2904891691402829,
|
|
"kl": 0.03411865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.8035714775323868,
|
|
"reward_std": 0.09128634724766016,
|
|
"rewards/accuracy_reward": 0.8035714775323868,
|
|
"step": 581
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.07814025878906,
|
|
"epoch": 2.9669211195928753,
|
|
"grad_norm": 0.3715192106038954,
|
|
"kl": 0.03704833984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0121,
|
|
"reward": 0.5825893059372902,
|
|
"reward_std": 0.14649997185915709,
|
|
"rewards/accuracy_reward": 0.5825893059372902,
|
|
"step": 582
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.96207427978516,
|
|
"epoch": 2.9720101781170483,
|
|
"grad_norm": 0.37146966248031055,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.7187500298023224,
|
|
"reward_std": 0.18226324021816254,
|
|
"rewards/accuracy_reward": 0.7187500298023224,
|
|
"step": 583
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.5424270629883,
|
|
"epoch": 2.9770992366412212,
|
|
"grad_norm": 0.3870491567373662,
|
|
"kl": 0.034027099609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0108,
|
|
"reward": 0.5758928880095482,
|
|
"reward_std": 0.1732867918908596,
|
|
"rewards/accuracy_reward": 0.5758928880095482,
|
|
"step": 584
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.3437728881836,
|
|
"epoch": 2.982188295165394,
|
|
"grad_norm": 0.33433589615968345,
|
|
"kl": 0.036346435546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0018,
|
|
"reward": 0.6651785969734192,
|
|
"reward_std": 0.11502275057137012,
|
|
"rewards/accuracy_reward": 0.6651785969734192,
|
|
"step": 585
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.1852798461914,
|
|
"epoch": 2.9872773536895676,
|
|
"grad_norm": 0.386757527349345,
|
|
"kl": 0.03863525390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.7388393133878708,
|
|
"reward_std": 0.10420894343405962,
|
|
"rewards/accuracy_reward": 0.7388393133878708,
|
|
"step": 586
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.43528747558594,
|
|
"epoch": 2.9923664122137406,
|
|
"grad_norm": 0.365220944421569,
|
|
"kl": 0.034271240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0052,
|
|
"reward": 0.6294643208384514,
|
|
"reward_std": 0.16460285522043705,
|
|
"rewards/accuracy_reward": 0.6294643208384514,
|
|
"step": 587
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 343.00001525878906,
|
|
"epoch": 2.9974554707379135,
|
|
"grad_norm": 0.40809859576380375,
|
|
"kl": 0.040771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0079,
|
|
"reward": 0.651785746216774,
|
|
"reward_std": 0.21456751227378845,
|
|
"rewards/accuracy_reward": 0.651785746216774,
|
|
"step": 588
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.8236770629883,
|
|
"epoch": 3.005089058524173,
|
|
"grad_norm": 0.5054490077348739,
|
|
"kl": 0.04595947265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.11722088698297739,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 589
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.66966247558594,
|
|
"epoch": 3.010178117048346,
|
|
"grad_norm": 0.29671769278277893,
|
|
"kl": 0.04132080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0086,
|
|
"reward": 0.8281250447034836,
|
|
"reward_std": 0.08452587015926838,
|
|
"rewards/accuracy_reward": 0.8281250447034836,
|
|
"step": 590
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.2656478881836,
|
|
"epoch": 3.015267175572519,
|
|
"grad_norm": 0.3553393765638119,
|
|
"kl": 0.034576416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0078,
|
|
"reward": 0.7053571939468384,
|
|
"reward_std": 0.11977626010775566,
|
|
"rewards/accuracy_reward": 0.7053571939468384,
|
|
"step": 591
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.9285888671875,
|
|
"epoch": 3.0203562340966923,
|
|
"grad_norm": 0.32713588955017314,
|
|
"kl": 0.038970947265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0084,
|
|
"reward": 0.645089328289032,
|
|
"reward_std": 0.07450868003070354,
|
|
"rewards/accuracy_reward": 0.645089328289032,
|
|
"step": 592
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 364.0178680419922,
|
|
"epoch": 3.0254452926208653,
|
|
"grad_norm": 0.44394381709580466,
|
|
"kl": 0.049652099609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.6718750149011612,
|
|
"reward_std": 0.14425750821828842,
|
|
"rewards/accuracy_reward": 0.6718750149011612,
|
|
"step": 593
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.5134048461914,
|
|
"epoch": 3.030534351145038,
|
|
"grad_norm": 0.33120930284551425,
|
|
"kl": 0.0418701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0054,
|
|
"reward": 0.6852678805589676,
|
|
"reward_std": 0.11204889789223671,
|
|
"rewards/accuracy_reward": 0.6852678805589676,
|
|
"step": 594
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.0535888671875,
|
|
"epoch": 3.035623409669211,
|
|
"grad_norm": 0.26027634818081746,
|
|
"kl": 0.0386962890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.770089328289032,
|
|
"reward_std": 0.0650099515914917,
|
|
"rewards/accuracy_reward": 0.770089328289032,
|
|
"step": 595
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.56028747558594,
|
|
"epoch": 3.040712468193384,
|
|
"grad_norm": 0.35247817317411145,
|
|
"kl": 0.04168701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0044,
|
|
"reward": 0.7098214775323868,
|
|
"reward_std": 0.14996132999658585,
|
|
"rewards/accuracy_reward": 0.7098214775323868,
|
|
"step": 596
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.66295623779297,
|
|
"epoch": 3.045801526717557,
|
|
"grad_norm": 0.27449168635641785,
|
|
"kl": 0.04217529296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.6830357313156128,
|
|
"reward_std": 0.08047888334840536,
|
|
"rewards/accuracy_reward": 0.6830357313156128,
|
|
"step": 597
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.22322845458984,
|
|
"epoch": 3.05089058524173,
|
|
"grad_norm": 0.31346122011455496,
|
|
"kl": 0.04010009765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.7968750298023224,
|
|
"reward_std": 0.10267934296280146,
|
|
"rewards/accuracy_reward": 0.7968750298023224,
|
|
"step": 598
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.3616256713867,
|
|
"epoch": 3.0559796437659035,
|
|
"grad_norm": 0.3091595921253676,
|
|
"kl": 0.0404052734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.6919643133878708,
|
|
"reward_std": 0.09394084382802248,
|
|
"rewards/accuracy_reward": 0.6919643133878708,
|
|
"step": 599
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 451.9375228881836,
|
|
"epoch": 3.0610687022900764,
|
|
"grad_norm": 0.2692371232740381,
|
|
"kl": 0.03826904296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.707589328289032,
|
|
"reward_std": 0.10160730499774218,
|
|
"rewards/accuracy_reward": 0.707589328289032,
|
|
"step": 600
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.5625228881836,
|
|
"epoch": 3.0661577608142494,
|
|
"grad_norm": 0.3397450164084543,
|
|
"kl": 0.0394287109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0016,
|
|
"reward": 0.7254464626312256,
|
|
"reward_std": 0.11211089976131916,
|
|
"rewards/accuracy_reward": 0.7254464626312256,
|
|
"step": 601
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.9129638671875,
|
|
"epoch": 3.0712468193384224,
|
|
"grad_norm": 0.27695285703523814,
|
|
"kl": 0.042236328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0087,
|
|
"reward": 0.6316964626312256,
|
|
"reward_std": 0.11891603656113148,
|
|
"rewards/accuracy_reward": 0.6316964626312256,
|
|
"step": 602
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.9107360839844,
|
|
"epoch": 3.0763358778625953,
|
|
"grad_norm": 0.35187548455133666,
|
|
"kl": 0.04229736328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.7991071790456772,
|
|
"reward_std": 0.14297254383563995,
|
|
"rewards/accuracy_reward": 0.7991071790456772,
|
|
"step": 603
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 449.98439025878906,
|
|
"epoch": 3.0814249363867683,
|
|
"grad_norm": 181.87528038860185,
|
|
"kl": 7.3677978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.3632,
|
|
"reward": 0.6093750298023224,
|
|
"reward_std": 0.03898446634411812,
|
|
"rewards/accuracy_reward": 0.6093750298023224,
|
|
"step": 604
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.67413330078125,
|
|
"epoch": 3.0865139949109412,
|
|
"grad_norm": 0.2238747470587776,
|
|
"kl": 0.0469970703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.8370536267757416,
|
|
"reward_std": 0.07155031058937311,
|
|
"rewards/accuracy_reward": 0.8370536267757416,
|
|
"step": 605
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.4620666503906,
|
|
"epoch": 3.0916030534351147,
|
|
"grad_norm": 0.29689243902093276,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.7968750149011612,
|
|
"reward_std": 0.09425375983119011,
|
|
"rewards/accuracy_reward": 0.7968750149011612,
|
|
"step": 606
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.0201110839844,
|
|
"epoch": 3.0966921119592876,
|
|
"grad_norm": 0.29530092564323684,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0047,
|
|
"reward": 0.6964286044239998,
|
|
"reward_std": 0.08155202306807041,
|
|
"rewards/accuracy_reward": 0.6964286044239998,
|
|
"step": 607
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.64064025878906,
|
|
"epoch": 3.1017811704834606,
|
|
"grad_norm": 0.3836516877093382,
|
|
"kl": 0.0430908203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.7901786118745804,
|
|
"reward_std": 0.15968108735978603,
|
|
"rewards/accuracy_reward": 0.7901786118745804,
|
|
"step": 608
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.1428680419922,
|
|
"epoch": 3.1068702290076335,
|
|
"grad_norm": 0.31082000201458926,
|
|
"kl": 0.044677734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0085,
|
|
"reward": 0.79464291036129,
|
|
"reward_std": 0.08997760713100433,
|
|
"rewards/accuracy_reward": 0.79464291036129,
|
|
"step": 609
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.3393096923828,
|
|
"epoch": 3.1119592875318065,
|
|
"grad_norm": 0.3296703166146194,
|
|
"kl": 0.0462646484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.6406250298023224,
|
|
"reward_std": 0.12349375803023577,
|
|
"rewards/accuracy_reward": 0.6406250298023224,
|
|
"step": 610
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.76341247558594,
|
|
"epoch": 3.1170483460559795,
|
|
"grad_norm": 0.27837557288999537,
|
|
"kl": 0.03546142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.816964328289032,
|
|
"reward_std": 0.09958386048674583,
|
|
"rewards/accuracy_reward": 0.816964328289032,
|
|
"step": 611
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.76116943359375,
|
|
"epoch": 3.122137404580153,
|
|
"grad_norm": 0.16182405484609294,
|
|
"kl": 0.036376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.8169643133878708,
|
|
"reward_std": 0.03539504110813141,
|
|
"rewards/accuracy_reward": 0.8169643133878708,
|
|
"step": 612
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.9397506713867,
|
|
"epoch": 3.127226463104326,
|
|
"grad_norm": 0.29466200983917096,
|
|
"kl": 0.04132080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0079,
|
|
"reward": 0.8191964626312256,
|
|
"reward_std": 0.06608309131115675,
|
|
"rewards/accuracy_reward": 0.8191964626312256,
|
|
"step": 613
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.18528747558594,
|
|
"epoch": 3.132315521628499,
|
|
"grad_norm": 0.37028790033354736,
|
|
"kl": 0.040283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0059,
|
|
"reward": 0.6785714626312256,
|
|
"reward_std": 0.14488856680691242,
|
|
"rewards/accuracy_reward": 0.6785714626312256,
|
|
"step": 614
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.8326110839844,
|
|
"epoch": 3.1374045801526718,
|
|
"grad_norm": 0.24426900672685442,
|
|
"kl": 0.04193115234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.720982164144516,
|
|
"reward_std": 0.08595624193549156,
|
|
"rewards/accuracy_reward": 0.720982164144516,
|
|
"step": 615
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.8616256713867,
|
|
"epoch": 3.1424936386768447,
|
|
"grad_norm": 0.2820551616395041,
|
|
"kl": 0.04302978515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.8281250447034836,
|
|
"reward_std": 0.0801670728251338,
|
|
"rewards/accuracy_reward": 0.8281250447034836,
|
|
"step": 616
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.2522506713867,
|
|
"epoch": 3.1475826972010177,
|
|
"grad_norm": 0.26225698657668917,
|
|
"kl": 0.04449462890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.6116071715950966,
|
|
"reward_std": 0.07501166500151157,
|
|
"rewards/accuracy_reward": 0.6116071715950966,
|
|
"step": 617
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 346.5401916503906,
|
|
"epoch": 3.1526717557251906,
|
|
"grad_norm": 0.2630733654921422,
|
|
"kl": 0.0440673828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0024,
|
|
"reward": 0.870535746216774,
|
|
"reward_std": 0.06388495210558176,
|
|
"rewards/accuracy_reward": 0.870535746216774,
|
|
"step": 618
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.43751525878906,
|
|
"epoch": 3.157760814249364,
|
|
"grad_norm": 0.21666299827365587,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.832589328289032,
|
|
"reward_std": 0.05401241313666105,
|
|
"rewards/accuracy_reward": 0.832589328289032,
|
|
"step": 619
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.2835006713867,
|
|
"epoch": 3.162849872773537,
|
|
"grad_norm": 0.23041363980945273,
|
|
"kl": 0.0338134765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0029,
|
|
"reward": 0.5937500223517418,
|
|
"reward_std": 0.07110932841897011,
|
|
"rewards/accuracy_reward": 0.5937500223517418,
|
|
"step": 620
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.21654510498047,
|
|
"epoch": 3.16793893129771,
|
|
"grad_norm": 0.31624338409159386,
|
|
"kl": 0.04266357421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.723214328289032,
|
|
"reward_std": 0.12588467728346586,
|
|
"rewards/accuracy_reward": 0.723214328289032,
|
|
"step": 621
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.23885345458984,
|
|
"epoch": 3.173027989821883,
|
|
"grad_norm": 0.23240927253886212,
|
|
"kl": 0.038848876953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.8080357611179352,
|
|
"reward_std": 0.06388495303690434,
|
|
"rewards/accuracy_reward": 0.8080357611179352,
|
|
"step": 622
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.7611846923828,
|
|
"epoch": 3.178117048346056,
|
|
"grad_norm": 0.25495663006769903,
|
|
"kl": 0.04083251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.062309928238391876,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 623
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.2500228881836,
|
|
"epoch": 3.183206106870229,
|
|
"grad_norm": 0.2430907572264551,
|
|
"kl": 0.03887939453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0077,
|
|
"reward": 0.7455357611179352,
|
|
"reward_std": 0.04791303817182779,
|
|
"rewards/accuracy_reward": 0.7455357611179352,
|
|
"step": 624
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.83260345458984,
|
|
"epoch": 3.188295165394402,
|
|
"grad_norm": 0.18846428669679094,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.7410714626312256,
|
|
"reward_std": 0.03441469371318817,
|
|
"rewards/accuracy_reward": 0.7410714626312256,
|
|
"step": 625
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.01341247558594,
|
|
"epoch": 3.1933842239185752,
|
|
"grad_norm": 0.25244327963870355,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.7522321790456772,
|
|
"reward_std": 0.058729544281959534,
|
|
"rewards/accuracy_reward": 0.7522321790456772,
|
|
"step": 626
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.61163330078125,
|
|
"epoch": 3.198473282442748,
|
|
"grad_norm": 0.35993623996595636,
|
|
"kl": 0.0455322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0009,
|
|
"reward": 0.6964285969734192,
|
|
"reward_std": 0.15657011978328228,
|
|
"rewards/accuracy_reward": 0.6964285969734192,
|
|
"step": 627
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.1785888671875,
|
|
"epoch": 3.203562340966921,
|
|
"grad_norm": 0.270880235517583,
|
|
"kl": 0.03558349609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0075,
|
|
"reward": 0.6852678805589676,
|
|
"reward_std": 0.11927327513694763,
|
|
"rewards/accuracy_reward": 0.6852678805589676,
|
|
"step": 628
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.5491256713867,
|
|
"epoch": 3.208651399491094,
|
|
"grad_norm": 0.17017632831466686,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0024,
|
|
"reward": 0.7611607611179352,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.7611607611179352,
|
|
"step": 629
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.70760345458984,
|
|
"epoch": 3.213740458015267,
|
|
"grad_norm": 0.2563735068076232,
|
|
"kl": 0.04339599609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0106,
|
|
"reward": 0.698660746216774,
|
|
"reward_std": 0.07047826889902353,
|
|
"rewards/accuracy_reward": 0.698660746216774,
|
|
"step": 630
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.7053756713867,
|
|
"epoch": 3.21882951653944,
|
|
"grad_norm": 0.28587291112904484,
|
|
"kl": 0.0439453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0032,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.09940916765481234,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 631
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 443.2723388671875,
|
|
"epoch": 3.223918575063613,
|
|
"grad_norm": 0.23957626775845092,
|
|
"kl": 0.033935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.6808035969734192,
|
|
"reward_std": 0.053694264963269234,
|
|
"rewards/accuracy_reward": 0.6808035969734192,
|
|
"step": 632
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.56251525878906,
|
|
"epoch": 3.2290076335877864,
|
|
"grad_norm": 0.24762671660341598,
|
|
"kl": 0.041046142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0018,
|
|
"reward": 0.6986607611179352,
|
|
"reward_std": 0.07905755192041397,
|
|
"rewards/accuracy_reward": 0.6986607611179352,
|
|
"step": 633
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.37278747558594,
|
|
"epoch": 3.2340966921119594,
|
|
"grad_norm": 0.21421616813622704,
|
|
"kl": 0.0419921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0018,
|
|
"reward": 0.926339328289032,
|
|
"reward_std": 0.07376392930746078,
|
|
"rewards/accuracy_reward": 0.926339328289032,
|
|
"step": 634
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 440.86832427978516,
|
|
"epoch": 3.2391857506361323,
|
|
"grad_norm": 0.29100501812736695,
|
|
"kl": 0.036956787109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.7477678954601288,
|
|
"reward_std": 0.0895366258919239,
|
|
"rewards/accuracy_reward": 0.7477678954601288,
|
|
"step": 635
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.35939025878906,
|
|
"epoch": 3.2442748091603053,
|
|
"grad_norm": 0.2336682314464919,
|
|
"kl": 0.042816162109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0002,
|
|
"reward": 0.783482164144516,
|
|
"reward_std": 0.07658668980002403,
|
|
"rewards/accuracy_reward": 0.783482164144516,
|
|
"step": 636
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.7924270629883,
|
|
"epoch": 3.2493638676844783,
|
|
"grad_norm": 0.3409330327011361,
|
|
"kl": 0.04486083984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.012,
|
|
"reward": 0.714285746216774,
|
|
"reward_std": 0.12598120048642159,
|
|
"rewards/accuracy_reward": 0.714285746216774,
|
|
"step": 637
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.5960006713867,
|
|
"epoch": 3.2544529262086512,
|
|
"grad_norm": 0.2721761181692073,
|
|
"kl": 0.036407470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0129,
|
|
"reward": 0.676339328289032,
|
|
"reward_std": 0.09084536600857973,
|
|
"rewards/accuracy_reward": 0.676339328289032,
|
|
"step": 638
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.20091247558594,
|
|
"epoch": 3.2595419847328246,
|
|
"grad_norm": 0.2254530976772047,
|
|
"kl": 0.0411376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.863839328289032,
|
|
"reward_std": 0.08098186552524567,
|
|
"rewards/accuracy_reward": 0.863839328289032,
|
|
"step": 639
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.75894927978516,
|
|
"epoch": 3.2646310432569976,
|
|
"grad_norm": 0.23090982114457095,
|
|
"kl": 0.034576416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.6562500298023224,
|
|
"reward_std": 0.058849673718214035,
|
|
"rewards/accuracy_reward": 0.6562500298023224,
|
|
"step": 640
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 420.1986770629883,
|
|
"epoch": 3.2697201017811706,
|
|
"grad_norm": 0.20970869776093362,
|
|
"kl": 0.038055419921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.7433036118745804,
|
|
"reward_std": 0.056394290179014206,
|
|
"rewards/accuracy_reward": 0.7433036118745804,
|
|
"step": 641
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.8504638671875,
|
|
"epoch": 3.2748091603053435,
|
|
"grad_norm": 0.3028783447423427,
|
|
"kl": 0.04852294921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.005,
|
|
"reward": 0.7589286118745804,
|
|
"reward_std": 0.08613727707415819,
|
|
"rewards/accuracy_reward": 0.7589286118745804,
|
|
"step": 642
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.9643020629883,
|
|
"epoch": 3.2798982188295165,
|
|
"grad_norm": 0.25492374875740365,
|
|
"kl": 0.037567138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.895089328289032,
|
|
"reward_std": 0.0625000037252903,
|
|
"rewards/accuracy_reward": 0.895089328289032,
|
|
"step": 643
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.77010345458984,
|
|
"epoch": 3.2849872773536894,
|
|
"grad_norm": 0.31517195871528625,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.814732164144516,
|
|
"reward_std": 0.10004021972417831,
|
|
"rewards/accuracy_reward": 0.814732164144516,
|
|
"step": 644
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.52010345458984,
|
|
"epoch": 3.2900763358778624,
|
|
"grad_norm": 0.2955813186190452,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.013,
|
|
"reward": 0.7656250447034836,
|
|
"reward_std": 0.056651534512639046,
|
|
"rewards/accuracy_reward": 0.7656250447034836,
|
|
"step": 645
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.21207427978516,
|
|
"epoch": 3.2951653944020354,
|
|
"grad_norm": 0.23760306636953452,
|
|
"kl": 0.039703369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0101,
|
|
"reward": 0.6696428805589676,
|
|
"reward_std": 0.0654509337618947,
|
|
"rewards/accuracy_reward": 0.6696428805589676,
|
|
"step": 646
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 363.0669708251953,
|
|
"epoch": 3.300254452926209,
|
|
"grad_norm": 0.3550598292123022,
|
|
"kl": 0.0421142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0031,
|
|
"reward": 0.8995536267757416,
|
|
"reward_std": 0.1182012278586626,
|
|
"rewards/accuracy_reward": 0.8995536267757416,
|
|
"step": 647
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.19197845458984,
|
|
"epoch": 3.3053435114503817,
|
|
"grad_norm": 0.2785541822365125,
|
|
"kl": 0.0462646484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0018,
|
|
"reward": 0.7366071790456772,
|
|
"reward_std": 0.0814228504896164,
|
|
"rewards/accuracy_reward": 0.7366071790456772,
|
|
"step": 648
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.6607360839844,
|
|
"epoch": 3.3104325699745547,
|
|
"grad_norm": 0.2349895923734611,
|
|
"kl": 0.0372314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.7410714700818062,
|
|
"reward_std": 0.061055757105350494,
|
|
"rewards/accuracy_reward": 0.7410714700818062,
|
|
"step": 649
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 367.2835006713867,
|
|
"epoch": 3.3155216284987277,
|
|
"grad_norm": 0.22215560396533796,
|
|
"kl": 0.04229736328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0039,
|
|
"reward": 0.6629464626312256,
|
|
"reward_std": 0.05118321720510721,
|
|
"rewards/accuracy_reward": 0.6629464626312256,
|
|
"step": 650
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 388.7745666503906,
|
|
"epoch": 3.3206106870229006,
|
|
"grad_norm": 0.283289296655935,
|
|
"kl": 0.0408935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0051,
|
|
"reward": 0.8973214775323868,
|
|
"reward_std": 0.07374844886362553,
|
|
"rewards/accuracy_reward": 0.8973214775323868,
|
|
"step": 651
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.4285888671875,
|
|
"epoch": 3.325699745547074,
|
|
"grad_norm": 0.11145182451237014,
|
|
"kl": 0.03375244140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0028,
|
|
"reward": 0.8995536118745804,
|
|
"reward_std": 0.03225403279066086,
|
|
"rewards/accuracy_reward": 0.8995536118745804,
|
|
"step": 652
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.3616256713867,
|
|
"epoch": 3.330788804071247,
|
|
"grad_norm": 0.2587207892795744,
|
|
"kl": 0.039306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.7254464775323868,
|
|
"reward_std": 0.06777824554592371,
|
|
"rewards/accuracy_reward": 0.7254464775323868,
|
|
"step": 653
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.97769927978516,
|
|
"epoch": 3.33587786259542,
|
|
"grad_norm": 0.17074145757257464,
|
|
"kl": 0.035491943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.9218750447034836,
|
|
"reward_std": 0.03332607354968786,
|
|
"rewards/accuracy_reward": 0.9218750447034836,
|
|
"step": 654
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.5669860839844,
|
|
"epoch": 3.340966921119593,
|
|
"grad_norm": 0.22564362087366385,
|
|
"kl": 0.04010009765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.7879464775323868,
|
|
"reward_std": 0.05759550258517265,
|
|
"rewards/accuracy_reward": 0.7879464775323868,
|
|
"step": 655
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.0491256713867,
|
|
"epoch": 3.346055979643766,
|
|
"grad_norm": 0.28956084194362786,
|
|
"kl": 0.039215087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0044,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.0777943367138505,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 656
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.8772506713867,
|
|
"epoch": 3.351145038167939,
|
|
"grad_norm": 0.25403898238670936,
|
|
"kl": 0.038360595703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0075,
|
|
"reward": 0.8169643133878708,
|
|
"reward_std": 0.06450806744396687,
|
|
"rewards/accuracy_reward": 0.8169643133878708,
|
|
"step": 657
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.0580596923828,
|
|
"epoch": 3.356234096692112,
|
|
"grad_norm": 0.27920868291769374,
|
|
"kl": 0.04071044921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0059,
|
|
"reward": 0.7500000298023224,
|
|
"reward_std": 0.0655801072716713,
|
|
"rewards/accuracy_reward": 0.7500000298023224,
|
|
"step": 658
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 388.95314025878906,
|
|
"epoch": 3.3613231552162848,
|
|
"grad_norm": 0.1555965359034967,
|
|
"kl": 0.0413818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.877232164144516,
|
|
"reward_std": 0.04162358958274126,
|
|
"rewards/accuracy_reward": 0.877232164144516,
|
|
"step": 659
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 425.2455596923828,
|
|
"epoch": 3.366412213740458,
|
|
"grad_norm": 0.282783654593396,
|
|
"kl": 0.03851318359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0047,
|
|
"reward": 0.7187500298023224,
|
|
"reward_std": 0.07644847221672535,
|
|
"rewards/accuracy_reward": 0.7187500298023224,
|
|
"step": 660
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.3214416503906,
|
|
"epoch": 3.371501272264631,
|
|
"grad_norm": 0.3083253632524364,
|
|
"kl": 0.044281005859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0064,
|
|
"reward": 0.7566964775323868,
|
|
"reward_std": 0.1052577355876565,
|
|
"rewards/accuracy_reward": 0.7566964775323868,
|
|
"step": 661
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 378.37501525878906,
|
|
"epoch": 3.376590330788804,
|
|
"grad_norm": 0.26469576861324823,
|
|
"kl": 0.0411376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0113,
|
|
"reward": 0.77901791036129,
|
|
"reward_std": 0.03898446634411812,
|
|
"rewards/accuracy_reward": 0.77901791036129,
|
|
"step": 662
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.6986846923828,
|
|
"epoch": 3.381679389312977,
|
|
"grad_norm": 0.2627735047617527,
|
|
"kl": 0.04168701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0063,
|
|
"reward": 0.7901785969734192,
|
|
"reward_std": 0.0760837011039257,
|
|
"rewards/accuracy_reward": 0.7901785969734192,
|
|
"step": 663
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.10939025878906,
|
|
"epoch": 3.38676844783715,
|
|
"grad_norm": 0.18452333139172977,
|
|
"kl": 0.03875732421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.7232143133878708,
|
|
"reward_std": 0.05419615097343922,
|
|
"rewards/accuracy_reward": 0.7232143133878708,
|
|
"step": 664
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.3750228881836,
|
|
"epoch": 3.391857506361323,
|
|
"grad_norm": 0.2461374771338689,
|
|
"kl": 0.03839111328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.8772321939468384,
|
|
"reward_std": 0.05495638120919466,
|
|
"rewards/accuracy_reward": 0.8772321939468384,
|
|
"step": 665
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.0558166503906,
|
|
"epoch": 3.3969465648854964,
|
|
"grad_norm": 0.2725249106011546,
|
|
"kl": 0.03228759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.6495536044239998,
|
|
"reward_std": 0.05011117737740278,
|
|
"rewards/accuracy_reward": 0.6495536044239998,
|
|
"step": 666
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.5000228881836,
|
|
"epoch": 3.4020356234096694,
|
|
"grad_norm": 0.3329227921759514,
|
|
"kl": 0.037139892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0064,
|
|
"reward": 0.6517857313156128,
|
|
"reward_std": 0.13027985487133265,
|
|
"rewards/accuracy_reward": 0.6517857313156128,
|
|
"step": 667
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.6138610839844,
|
|
"epoch": 3.4071246819338423,
|
|
"grad_norm": 0.17960047933009235,
|
|
"kl": 0.04449462890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.8861607611179352,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.8861607611179352,
|
|
"step": 668
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.6808166503906,
|
|
"epoch": 3.4122137404580153,
|
|
"grad_norm": 0.28774083314625865,
|
|
"kl": 0.03741455078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0056,
|
|
"reward": 0.8191964477300644,
|
|
"reward_std": 0.08094278071075678,
|
|
"rewards/accuracy_reward": 0.8191964477300644,
|
|
"step": 669
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.6428680419922,
|
|
"epoch": 3.4173027989821882,
|
|
"grad_norm": 0.1745721965526644,
|
|
"kl": 0.035064697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0036,
|
|
"reward": 0.7053571678698063,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.7053571678698063,
|
|
"step": 670
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.78572845458984,
|
|
"epoch": 3.422391857506361,
|
|
"grad_norm": 0.3028643599835002,
|
|
"kl": 0.03973388671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0043,
|
|
"reward": 0.7879464626312256,
|
|
"reward_std": 0.0633819680660963,
|
|
"rewards/accuracy_reward": 0.7879464626312256,
|
|
"step": 671
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.3616256713867,
|
|
"epoch": 3.427480916030534,
|
|
"grad_norm": 0.1785298830645011,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.7254464477300644,
|
|
"reward_std": 0.042757630348205566,
|
|
"rewards/accuracy_reward": 0.7254464477300644,
|
|
"step": 672
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.73439025878906,
|
|
"epoch": 3.432569974554707,
|
|
"grad_norm": 0.1509571326952281,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0046,
|
|
"reward": 0.720982164144516,
|
|
"reward_std": 0.035539694130420685,
|
|
"rewards/accuracy_reward": 0.720982164144516,
|
|
"step": 673
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.36832427978516,
|
|
"epoch": 3.4376590330788805,
|
|
"grad_norm": 0.27893651104570416,
|
|
"kl": 0.04473876953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.8437500298023224,
|
|
"reward_std": 0.09173476230353117,
|
|
"rewards/accuracy_reward": 0.8437500298023224,
|
|
"step": 674
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.1942138671875,
|
|
"epoch": 3.4427480916030535,
|
|
"grad_norm": 0.26786490204804564,
|
|
"kl": 0.04034423828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0028,
|
|
"reward": 0.7142857611179352,
|
|
"reward_std": 0.0712384982034564,
|
|
"rewards/accuracy_reward": 0.7142857611179352,
|
|
"step": 675
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.5111770629883,
|
|
"epoch": 3.4478371501272265,
|
|
"grad_norm": 0.1557097339950374,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.7991071939468384,
|
|
"reward_std": 0.036299921572208405,
|
|
"rewards/accuracy_reward": 0.7991071939468384,
|
|
"step": 676
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.59153747558594,
|
|
"epoch": 3.4529262086513994,
|
|
"grad_norm": 0.22808365057294916,
|
|
"kl": 0.038299560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0045,
|
|
"reward": 0.7343750447034836,
|
|
"reward_std": 0.05011117737740278,
|
|
"rewards/accuracy_reward": 0.7343750447034836,
|
|
"step": 677
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 432.03797149658203,
|
|
"epoch": 3.4580152671755724,
|
|
"grad_norm": 0.13826976815213973,
|
|
"kl": 0.038330078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.7142857536673546,
|
|
"reward_std": 0.01785714365541935,
|
|
"rewards/accuracy_reward": 0.7142857536673546,
|
|
"step": 678
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.4308166503906,
|
|
"epoch": 3.4631043256997454,
|
|
"grad_norm": 0.3178882181706591,
|
|
"kl": 0.04876708984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.002,
|
|
"reward": 0.8125000447034836,
|
|
"reward_std": 0.05903974920511246,
|
|
"rewards/accuracy_reward": 0.8125000447034836,
|
|
"step": 679
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.04689025878906,
|
|
"epoch": 3.4681933842239188,
|
|
"grad_norm": 0.19129672393618688,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.816964328289032,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.816964328289032,
|
|
"step": 680
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.73438262939453,
|
|
"epoch": 3.4732824427480917,
|
|
"grad_norm": 0.29768289719047186,
|
|
"kl": 0.0411376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.7031250521540642,
|
|
"reward_std": 0.05665153544396162,
|
|
"rewards/accuracy_reward": 0.7031250521540642,
|
|
"step": 681
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.2768096923828,
|
|
"epoch": 3.4783715012722647,
|
|
"grad_norm": 0.194194929002816,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.011,
|
|
"reward": 0.7410714626312256,
|
|
"reward_std": 0.04118260461837053,
|
|
"rewards/accuracy_reward": 0.7410714626312256,
|
|
"step": 682
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.7076110839844,
|
|
"epoch": 3.4834605597964376,
|
|
"grad_norm": 0.23879471863159246,
|
|
"kl": 0.0382080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.6517857387661934,
|
|
"reward_std": 0.07312643527984619,
|
|
"rewards/accuracy_reward": 0.6517857387661934,
|
|
"step": 683
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 364.2433166503906,
|
|
"epoch": 3.4885496183206106,
|
|
"grad_norm": 0.21154424421464163,
|
|
"kl": 0.04107666015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.7566964477300644,
|
|
"reward_std": 0.04334326460957527,
|
|
"rewards/accuracy_reward": 0.7566964477300644,
|
|
"step": 684
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 369.3303756713867,
|
|
"epoch": 3.4936386768447836,
|
|
"grad_norm": 0.30815543283372915,
|
|
"kl": 0.0421142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.7500000521540642,
|
|
"reward_std": 0.10890679247677326,
|
|
"rewards/accuracy_reward": 0.7500000521540642,
|
|
"step": 685
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.60269927978516,
|
|
"epoch": 3.4987277353689565,
|
|
"grad_norm": 0.26352389485066563,
|
|
"kl": 0.033660888671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.8928571790456772,
|
|
"reward_std": 0.07110932655632496,
|
|
"rewards/accuracy_reward": 0.8928571790456772,
|
|
"step": 686
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.5937728881836,
|
|
"epoch": 3.5038167938931295,
|
|
"grad_norm": 0.24194299290176932,
|
|
"kl": 0.04248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.7477678805589676,
|
|
"reward_std": 0.05459804926067591,
|
|
"rewards/accuracy_reward": 0.7477678805589676,
|
|
"step": 687
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.6785888671875,
|
|
"epoch": 3.508905852417303,
|
|
"grad_norm": 0.18752950685020256,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.7165178880095482,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.7165178880095482,
|
|
"step": 688
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.85269927978516,
|
|
"epoch": 3.513994910941476,
|
|
"grad_norm": 0.24417620390632744,
|
|
"kl": 0.04461669921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0083,
|
|
"reward": 0.861607164144516,
|
|
"reward_std": 0.06701801717281342,
|
|
"rewards/accuracy_reward": 0.861607164144516,
|
|
"step": 689
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 372.46876525878906,
|
|
"epoch": 3.519083969465649,
|
|
"grad_norm": 0.029484001824464336,
|
|
"kl": 0.040771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.7500000447034836,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.7500000447034836,
|
|
"step": 690
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.6116256713867,
|
|
"epoch": 3.524173027989822,
|
|
"grad_norm": 0.2414633809952364,
|
|
"kl": 0.03277587890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.9129464626312256,
|
|
"reward_std": 0.0887854415923357,
|
|
"rewards/accuracy_reward": 0.9129464626312256,
|
|
"step": 691
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.06475830078125,
|
|
"epoch": 3.5292620865139948,
|
|
"grad_norm": 0.11041258581578524,
|
|
"kl": 0.037139892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.7455357685685158,
|
|
"reward_std": 0.012198750860989094,
|
|
"rewards/accuracy_reward": 0.7455357685685158,
|
|
"step": 692
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.4754638671875,
|
|
"epoch": 3.534351145038168,
|
|
"grad_norm": 0.19616251845327218,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.814732164144516,
|
|
"reward_std": 0.03495405800640583,
|
|
"rewards/accuracy_reward": 0.814732164144516,
|
|
"step": 693
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.10716247558594,
|
|
"epoch": 3.539440203562341,
|
|
"grad_norm": 0.24316588568548692,
|
|
"kl": 0.0418701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.006,
|
|
"reward": 0.8125000447034836,
|
|
"reward_std": 0.06369487755000591,
|
|
"rewards/accuracy_reward": 0.8125000447034836,
|
|
"step": 694
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.47100830078125,
|
|
"epoch": 3.544529262086514,
|
|
"grad_norm": 0.1773523052343864,
|
|
"kl": 0.034271240234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.8370536267757416,
|
|
"reward_std": 0.03615527041256428,
|
|
"rewards/accuracy_reward": 0.8370536267757416,
|
|
"step": 695
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 379.3460006713867,
|
|
"epoch": 3.549618320610687,
|
|
"grad_norm": 0.2892963083247234,
|
|
"kl": 0.033294677734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.6808036118745804,
|
|
"reward_std": 0.08977331966161728,
|
|
"rewards/accuracy_reward": 0.6808036118745804,
|
|
"step": 696
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.8683166503906,
|
|
"epoch": 3.55470737913486,
|
|
"grad_norm": 0.28702790607198414,
|
|
"kl": 0.0399169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0075,
|
|
"reward": 0.8191964626312256,
|
|
"reward_std": 0.09946212265640497,
|
|
"rewards/accuracy_reward": 0.8191964626312256,
|
|
"step": 697
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.49778747558594,
|
|
"epoch": 3.559796437659033,
|
|
"grad_norm": 0.2780939572367632,
|
|
"kl": 0.0367431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.698660746216774,
|
|
"reward_std": 0.075452646240592,
|
|
"rewards/accuracy_reward": 0.698660746216774,
|
|
"step": 698
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.7031478881836,
|
|
"epoch": 3.564885496183206,
|
|
"grad_norm": 0.18821460351310842,
|
|
"kl": 0.0345458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0023,
|
|
"reward": 0.7745535969734192,
|
|
"reward_std": 0.043343265540897846,
|
|
"rewards/accuracy_reward": 0.7745535969734192,
|
|
"step": 699
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 379.50001525878906,
|
|
"epoch": 3.569974554707379,
|
|
"grad_norm": 0.24953673790939787,
|
|
"kl": 0.041778564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.6071428880095482,
|
|
"reward_std": 0.05715452041476965,
|
|
"rewards/accuracy_reward": 0.6071428880095482,
|
|
"step": 700
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.5580520629883,
|
|
"epoch": 3.5750636132315523,
|
|
"grad_norm": 0.24754069376089746,
|
|
"kl": 0.0350341796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0042,
|
|
"reward": 0.7343750298023224,
|
|
"reward_std": 0.06836387887597084,
|
|
"rewards/accuracy_reward": 0.7343750298023224,
|
|
"step": 701
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.0736770629883,
|
|
"epoch": 3.5801526717557253,
|
|
"grad_norm": 0.16666219668110593,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.7410714626312256,
|
|
"reward_std": 0.026025486178696156,
|
|
"rewards/accuracy_reward": 0.7410714626312256,
|
|
"step": 702
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.9643020629883,
|
|
"epoch": 3.5852417302798982,
|
|
"grad_norm": 0.29210630646822694,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.004,
|
|
"reward": 0.8459821939468384,
|
|
"reward_std": 0.11624883487820625,
|
|
"rewards/accuracy_reward": 0.8459821939468384,
|
|
"step": 703
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.2232360839844,
|
|
"epoch": 3.590330788804071,
|
|
"grad_norm": 0.22899866289339668,
|
|
"kl": 0.042236328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.7901785969734192,
|
|
"reward_std": 0.04118260461837053,
|
|
"rewards/accuracy_reward": 0.7901785969734192,
|
|
"step": 704
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.30135345458984,
|
|
"epoch": 3.595419847328244,
|
|
"grad_norm": 0.1885142236786285,
|
|
"kl": 0.0372314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0056,
|
|
"reward": 0.7678571939468384,
|
|
"reward_std": 0.04099252913147211,
|
|
"rewards/accuracy_reward": 0.7678571939468384,
|
|
"step": 705
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.07366943359375,
|
|
"epoch": 3.6005089058524176,
|
|
"grad_norm": 0.2685729098856735,
|
|
"kl": 0.04595947265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.6294643133878708,
|
|
"reward_std": 0.06828013248741627,
|
|
"rewards/accuracy_reward": 0.6294643133878708,
|
|
"step": 706
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.5736846923828,
|
|
"epoch": 3.6055979643765905,
|
|
"grad_norm": 0.2559042774165835,
|
|
"kl": 0.036285400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.006,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.0840228870511055,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 707
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.39733123779297,
|
|
"epoch": 3.6106870229007635,
|
|
"grad_norm": 0.22606297406685444,
|
|
"kl": 0.038909912109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.06576384603977203,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 708
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.70314025878906,
|
|
"epoch": 3.6157760814249365,
|
|
"grad_norm": 0.19824773731654147,
|
|
"kl": 0.036285400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.8392857611179352,
|
|
"reward_std": 0.04099252913147211,
|
|
"rewards/accuracy_reward": 0.8392857611179352,
|
|
"step": 709
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.9129638671875,
|
|
"epoch": 3.6208651399491094,
|
|
"grad_norm": 0.1983818756553478,
|
|
"kl": 0.03997802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.03225403279066086,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 710
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 365.7187728881836,
|
|
"epoch": 3.6259541984732824,
|
|
"grad_norm": 0.21770916986776495,
|
|
"kl": 0.044189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0111,
|
|
"reward": 0.785714328289032,
|
|
"reward_std": 0.04980097245424986,
|
|
"rewards/accuracy_reward": 0.785714328289032,
|
|
"step": 711
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.2968978881836,
|
|
"epoch": 3.6310432569974553,
|
|
"grad_norm": 0.2510553792550636,
|
|
"kl": 0.034759521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.7031250298023224,
|
|
"reward_std": 0.062119850888848305,
|
|
"rewards/accuracy_reward": 0.7031250298023224,
|
|
"step": 712
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.61385345458984,
|
|
"epoch": 3.6361323155216283,
|
|
"grad_norm": 0.20506565001429988,
|
|
"kl": 0.03497314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.761160746216774,
|
|
"reward_std": 0.05118321720510721,
|
|
"rewards/accuracy_reward": 0.761160746216774,
|
|
"step": 713
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 390.6406478881836,
|
|
"epoch": 3.6412213740458013,
|
|
"grad_norm": 0.21185422242343718,
|
|
"kl": 0.0382080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0099,
|
|
"reward": 0.8035714626312256,
|
|
"reward_std": 0.04508384130895138,
|
|
"rewards/accuracy_reward": 0.8035714626312256,
|
|
"step": 714
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.17635345458984,
|
|
"epoch": 3.6463104325699747,
|
|
"grad_norm": 0.25698651382335774,
|
|
"kl": 0.04144287109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.7276785969734192,
|
|
"reward_std": 0.06635581515729427,
|
|
"rewards/accuracy_reward": 0.7276785969734192,
|
|
"step": 715
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.41297149658203,
|
|
"epoch": 3.6513994910941476,
|
|
"grad_norm": 0.20627442533846088,
|
|
"kl": 0.0361328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.7522321939468384,
|
|
"reward_std": 0.05742724426090717,
|
|
"rewards/accuracy_reward": 0.7522321939468384,
|
|
"step": 716
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.09376525878906,
|
|
"epoch": 3.6564885496183206,
|
|
"grad_norm": 0.27671482024942784,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.832589328289032,
|
|
"reward_std": 0.10420895181596279,
|
|
"rewards/accuracy_reward": 0.832589328289032,
|
|
"step": 717
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.9933166503906,
|
|
"epoch": 3.6615776081424936,
|
|
"grad_norm": 0.15101557170066765,
|
|
"kl": 0.03948974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0058,
|
|
"reward": 0.7924107313156128,
|
|
"reward_std": 0.024900485761463642,
|
|
"rewards/accuracy_reward": 0.7924107313156128,
|
|
"step": 718
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.79689025878906,
|
|
"epoch": 3.6666666666666665,
|
|
"grad_norm": 0.25862193506295234,
|
|
"kl": 0.0391845703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0061,
|
|
"reward": 0.8013393133878708,
|
|
"reward_std": 0.06872111465781927,
|
|
"rewards/accuracy_reward": 0.8013393133878708,
|
|
"step": 719
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.7522506713867,
|
|
"epoch": 3.67175572519084,
|
|
"grad_norm": 0.260646792784835,
|
|
"kl": 0.03924560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0071,
|
|
"reward": 0.7879464626312256,
|
|
"reward_std": 0.06715513207018375,
|
|
"rewards/accuracy_reward": 0.7879464626312256,
|
|
"step": 720
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.3125228881836,
|
|
"epoch": 3.676844783715013,
|
|
"grad_norm": 0.28823526002549565,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0069,
|
|
"reward": 0.8191964775323868,
|
|
"reward_std": 0.1204148530960083,
|
|
"rewards/accuracy_reward": 0.8191964775323868,
|
|
"step": 721
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.2120666503906,
|
|
"epoch": 3.681933842239186,
|
|
"grad_norm": 0.25419214465949347,
|
|
"kl": 0.04248046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.6339285969734192,
|
|
"reward_std": 0.05539736431092024,
|
|
"rewards/accuracy_reward": 0.6339285969734192,
|
|
"step": 722
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.3504638671875,
|
|
"epoch": 3.687022900763359,
|
|
"grad_norm": 0.22372868706439392,
|
|
"kl": 0.040069580078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.8392857611179352,
|
|
"reward_std": 0.048795003443956375,
|
|
"rewards/accuracy_reward": 0.8392857611179352,
|
|
"step": 723
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.1473388671875,
|
|
"epoch": 3.6921119592875318,
|
|
"grad_norm": 0.2294228001152274,
|
|
"kl": 0.03973388671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.7120535969734192,
|
|
"reward_std": 0.06374783627688885,
|
|
"rewards/accuracy_reward": 0.7120535969734192,
|
|
"step": 724
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.93528747558594,
|
|
"epoch": 3.6972010178117047,
|
|
"grad_norm": 0.2442438691995658,
|
|
"kl": 0.03082275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.6718750298023224,
|
|
"reward_std": 0.06312472466379404,
|
|
"rewards/accuracy_reward": 0.6718750298023224,
|
|
"step": 725
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.18751525878906,
|
|
"epoch": 3.7022900763358777,
|
|
"grad_norm": 0.11866603878757535,
|
|
"kl": 0.041107177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.7477678805589676,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.7477678805589676,
|
|
"step": 726
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.5067138671875,
|
|
"epoch": 3.7073791348600507,
|
|
"grad_norm": 0.2954740207443915,
|
|
"kl": 0.037200927734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0045,
|
|
"reward": 0.678571455180645,
|
|
"reward_std": 0.09267873037606478,
|
|
"rewards/accuracy_reward": 0.678571455180645,
|
|
"step": 727
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.91072845458984,
|
|
"epoch": 3.712468193384224,
|
|
"grad_norm": 0.3182081737785432,
|
|
"kl": 0.0426025390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0019,
|
|
"reward": 0.7812500298023224,
|
|
"reward_std": 0.11135067138820887,
|
|
"rewards/accuracy_reward": 0.7812500298023224,
|
|
"step": 728
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.5468978881836,
|
|
"epoch": 3.717557251908397,
|
|
"grad_norm": 0.24677606658962203,
|
|
"kl": 0.04034423828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0097,
|
|
"reward": 0.8035714626312256,
|
|
"reward_std": 0.08318000379949808,
|
|
"rewards/accuracy_reward": 0.8035714626312256,
|
|
"step": 729
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.0111770629883,
|
|
"epoch": 3.72264631043257,
|
|
"grad_norm": 0.2823707794307348,
|
|
"kl": 0.03692626953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.761160746216774,
|
|
"reward_std": 0.11374405398964882,
|
|
"rewards/accuracy_reward": 0.761160746216774,
|
|
"step": 730
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.6361770629883,
|
|
"epoch": 3.727735368956743,
|
|
"grad_norm": 0.15913933249198417,
|
|
"kl": 0.0328369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0032,
|
|
"reward": 0.7879464477300644,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.7879464477300644,
|
|
"step": 731
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.1897506713867,
|
|
"epoch": 3.732824427480916,
|
|
"grad_norm": 0.19045907251634567,
|
|
"kl": 0.033905029296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.6986607313156128,
|
|
"reward_std": 0.05118321720510721,
|
|
"rewards/accuracy_reward": 0.6986607313156128,
|
|
"step": 732
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.0491256713867,
|
|
"epoch": 3.7379134860050893,
|
|
"grad_norm": 0.2388549367648875,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.8861607313156128,
|
|
"reward_std": 0.05665153358131647,
|
|
"rewards/accuracy_reward": 0.8861607313156128,
|
|
"step": 733
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.5759048461914,
|
|
"epoch": 3.7430025445292623,
|
|
"grad_norm": 0.20600297490821115,
|
|
"kl": 0.036651611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0003,
|
|
"reward": 0.7656250447034836,
|
|
"reward_std": 0.06885028630495071,
|
|
"rewards/accuracy_reward": 0.7656250447034836,
|
|
"step": 734
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.54466247558594,
|
|
"epoch": 3.7480916030534353,
|
|
"grad_norm": 0.29283524951866263,
|
|
"kl": 0.039947509765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.6852678805589676,
|
|
"reward_std": 0.06777824461460114,
|
|
"rewards/accuracy_reward": 0.6852678805589676,
|
|
"step": 735
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.0826110839844,
|
|
"epoch": 3.753180661577608,
|
|
"grad_norm": 0.27184449037617886,
|
|
"kl": 0.0440673828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.7901786267757416,
|
|
"reward_std": 0.06105575617402792,
|
|
"rewards/accuracy_reward": 0.7901786267757416,
|
|
"step": 736
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 437.75894927978516,
|
|
"epoch": 3.758269720101781,
|
|
"grad_norm": 0.30749464852086206,
|
|
"kl": 0.037384033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0028,
|
|
"reward": 0.8549107611179352,
|
|
"reward_std": 0.10424642264842987,
|
|
"rewards/accuracy_reward": 0.8549107611179352,
|
|
"step": 737
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.25447845458984,
|
|
"epoch": 3.763358778625954,
|
|
"grad_norm": 0.2028011950869312,
|
|
"kl": 0.0340576171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.676339328289032,
|
|
"reward_std": 0.05557949375361204,
|
|
"rewards/accuracy_reward": 0.676339328289032,
|
|
"step": 738
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.3236770629883,
|
|
"epoch": 3.768447837150127,
|
|
"grad_norm": 0.6844046164923124,
|
|
"kl": 0.04296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.7901786118745804,
|
|
"reward_std": 0.05227183550596237,
|
|
"rewards/accuracy_reward": 0.7901786118745804,
|
|
"step": 739
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.92858123779297,
|
|
"epoch": 3.7735368956743,
|
|
"grad_norm": 0.24123348419814913,
|
|
"kl": 0.033660888671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.7723214775323868,
|
|
"reward_std": 0.06262173876166344,
|
|
"rewards/accuracy_reward": 0.7723214775323868,
|
|
"step": 740
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.2143020629883,
|
|
"epoch": 3.778625954198473,
|
|
"grad_norm": 0.3028445418192028,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.7209821790456772,
|
|
"reward_std": 0.09496746677905321,
|
|
"rewards/accuracy_reward": 0.7209821790456772,
|
|
"step": 741
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.9799270629883,
|
|
"epoch": 3.7837150127226464,
|
|
"grad_norm": 0.2292052849231483,
|
|
"kl": 0.03533935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.854910746216774,
|
|
"reward_std": 0.07339916005730629,
|
|
"rewards/accuracy_reward": 0.854910746216774,
|
|
"step": 742
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 372.6451110839844,
|
|
"epoch": 3.7888040712468194,
|
|
"grad_norm": 0.4054198167074397,
|
|
"kl": 0.04443359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0056,
|
|
"reward": 0.801339328289032,
|
|
"reward_std": 0.15239506028592587,
|
|
"rewards/accuracy_reward": 0.801339328289032,
|
|
"step": 743
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.02457427978516,
|
|
"epoch": 3.7938931297709924,
|
|
"grad_norm": 0.22291741053205125,
|
|
"kl": 0.03350830078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.7700893133878708,
|
|
"reward_std": 0.07167948130518198,
|
|
"rewards/accuracy_reward": 0.7700893133878708,
|
|
"step": 744
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.22322845458984,
|
|
"epoch": 3.7989821882951653,
|
|
"grad_norm": 0.18747436838696635,
|
|
"kl": 0.040924072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.7834821715950966,
|
|
"reward_std": 0.04526757914572954,
|
|
"rewards/accuracy_reward": 0.7834821715950966,
|
|
"step": 745
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.70091247558594,
|
|
"epoch": 3.8040712468193383,
|
|
"grad_norm": 0.24896140649220513,
|
|
"kl": 0.03607177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0023,
|
|
"reward": 0.6473214626312256,
|
|
"reward_std": 0.07752161100506783,
|
|
"rewards/accuracy_reward": 0.6473214626312256,
|
|
"step": 746
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.6741180419922,
|
|
"epoch": 3.8091603053435117,
|
|
"grad_norm": 0.31022142257006696,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.6696428954601288,
|
|
"reward_std": 0.09833602514117956,
|
|
"rewards/accuracy_reward": 0.6696428954601288,
|
|
"step": 747
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.4174270629883,
|
|
"epoch": 3.8142493638676847,
|
|
"grad_norm": 0.2913253699428815,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0109,
|
|
"reward": 0.642857164144516,
|
|
"reward_std": 0.1200955081731081,
|
|
"rewards/accuracy_reward": 0.642857164144516,
|
|
"step": 748
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.7902069091797,
|
|
"epoch": 3.8193384223918576,
|
|
"grad_norm": 0.3712597157276719,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.665178619325161,
|
|
"reward_std": 0.16140617989003658,
|
|
"rewards/accuracy_reward": 0.665178619325161,
|
|
"step": 749
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.03126525878906,
|
|
"epoch": 3.8244274809160306,
|
|
"grad_norm": 0.27093902520037944,
|
|
"kl": 0.04632568359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.6919643133878708,
|
|
"reward_std": 0.09154469147324562,
|
|
"rewards/accuracy_reward": 0.6919643133878708,
|
|
"step": 750
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.2276916503906,
|
|
"epoch": 3.8295165394402035,
|
|
"grad_norm": 0.29686546192911756,
|
|
"kl": 0.033721923828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.7008928805589676,
|
|
"reward_std": 0.14212027192115784,
|
|
"rewards/accuracy_reward": 0.7008928805589676,
|
|
"step": 751
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.30582427978516,
|
|
"epoch": 3.8346055979643765,
|
|
"grad_norm": 0.22085004970342131,
|
|
"kl": 0.03033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.7120535969734192,
|
|
"reward_std": 0.06885028723627329,
|
|
"rewards/accuracy_reward": 0.7120535969734192,
|
|
"step": 752
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.79466247558594,
|
|
"epoch": 3.8396946564885495,
|
|
"grad_norm": 0.43788533520067846,
|
|
"kl": 0.05401611328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.7477678954601288,
|
|
"reward_std": 0.09613898769021034,
|
|
"rewards/accuracy_reward": 0.7477678954601288,
|
|
"step": 753
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.87278747558594,
|
|
"epoch": 3.8447837150127224,
|
|
"grad_norm": 0.35542519471917566,
|
|
"kl": 0.03582763671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0039,
|
|
"reward": 0.5937500223517418,
|
|
"reward_std": 0.13493337389081717,
|
|
"rewards/accuracy_reward": 0.5937500223517418,
|
|
"step": 754
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.89733123779297,
|
|
"epoch": 3.849872773536896,
|
|
"grad_norm": 0.3380151640344136,
|
|
"kl": 0.04644775390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0155,
|
|
"reward": 0.7098214477300644,
|
|
"reward_std": 0.11691068299114704,
|
|
"rewards/accuracy_reward": 0.7098214477300644,
|
|
"step": 755
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.06697845458984,
|
|
"epoch": 3.854961832061069,
|
|
"grad_norm": 0.2347734539806386,
|
|
"kl": 0.034698486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0039,
|
|
"reward": 0.8258928656578064,
|
|
"reward_std": 0.044468265026807785,
|
|
"rewards/accuracy_reward": 0.8258928656578064,
|
|
"step": 756
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.1830520629883,
|
|
"epoch": 3.8600508905852418,
|
|
"grad_norm": 0.24185712017685304,
|
|
"kl": 0.03448486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.7165178954601288,
|
|
"reward_std": 0.07425143383443356,
|
|
"rewards/accuracy_reward": 0.7165178954601288,
|
|
"step": 757
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.4910888671875,
|
|
"epoch": 3.8651399491094147,
|
|
"grad_norm": 0.2951963968051688,
|
|
"kl": 0.033416748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0075,
|
|
"reward": 0.7500000298023224,
|
|
"reward_std": 0.09267763141542673,
|
|
"rewards/accuracy_reward": 0.7500000298023224,
|
|
"step": 758
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.8504638671875,
|
|
"epoch": 3.8702290076335877,
|
|
"grad_norm": 0.18748248021725178,
|
|
"kl": 0.032257080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.7276785969734192,
|
|
"reward_std": 0.05162420216947794,
|
|
"rewards/accuracy_reward": 0.7276785969734192,
|
|
"step": 759
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.2143096923828,
|
|
"epoch": 3.875318066157761,
|
|
"grad_norm": 0.29791662797435103,
|
|
"kl": 0.041748046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.6808036044239998,
|
|
"reward_std": 0.06855395622551441,
|
|
"rewards/accuracy_reward": 0.6808036044239998,
|
|
"step": 760
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 373.83707427978516,
|
|
"epoch": 3.880407124681934,
|
|
"grad_norm": 0.26490003174417803,
|
|
"kl": 0.035186767578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.776785746216774,
|
|
"reward_std": 0.04791303817182779,
|
|
"rewards/accuracy_reward": 0.776785746216774,
|
|
"step": 761
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.9643020629883,
|
|
"epoch": 3.885496183206107,
|
|
"grad_norm": 0.30981060200734195,
|
|
"kl": 0.0364990234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.8125000447034836,
|
|
"reward_std": 0.10770558193325996,
|
|
"rewards/accuracy_reward": 0.8125000447034836,
|
|
"step": 762
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.44197845458984,
|
|
"epoch": 3.89058524173028,
|
|
"grad_norm": 0.38912043595099843,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.6875000298023224,
|
|
"reward_std": 0.2025979347527027,
|
|
"rewards/accuracy_reward": 0.6875000298023224,
|
|
"step": 763
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.7433166503906,
|
|
"epoch": 3.895674300254453,
|
|
"grad_norm": 0.2557629441553287,
|
|
"kl": 0.033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0106,
|
|
"reward": 0.6718750298023224,
|
|
"reward_std": 0.07469875458627939,
|
|
"rewards/accuracy_reward": 0.6718750298023224,
|
|
"step": 764
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.25001525878906,
|
|
"epoch": 3.900763358778626,
|
|
"grad_norm": 0.30197159689609787,
|
|
"kl": 0.0399169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.8102678805589676,
|
|
"reward_std": 0.11627852451056242,
|
|
"rewards/accuracy_reward": 0.8102678805589676,
|
|
"step": 765
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.06250762939453,
|
|
"epoch": 3.905852417302799,
|
|
"grad_norm": 0.254742125961602,
|
|
"kl": 0.034332275390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0058,
|
|
"reward": 0.709821455180645,
|
|
"reward_std": 0.09593360032886267,
|
|
"rewards/accuracy_reward": 0.709821455180645,
|
|
"step": 766
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.77903747558594,
|
|
"epoch": 3.910941475826972,
|
|
"grad_norm": 0.47368647427928606,
|
|
"kl": 0.039154052734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.6183036118745804,
|
|
"reward_std": 0.10362331382930279,
|
|
"rewards/accuracy_reward": 0.6183036118745804,
|
|
"step": 767
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.5803756713867,
|
|
"epoch": 3.916030534351145,
|
|
"grad_norm": 0.3275919698798669,
|
|
"kl": 0.047607421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0006,
|
|
"reward": 0.6852678880095482,
|
|
"reward_std": 0.09751489572227001,
|
|
"rewards/accuracy_reward": 0.6852678880095482,
|
|
"step": 768
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.2076110839844,
|
|
"epoch": 3.921119592875318,
|
|
"grad_norm": 0.29223166880501233,
|
|
"kl": 0.03411865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.7946428805589676,
|
|
"reward_std": 0.06948134396225214,
|
|
"rewards/accuracy_reward": 0.7946428805589676,
|
|
"step": 769
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.52457427978516,
|
|
"epoch": 3.926208651399491,
|
|
"grad_norm": 0.25396921114788445,
|
|
"kl": 0.03924560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0082,
|
|
"reward": 0.7321428954601288,
|
|
"reward_std": 0.09469474013894796,
|
|
"rewards/accuracy_reward": 0.7321428954601288,
|
|
"step": 770
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 440.8549270629883,
|
|
"epoch": 3.931297709923664,
|
|
"grad_norm": 0.31300292089138565,
|
|
"kl": 0.0333251953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0027,
|
|
"reward": 0.705357164144516,
|
|
"reward_std": 0.13857736438512802,
|
|
"rewards/accuracy_reward": 0.705357164144516,
|
|
"step": 771
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.55135345458984,
|
|
"epoch": 3.936386768447837,
|
|
"grad_norm": 0.2058160494381228,
|
|
"kl": 0.03912353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.7075893133878708,
|
|
"reward_std": 0.07281352579593658,
|
|
"rewards/accuracy_reward": 0.7075893133878708,
|
|
"step": 772
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.27903747558594,
|
|
"epoch": 3.94147582697201,
|
|
"grad_norm": 0.3067818237722658,
|
|
"kl": 0.042236328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0017,
|
|
"reward": 0.8325893431901932,
|
|
"reward_std": 0.08645018190145493,
|
|
"rewards/accuracy_reward": 0.8325893431901932,
|
|
"step": 773
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.6205520629883,
|
|
"epoch": 3.9465648854961835,
|
|
"grad_norm": 0.16636432307267573,
|
|
"kl": 0.0379638671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.808035746216774,
|
|
"reward_std": 0.04602780845016241,
|
|
"rewards/accuracy_reward": 0.808035746216774,
|
|
"step": 774
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.9241180419922,
|
|
"epoch": 3.9516539440203564,
|
|
"grad_norm": 0.33787964544248683,
|
|
"kl": 0.03533935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0037,
|
|
"reward": 0.7812500447034836,
|
|
"reward_std": 0.1018800288438797,
|
|
"rewards/accuracy_reward": 0.7812500447034836,
|
|
"step": 775
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.5870819091797,
|
|
"epoch": 3.9567430025445294,
|
|
"grad_norm": 0.3350431149895565,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0054,
|
|
"reward": 0.7187500298023224,
|
|
"reward_std": 0.09280680492520332,
|
|
"rewards/accuracy_reward": 0.7187500298023224,
|
|
"step": 776
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.7388610839844,
|
|
"epoch": 3.9618320610687023,
|
|
"grad_norm": 0.24551802529542818,
|
|
"kl": 0.03851318359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0063,
|
|
"reward": 0.6339285895228386,
|
|
"reward_std": 0.07042531110346317,
|
|
"rewards/accuracy_reward": 0.6339285895228386,
|
|
"step": 777
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 447.67859649658203,
|
|
"epoch": 3.9669211195928753,
|
|
"grad_norm": 0.24809340170982908,
|
|
"kl": 0.035614013671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.6651785969734192,
|
|
"reward_std": 0.09752393886446953,
|
|
"rewards/accuracy_reward": 0.6651785969734192,
|
|
"step": 778
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.8526916503906,
|
|
"epoch": 3.9720101781170483,
|
|
"grad_norm": 0.23088066896309153,
|
|
"kl": 0.032989501953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.007,
|
|
"reward": 0.7433035969734192,
|
|
"reward_std": 0.0267857164144516,
|
|
"rewards/accuracy_reward": 0.7433035969734192,
|
|
"step": 779
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.15179443359375,
|
|
"epoch": 3.9770992366412212,
|
|
"grad_norm": 0.3047130584118383,
|
|
"kl": 0.0352783203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.808035746216774,
|
|
"reward_std": 0.09788870345801115,
|
|
"rewards/accuracy_reward": 0.808035746216774,
|
|
"step": 780
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 439.7366256713867,
|
|
"epoch": 3.982188295165394,
|
|
"grad_norm": 0.17604449597052646,
|
|
"kl": 0.04547119140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0046,
|
|
"reward": 0.707589328289032,
|
|
"reward_std": 0.05684160999953747,
|
|
"rewards/accuracy_reward": 0.707589328289032,
|
|
"step": 781
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 443.91297149658203,
|
|
"epoch": 3.9872773536895676,
|
|
"grad_norm": 0.2775453914832509,
|
|
"kl": 0.036407470703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0085,
|
|
"reward": 0.7388393133878708,
|
|
"reward_std": 0.09757689572870731,
|
|
"rewards/accuracy_reward": 0.7388393133878708,
|
|
"step": 782
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 433.3080596923828,
|
|
"epoch": 3.9923664122137406,
|
|
"grad_norm": 0.1940226995371348,
|
|
"kl": 0.031036376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0048,
|
|
"reward": 0.8281250298023224,
|
|
"reward_std": 0.029295665211975574,
|
|
"rewards/accuracy_reward": 0.8281250298023224,
|
|
"step": 783
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 494.0384750366211,
|
|
"epoch": 3.9974554707379135,
|
|
"grad_norm": 0.7250192035802584,
|
|
"kl": 0.04022216796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.7544643133878708,
|
|
"reward_std": 0.12348742038011551,
|
|
"rewards/accuracy_reward": 0.7544643133878708,
|
|
"step": 784
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.74779510498047,
|
|
"epoch": 4.005089058524173,
|
|
"grad_norm": 2.811524157992115,
|
|
"kl": 0.28778076171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0204,
|
|
"reward": 0.8080357760190964,
|
|
"reward_std": 0.0357142873108387,
|
|
"rewards/accuracy_reward": 0.8080357760190964,
|
|
"step": 785
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.4241256713867,
|
|
"epoch": 4.010178117048346,
|
|
"grad_norm": 0.1993181181940805,
|
|
"kl": 0.033721923828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0016,
|
|
"reward": 0.832589328289032,
|
|
"reward_std": 0.034954057075083256,
|
|
"rewards/accuracy_reward": 0.832589328289032,
|
|
"step": 786
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 444.6741180419922,
|
|
"epoch": 4.015267175572519,
|
|
"grad_norm": 0.2417088182399602,
|
|
"kl": 0.03570556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.04388263076543808,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 787
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.4799270629883,
|
|
"epoch": 4.020356234096692,
|
|
"grad_norm": 0.2548614348529178,
|
|
"kl": 0.0400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0073,
|
|
"reward": 0.7901786118745804,
|
|
"reward_std": 0.0794068444520235,
|
|
"rewards/accuracy_reward": 0.7901786118745804,
|
|
"step": 788
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.59376525878906,
|
|
"epoch": 4.025445292620865,
|
|
"grad_norm": 0.2434582564987702,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.005,
|
|
"reward": 0.7767857611179352,
|
|
"reward_std": 0.04225464537739754,
|
|
"rewards/accuracy_reward": 0.7767857611179352,
|
|
"step": 789
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.1607360839844,
|
|
"epoch": 4.030534351145038,
|
|
"grad_norm": 0.13928089190456783,
|
|
"kl": 0.029876708984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.7098214626312256,
|
|
"reward_std": 0.01785714365541935,
|
|
"rewards/accuracy_reward": 0.7098214626312256,
|
|
"step": 790
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.5044860839844,
|
|
"epoch": 4.035623409669212,
|
|
"grad_norm": 0.16853906355765638,
|
|
"kl": 0.035400390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.7790178954601288,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.7790178954601288,
|
|
"step": 791
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.5067138671875,
|
|
"epoch": 4.040712468193385,
|
|
"grad_norm": 0.2646094800706115,
|
|
"kl": 0.0355224609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.006,
|
|
"reward": 0.8147321790456772,
|
|
"reward_std": 0.08098186831921339,
|
|
"rewards/accuracy_reward": 0.8147321790456772,
|
|
"step": 792
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.49778747558594,
|
|
"epoch": 4.0458015267175576,
|
|
"grad_norm": 0.6341440366984267,
|
|
"kl": 0.069244384765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0071,
|
|
"reward": 0.9218750447034836,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.9218750447034836,
|
|
"step": 793
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.3616256713867,
|
|
"epoch": 4.0508905852417305,
|
|
"grad_norm": 0.2979099724041331,
|
|
"kl": 0.03765869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.009,
|
|
"reward": 0.6897321790456772,
|
|
"reward_std": 0.08544531557708979,
|
|
"rewards/accuracy_reward": 0.6897321790456772,
|
|
"step": 794
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.60047149658203,
|
|
"epoch": 4.0559796437659035,
|
|
"grad_norm": 0.19214655493721194,
|
|
"kl": 0.04290771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0059,
|
|
"reward": 0.8058035969734192,
|
|
"reward_std": 0.05118321813642979,
|
|
"rewards/accuracy_reward": 0.8058035969734192,
|
|
"step": 795
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.7254638671875,
|
|
"epoch": 4.061068702290076,
|
|
"grad_norm": 0.3162787306031356,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0002,
|
|
"reward": 0.8683036267757416,
|
|
"reward_std": 0.09048059489578009,
|
|
"rewards/accuracy_reward": 0.8683036267757416,
|
|
"step": 796
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.0848388671875,
|
|
"epoch": 4.066157760814249,
|
|
"grad_norm": 0.2111638145807613,
|
|
"kl": 0.032012939453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0096,
|
|
"reward": 0.6852678991854191,
|
|
"reward_std": 0.03495405800640583,
|
|
"rewards/accuracy_reward": 0.6852678991854191,
|
|
"step": 797
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.00670623779297,
|
|
"epoch": 4.071246819338422,
|
|
"grad_norm": 0.1517608878738625,
|
|
"kl": 0.031402587890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0061,
|
|
"reward": 0.7767857760190964,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.7767857760190964,
|
|
"step": 798
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.3393096923828,
|
|
"epoch": 4.076335877862595,
|
|
"grad_norm": 0.3384293100098083,
|
|
"kl": 0.03985595703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0072,
|
|
"reward": 0.622767873108387,
|
|
"reward_std": 0.119829211384058,
|
|
"rewards/accuracy_reward": 0.622767873108387,
|
|
"step": 799
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 366.1942138671875,
|
|
"epoch": 4.081424936386768,
|
|
"grad_norm": 0.2651538321305464,
|
|
"kl": 0.0443115234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0115,
|
|
"reward": 0.8727678805589676,
|
|
"reward_std": 0.04715281073004007,
|
|
"rewards/accuracy_reward": 0.8727678805589676,
|
|
"step": 800
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.8928680419922,
|
|
"epoch": 4.086513994910941,
|
|
"grad_norm": 0.30472371616923044,
|
|
"kl": 0.03515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0055,
|
|
"reward": 0.8147321790456772,
|
|
"reward_std": 0.0788674782961607,
|
|
"rewards/accuracy_reward": 0.8147321790456772,
|
|
"step": 801
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.50672149658203,
|
|
"epoch": 4.091603053435114,
|
|
"grad_norm": 0.3421851714183259,
|
|
"kl": 0.0428466796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0053,
|
|
"reward": 0.6986607536673546,
|
|
"reward_std": 0.10820856783539057,
|
|
"rewards/accuracy_reward": 0.6986607536673546,
|
|
"step": 802
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.34600830078125,
|
|
"epoch": 4.096692111959287,
|
|
"grad_norm": 0.17098161714820015,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.048225948587059975,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 803
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.76341247558594,
|
|
"epoch": 4.10178117048346,
|
|
"grad_norm": 0.22814406325744335,
|
|
"kl": 0.033172607421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.7031250298023224,
|
|
"reward_std": 0.07369549013674259,
|
|
"rewards/accuracy_reward": 0.7031250298023224,
|
|
"step": 804
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.1116256713867,
|
|
"epoch": 4.106870229007634,
|
|
"grad_norm": 0.19860233997921517,
|
|
"kl": 0.04412841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0017,
|
|
"reward": 0.607142873108387,
|
|
"reward_std": 0.03552421182394028,
|
|
"rewards/accuracy_reward": 0.607142873108387,
|
|
"step": 805
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.5602798461914,
|
|
"epoch": 4.111959287531807,
|
|
"grad_norm": 0.19275843985251473,
|
|
"kl": 0.04022216796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.8437500298023224,
|
|
"reward_std": 0.0357142873108387,
|
|
"rewards/accuracy_reward": 0.8437500298023224,
|
|
"step": 806
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.0826110839844,
|
|
"epoch": 4.11704834605598,
|
|
"grad_norm": 0.250704841682075,
|
|
"kl": 0.04388427734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.8660714775323868,
|
|
"reward_std": 0.04388262890279293,
|
|
"rewards/accuracy_reward": 0.8660714775323868,
|
|
"step": 807
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.1875228881836,
|
|
"epoch": 4.122137404580153,
|
|
"grad_norm": 0.16979984097662357,
|
|
"kl": 0.03594970703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.7433036118745804,
|
|
"reward_std": 0.037099236622452736,
|
|
"rewards/accuracy_reward": 0.7433036118745804,
|
|
"step": 808
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.0803756713867,
|
|
"epoch": 4.127226463104326,
|
|
"grad_norm": 0.2455151126214259,
|
|
"kl": 0.03570556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.8080357611179352,
|
|
"reward_std": 0.07739244028925896,
|
|
"rewards/accuracy_reward": 0.8080357611179352,
|
|
"step": 809
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 421.7835006713867,
|
|
"epoch": 4.132315521628499,
|
|
"grad_norm": 0.21382550164800357,
|
|
"kl": 0.0396728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.006,
|
|
"reward": 0.8526786118745804,
|
|
"reward_std": 0.0357142873108387,
|
|
"rewards/accuracy_reward": 0.8526786118745804,
|
|
"step": 810
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.0201110839844,
|
|
"epoch": 4.137404580152672,
|
|
"grad_norm": 0.19735525152233377,
|
|
"kl": 0.0390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0034,
|
|
"reward": 0.5937500298023224,
|
|
"reward_std": 0.03382905758917332,
|
|
"rewards/accuracy_reward": 0.5937500298023224,
|
|
"step": 811
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.32814025878906,
|
|
"epoch": 4.142493638676845,
|
|
"grad_norm": 0.13802141378939065,
|
|
"kl": 0.0361328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0032,
|
|
"reward": 0.7790178954601288,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.7790178954601288,
|
|
"step": 812
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.1250228881836,
|
|
"epoch": 4.147582697201018,
|
|
"grad_norm": 0.2310794642211151,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.6651786118745804,
|
|
"reward_std": 0.046027809381484985,
|
|
"rewards/accuracy_reward": 0.6651786118745804,
|
|
"step": 813
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 442.4710006713867,
|
|
"epoch": 4.152671755725191,
|
|
"grad_norm": 0.3053357942012366,
|
|
"kl": 0.037384033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0185,
|
|
"reward": 0.7254464477300644,
|
|
"reward_std": 0.0650099515914917,
|
|
"rewards/accuracy_reward": 0.7254464477300644,
|
|
"step": 814
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.62500762939453,
|
|
"epoch": 4.157760814249364,
|
|
"grad_norm": 0.21865734509827353,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.003,
|
|
"reward": 0.8437500447034836,
|
|
"reward_std": 0.04225464537739754,
|
|
"rewards/accuracy_reward": 0.8437500447034836,
|
|
"step": 815
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.86385345458984,
|
|
"epoch": 4.162849872773537,
|
|
"grad_norm": 0.24179231776701612,
|
|
"kl": 0.03814697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.6651785969734192,
|
|
"reward_std": 0.061055753380060196,
|
|
"rewards/accuracy_reward": 0.6651785969734192,
|
|
"step": 816
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.7053680419922,
|
|
"epoch": 4.1679389312977095,
|
|
"grad_norm": 0.19843637589348567,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.6919643133878708,
|
|
"reward_std": 0.04199740011245012,
|
|
"rewards/accuracy_reward": 0.6919643133878708,
|
|
"step": 817
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.93751525878906,
|
|
"epoch": 4.1730279898218825,
|
|
"grad_norm": 0.17704102142661893,
|
|
"kl": 0.035675048828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.6004464626312256,
|
|
"reward_std": 0.04634071886539459,
|
|
"rewards/accuracy_reward": 0.6004464626312256,
|
|
"step": 818
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.5915298461914,
|
|
"epoch": 4.178117048346056,
|
|
"grad_norm": 0.09242972303542982,
|
|
"kl": 0.0428466796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0026,
|
|
"reward": 0.8526786118745804,
|
|
"reward_std": 0.012198750860989094,
|
|
"rewards/accuracy_reward": 0.8526786118745804,
|
|
"step": 819
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.33260345458984,
|
|
"epoch": 4.183206106870229,
|
|
"grad_norm": 0.14372963375721587,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0027,
|
|
"reward": 0.7566964477300644,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.7566964477300644,
|
|
"step": 820
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.9643096923828,
|
|
"epoch": 4.188295165394402,
|
|
"grad_norm": 0.2023098467286473,
|
|
"kl": 0.036529541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0014,
|
|
"reward": 0.8660714477300644,
|
|
"reward_std": 0.04602780845016241,
|
|
"rewards/accuracy_reward": 0.8660714477300644,
|
|
"step": 821
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 377.4553756713867,
|
|
"epoch": 4.193384223918575,
|
|
"grad_norm": 0.3206684638226072,
|
|
"kl": 0.0447998046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0134,
|
|
"reward": 0.7700893133878708,
|
|
"reward_std": 0.11417082417756319,
|
|
"rewards/accuracy_reward": 0.7700893133878708,
|
|
"step": 822
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.52457427978516,
|
|
"epoch": 4.198473282442748,
|
|
"grad_norm": 0.32984764088242763,
|
|
"kl": 0.042755126953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0066,
|
|
"reward": 0.7946428954601288,
|
|
"reward_std": 0.06935327127575874,
|
|
"rewards/accuracy_reward": 0.7946428954601288,
|
|
"step": 823
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 368.04466247558594,
|
|
"epoch": 4.203562340966921,
|
|
"grad_norm": 0.22933184868143322,
|
|
"kl": 0.03912353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0132,
|
|
"reward": 0.7544643133878708,
|
|
"reward_std": 0.07796893268823624,
|
|
"rewards/accuracy_reward": 0.7544643133878708,
|
|
"step": 824
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 381.67189025878906,
|
|
"epoch": 4.208651399491094,
|
|
"grad_norm": 0.12470572414557056,
|
|
"kl": 0.0421142578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.854910746216774,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.854910746216774,
|
|
"step": 825
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.5960006713867,
|
|
"epoch": 4.213740458015267,
|
|
"grad_norm": 0.17091613965943417,
|
|
"kl": 0.035675048828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0081,
|
|
"reward": 0.8303571790456772,
|
|
"reward_std": 0.026025486178696156,
|
|
"rewards/accuracy_reward": 0.8303571790456772,
|
|
"step": 826
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.8973388671875,
|
|
"epoch": 4.21882951653944,
|
|
"grad_norm": 0.20923767153497144,
|
|
"kl": 0.0361328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0039,
|
|
"reward": 0.8995535969734192,
|
|
"reward_std": 0.03898446634411812,
|
|
"rewards/accuracy_reward": 0.8995535969734192,
|
|
"step": 827
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 389.24108123779297,
|
|
"epoch": 4.223918575063613,
|
|
"grad_norm": 0.2808827694307612,
|
|
"kl": 0.03924560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0028,
|
|
"reward": 0.9062500298023224,
|
|
"reward_std": 0.0960018690675497,
|
|
"rewards/accuracy_reward": 0.9062500298023224,
|
|
"step": 828
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.57814025878906,
|
|
"epoch": 4.229007633587786,
|
|
"grad_norm": 0.15295648452446617,
|
|
"kl": 0.038818359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0068,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.049800973385572433,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 829
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.3348388671875,
|
|
"epoch": 4.234096692111959,
|
|
"grad_norm": 0.16441625147372738,
|
|
"kl": 0.03814697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0052,
|
|
"reward": 0.854910746216774,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.854910746216774,
|
|
"step": 830
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.2053756713867,
|
|
"epoch": 4.239185750636132,
|
|
"grad_norm": 0.17638098777267436,
|
|
"kl": 0.036163330078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.8080357611179352,
|
|
"reward_std": 0.03552421275526285,
|
|
"rewards/accuracy_reward": 0.8080357611179352,
|
|
"step": 831
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.2567138671875,
|
|
"epoch": 4.244274809160306,
|
|
"grad_norm": 0.20975447502420955,
|
|
"kl": 0.03900146484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.7053571790456772,
|
|
"reward_std": 0.05985454376786947,
|
|
"rewards/accuracy_reward": 0.7053571790456772,
|
|
"step": 832
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.8750228881836,
|
|
"epoch": 4.249363867684479,
|
|
"grad_norm": 0.3182054604354566,
|
|
"kl": 0.04150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0093,
|
|
"reward": 0.7120535969734192,
|
|
"reward_std": 0.09380373358726501,
|
|
"rewards/accuracy_reward": 0.7120535969734192,
|
|
"step": 833
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.20091247558594,
|
|
"epoch": 4.254452926208652,
|
|
"grad_norm": 0.025211036575571052,
|
|
"kl": 0.033203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.7857143133878708,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.7857143133878708,
|
|
"step": 834
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.10939025878906,
|
|
"epoch": 4.259541984732825,
|
|
"grad_norm": 0.2525774392439054,
|
|
"kl": 0.043548583984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0032,
|
|
"reward": 0.714285746216774,
|
|
"reward_std": 0.0814228504896164,
|
|
"rewards/accuracy_reward": 0.714285746216774,
|
|
"step": 835
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.1272506713867,
|
|
"epoch": 4.264631043256998,
|
|
"grad_norm": 0.16986818196488973,
|
|
"kl": 0.03411865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.003,
|
|
"reward": 0.7031250298023224,
|
|
"reward_std": 0.03495405800640583,
|
|
"rewards/accuracy_reward": 0.7031250298023224,
|
|
"step": 836
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.1071548461914,
|
|
"epoch": 4.269720101781171,
|
|
"grad_norm": 0.22412542114448175,
|
|
"kl": 0.04412841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0023,
|
|
"reward": 0.8125000298023224,
|
|
"reward_std": 0.04118260461837053,
|
|
"rewards/accuracy_reward": 0.8125000298023224,
|
|
"step": 837
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.81920623779297,
|
|
"epoch": 4.2748091603053435,
|
|
"grad_norm": 0.2180672815383536,
|
|
"kl": 0.04071044921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.7812500298023224,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.7812500298023224,
|
|
"step": 838
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 376.39957427978516,
|
|
"epoch": 4.2798982188295165,
|
|
"grad_norm": 0.20833110755586887,
|
|
"kl": 0.044189453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0027,
|
|
"reward": 0.808035746216774,
|
|
"reward_std": 0.04118260554969311,
|
|
"rewards/accuracy_reward": 0.808035746216774,
|
|
"step": 839
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.4866256713867,
|
|
"epoch": 4.2849872773536894,
|
|
"grad_norm": 0.234711234213088,
|
|
"kl": 0.03778076171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.848214328289032,
|
|
"reward_std": 0.0642497232183814,
|
|
"rewards/accuracy_reward": 0.848214328289032,
|
|
"step": 840
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.42189025878906,
|
|
"epoch": 4.290076335877862,
|
|
"grad_norm": 0.24843309609148229,
|
|
"kl": 0.0455322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.7276785969734192,
|
|
"reward_std": 0.05728259216994047,
|
|
"rewards/accuracy_reward": 0.7276785969734192,
|
|
"step": 841
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.5089416503906,
|
|
"epoch": 4.295165394402035,
|
|
"grad_norm": 0.21050577592264016,
|
|
"kl": 0.039642333984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0138,
|
|
"reward": 0.7098214477300644,
|
|
"reward_std": 0.04980097245424986,
|
|
"rewards/accuracy_reward": 0.7098214477300644,
|
|
"step": 842
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.1049270629883,
|
|
"epoch": 4.300254452926208,
|
|
"grad_norm": 0.18409441357947398,
|
|
"kl": 0.03839111328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.8125000447034836,
|
|
"reward_std": 0.023325460962951183,
|
|
"rewards/accuracy_reward": 0.8125000447034836,
|
|
"step": 843
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.07814025878906,
|
|
"epoch": 4.305343511450381,
|
|
"grad_norm": 0.1467437806973369,
|
|
"kl": 0.0455322265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0024,
|
|
"reward": 0.7767857611179352,
|
|
"reward_std": 0.023325460962951183,
|
|
"rewards/accuracy_reward": 0.7767857611179352,
|
|
"step": 844
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.3147506713867,
|
|
"epoch": 4.310432569974555,
|
|
"grad_norm": 0.29829936191823747,
|
|
"kl": 0.04718017578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.863839328289032,
|
|
"reward_std": 0.07909663487225771,
|
|
"rewards/accuracy_reward": 0.863839328289032,
|
|
"step": 845
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.1138610839844,
|
|
"epoch": 4.315521628498728,
|
|
"grad_norm": 0.23707148778083859,
|
|
"kl": 0.04046630859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0066,
|
|
"reward": 0.7857143133878708,
|
|
"reward_std": 0.05539736431092024,
|
|
"rewards/accuracy_reward": 0.7857143133878708,
|
|
"step": 846
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.2835006713867,
|
|
"epoch": 4.320610687022901,
|
|
"grad_norm": 0.20599593766122942,
|
|
"kl": 0.0306396484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0037,
|
|
"reward": 0.9308036118745804,
|
|
"reward_std": 0.055542015470564365,
|
|
"rewards/accuracy_reward": 0.9308036118745804,
|
|
"step": 847
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 427.47322845458984,
|
|
"epoch": 4.325699745547074,
|
|
"grad_norm": 0.02636438452379431,
|
|
"kl": 0.03277587890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.7142857313156128,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.7142857313156128,
|
|
"step": 848
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.9754638671875,
|
|
"epoch": 4.330788804071247,
|
|
"grad_norm": 0.19087436827216261,
|
|
"kl": 0.0419921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.7209821715950966,
|
|
"reward_std": 0.03898446634411812,
|
|
"rewards/accuracy_reward": 0.7209821715950966,
|
|
"step": 849
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 368.35939025878906,
|
|
"epoch": 4.33587786259542,
|
|
"grad_norm": 0.2088613027630745,
|
|
"kl": 0.0362548828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0035,
|
|
"reward": 0.9196428954601288,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.9196428954601288,
|
|
"step": 850
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.8549270629883,
|
|
"epoch": 4.340966921119593,
|
|
"grad_norm": 0.17342645567006615,
|
|
"kl": 0.039215087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0058,
|
|
"reward": 0.8571428954601288,
|
|
"reward_std": 0.04199740011245012,
|
|
"rewards/accuracy_reward": 0.8571428954601288,
|
|
"step": 851
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 372.8236770629883,
|
|
"epoch": 4.346055979643766,
|
|
"grad_norm": 0.3317254897457984,
|
|
"kl": 0.04852294921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0061,
|
|
"reward": 0.76339291036129,
|
|
"reward_std": 0.07110932748764753,
|
|
"rewards/accuracy_reward": 0.76339291036129,
|
|
"step": 852
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.29466247558594,
|
|
"epoch": 4.351145038167939,
|
|
"grad_norm": 0.22875196579000068,
|
|
"kl": 0.04150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0018,
|
|
"reward": 0.7031250298023224,
|
|
"reward_std": 0.05262112617492676,
|
|
"rewards/accuracy_reward": 0.7031250298023224,
|
|
"step": 853
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.44197845458984,
|
|
"epoch": 4.356234096692112,
|
|
"grad_norm": 0.20864766284694214,
|
|
"kl": 0.037567138671875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.7723214626312256,
|
|
"reward_std": 0.0357142873108387,
|
|
"rewards/accuracy_reward": 0.7723214626312256,
|
|
"step": 854
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 413.02010345458984,
|
|
"epoch": 4.361323155216285,
|
|
"grad_norm": 0.24338668900360855,
|
|
"kl": 0.04296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0046,
|
|
"reward": 0.7991071939468384,
|
|
"reward_std": 0.07771168742328882,
|
|
"rewards/accuracy_reward": 0.7991071939468384,
|
|
"step": 855
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.0491180419922,
|
|
"epoch": 4.366412213740458,
|
|
"grad_norm": 0.17575756479187551,
|
|
"kl": 0.04254150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.7522321790456772,
|
|
"reward_std": 0.030368803068995476,
|
|
"rewards/accuracy_reward": 0.7522321790456772,
|
|
"step": 856
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.1071548461914,
|
|
"epoch": 4.371501272264631,
|
|
"grad_norm": 0.23331238391328157,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.8995535969734192,
|
|
"reward_std": 0.03898446727544069,
|
|
"rewards/accuracy_reward": 0.8995535969734192,
|
|
"step": 857
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.4910888671875,
|
|
"epoch": 4.376590330788804,
|
|
"grad_norm": 0.2336125881174906,
|
|
"kl": 0.043701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.004,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.07056996412575245,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 858
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.8884048461914,
|
|
"epoch": 4.3816793893129775,
|
|
"grad_norm": 0.25928277167365654,
|
|
"kl": 0.038299560546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.7232143208384514,
|
|
"reward_std": 0.07979486510157585,
|
|
"rewards/accuracy_reward": 0.7232143208384514,
|
|
"step": 859
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 375.59376525878906,
|
|
"epoch": 4.3867684478371505,
|
|
"grad_norm": 0.24894523001997013,
|
|
"kl": 0.03948974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.8325893133878708,
|
|
"reward_std": 0.07672229409217834,
|
|
"rewards/accuracy_reward": 0.8325893133878708,
|
|
"step": 860
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.9955520629883,
|
|
"epoch": 4.391857506361323,
|
|
"grad_norm": 0.19577810123274583,
|
|
"kl": 0.031036376953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.6852678954601288,
|
|
"reward_std": 0.03495405800640583,
|
|
"rewards/accuracy_reward": 0.6852678954601288,
|
|
"step": 861
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 370.6964416503906,
|
|
"epoch": 4.396946564885496,
|
|
"grad_norm": 0.20197536286494003,
|
|
"kl": 0.0345458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0055,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.047152808867394924,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 862
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.18082427978516,
|
|
"epoch": 4.402035623409669,
|
|
"grad_norm": 0.23976335187052797,
|
|
"kl": 0.03790283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0078,
|
|
"reward": 0.848214328289032,
|
|
"reward_std": 0.06154969893395901,
|
|
"rewards/accuracy_reward": 0.848214328289032,
|
|
"step": 863
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.3772506713867,
|
|
"epoch": 4.407124681933842,
|
|
"grad_norm": 0.26654222786104065,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.8504464775323868,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.8504464775323868,
|
|
"step": 864
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.76341247558594,
|
|
"epoch": 4.412213740458015,
|
|
"grad_norm": 0.1967512921259146,
|
|
"kl": 0.041778564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0029,
|
|
"reward": 0.8058036118745804,
|
|
"reward_std": 0.03495405800640583,
|
|
"rewards/accuracy_reward": 0.8058036118745804,
|
|
"step": 865
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.3593978881836,
|
|
"epoch": 4.417302798982188,
|
|
"grad_norm": 0.09163076895436262,
|
|
"kl": 0.030914306640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0049,
|
|
"reward": 0.993303582072258,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.993303582072258,
|
|
"step": 866
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.2522506713867,
|
|
"epoch": 4.422391857506361,
|
|
"grad_norm": 0.2931703029014787,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0146,
|
|
"reward": 0.850446492433548,
|
|
"reward_std": 0.08977332152426243,
|
|
"rewards/accuracy_reward": 0.850446492433548,
|
|
"step": 867
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.7522506713867,
|
|
"epoch": 4.427480916030534,
|
|
"grad_norm": 0.1968515926497254,
|
|
"kl": 0.0335693359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.8035714775323868,
|
|
"reward_std": 0.039297375828027725,
|
|
"rewards/accuracy_reward": 0.8035714775323868,
|
|
"step": 868
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.0268096923828,
|
|
"epoch": 4.432569974554707,
|
|
"grad_norm": 0.286430818687198,
|
|
"kl": 0.04132080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0035,
|
|
"reward": 0.636160746216774,
|
|
"reward_std": 0.07733787596225739,
|
|
"rewards/accuracy_reward": 0.636160746216774,
|
|
"step": 869
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.36832427978516,
|
|
"epoch": 4.43765903307888,
|
|
"grad_norm": 0.2341059195427911,
|
|
"kl": 0.03662109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0115,
|
|
"reward": 0.783482164144516,
|
|
"reward_std": 0.065534682944417,
|
|
"rewards/accuracy_reward": 0.783482164144516,
|
|
"step": 870
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 396.6919860839844,
|
|
"epoch": 4.442748091603053,
|
|
"grad_norm": 0.11970007418476948,
|
|
"kl": 0.04150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.01709691435098648,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 871
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.73885345458984,
|
|
"epoch": 4.447837150127226,
|
|
"grad_norm": 0.31282478039226147,
|
|
"kl": 0.0458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0098,
|
|
"reward": 0.698660746216774,
|
|
"reward_std": 0.09500494413077831,
|
|
"rewards/accuracy_reward": 0.698660746216774,
|
|
"step": 872
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 431.75894927978516,
|
|
"epoch": 4.4529262086514,
|
|
"grad_norm": 0.24356834862212692,
|
|
"kl": 0.03485107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0031,
|
|
"reward": 0.7500000447034836,
|
|
"reward_std": 0.04791303910315037,
|
|
"rewards/accuracy_reward": 0.7500000447034836,
|
|
"step": 873
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.5000228881836,
|
|
"epoch": 4.458015267175573,
|
|
"grad_norm": 0.1360092734005078,
|
|
"kl": 0.03814697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0017,
|
|
"reward": 0.7209821790456772,
|
|
"reward_std": 0.014396889135241508,
|
|
"rewards/accuracy_reward": 0.7209821790456772,
|
|
"step": 874
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.7321548461914,
|
|
"epoch": 4.463104325699746,
|
|
"grad_norm": 0.1426407133224715,
|
|
"kl": 0.03326416015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0014,
|
|
"reward": 0.8437500298023224,
|
|
"reward_std": 0.02817066479474306,
|
|
"rewards/accuracy_reward": 0.8437500298023224,
|
|
"step": 875
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.80358123779297,
|
|
"epoch": 4.468193384223919,
|
|
"grad_norm": 0.12763067916860432,
|
|
"kl": 0.037017822265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.8125000149011612,
|
|
"reward_std": 0.023325461894273758,
|
|
"rewards/accuracy_reward": 0.8125000149011612,
|
|
"step": 876
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 419.27903747558594,
|
|
"epoch": 4.473282442748092,
|
|
"grad_norm": 0.11032439569795521,
|
|
"kl": 0.03985595703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.7120536118745804,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.7120536118745804,
|
|
"step": 877
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.1093978881836,
|
|
"epoch": 4.478371501272265,
|
|
"grad_norm": 0.2320263882725396,
|
|
"kl": 0.0384521484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0034,
|
|
"reward": 0.7901785969734192,
|
|
"reward_std": 0.09605608507990837,
|
|
"rewards/accuracy_reward": 0.7901785969734192,
|
|
"step": 878
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.83038330078125,
|
|
"epoch": 4.483460559796438,
|
|
"grad_norm": 0.21192156496262965,
|
|
"kl": 0.04827880859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0045,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.036299921572208405,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 879
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.55135345458984,
|
|
"epoch": 4.488549618320611,
|
|
"grad_norm": 0.25738803265288984,
|
|
"kl": 0.031890869140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.8906250447034836,
|
|
"reward_std": 0.08501227665692568,
|
|
"rewards/accuracy_reward": 0.8906250447034836,
|
|
"step": 880
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.48663330078125,
|
|
"epoch": 4.493638676844784,
|
|
"grad_norm": 0.27092509914408425,
|
|
"kl": 0.035369873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.8571428805589676,
|
|
"reward_std": 0.054453397169709206,
|
|
"rewards/accuracy_reward": 0.8571428805589676,
|
|
"step": 881
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.95091247558594,
|
|
"epoch": 4.4987277353689565,
|
|
"grad_norm": 0.24018205422589223,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0064,
|
|
"reward": 0.8258928805589676,
|
|
"reward_std": 0.07589362747967243,
|
|
"rewards/accuracy_reward": 0.8258928805589676,
|
|
"step": 882
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.5156478881836,
|
|
"epoch": 4.5038167938931295,
|
|
"grad_norm": 0.23727106426883504,
|
|
"kl": 0.03857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.7232143059372902,
|
|
"reward_std": 0.05149612668901682,
|
|
"rewards/accuracy_reward": 0.7232143059372902,
|
|
"step": 883
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.11608123779297,
|
|
"epoch": 4.5089058524173025,
|
|
"grad_norm": 0.13483502286872068,
|
|
"kl": 0.037078857421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.001,
|
|
"reward": 0.7879464626312256,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.7879464626312256,
|
|
"step": 884
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 375.12054443359375,
|
|
"epoch": 4.513994910941475,
|
|
"grad_norm": 0.20034625013776441,
|
|
"kl": 0.038543701171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0,
|
|
"reward": 0.7566964477300644,
|
|
"reward_std": 0.060552770271897316,
|
|
"rewards/accuracy_reward": 0.7566964477300644,
|
|
"step": 885
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.7857360839844,
|
|
"epoch": 4.519083969465649,
|
|
"grad_norm": 0.19729643597777208,
|
|
"kl": 0.04119873046875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0011,
|
|
"reward": 0.6272321864962578,
|
|
"reward_std": 0.03772235009819269,
|
|
"rewards/accuracy_reward": 0.6272321864962578,
|
|
"step": 886
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.87501525878906,
|
|
"epoch": 4.524173027989822,
|
|
"grad_norm": 0.18801744726938205,
|
|
"kl": 0.03948974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.8616071939468384,
|
|
"reward_std": 0.056020475924015045,
|
|
"rewards/accuracy_reward": 0.8616071939468384,
|
|
"step": 887
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.9955520629883,
|
|
"epoch": 4.529262086513995,
|
|
"grad_norm": 0.19562726001505468,
|
|
"kl": 0.03466796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.8839286267757416,
|
|
"reward_std": 0.05445339623838663,
|
|
"rewards/accuracy_reward": 0.8839286267757416,
|
|
"step": 888
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.70760345458984,
|
|
"epoch": 4.534351145038168,
|
|
"grad_norm": 0.17042521269209113,
|
|
"kl": 0.03570556640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0041,
|
|
"reward": 0.8258928954601288,
|
|
"reward_std": 0.03149380348622799,
|
|
"rewards/accuracy_reward": 0.8258928954601288,
|
|
"step": 889
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.9643096923828,
|
|
"epoch": 4.539440203562341,
|
|
"grad_norm": 0.22965700334950423,
|
|
"kl": 0.04071044921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.6473214477300644,
|
|
"reward_std": 0.06532286200672388,
|
|
"rewards/accuracy_reward": 0.6473214477300644,
|
|
"step": 890
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 441.0245666503906,
|
|
"epoch": 4.544529262086514,
|
|
"grad_norm": 0.25719299513895816,
|
|
"kl": 0.03466796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0024,
|
|
"reward": 0.761160746216774,
|
|
"reward_std": 0.11224650964140892,
|
|
"rewards/accuracy_reward": 0.761160746216774,
|
|
"step": 891
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.10269927978516,
|
|
"epoch": 4.549618320610687,
|
|
"grad_norm": 0.11139812071369241,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.8504464775323868,
|
|
"reward_std": 0.014396889135241508,
|
|
"rewards/accuracy_reward": 0.8504464775323868,
|
|
"step": 892
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.9486770629883,
|
|
"epoch": 4.55470737913486,
|
|
"grad_norm": 0.23697728587123176,
|
|
"kl": 0.036773681640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.9062500447034836,
|
|
"reward_std": 0.053571431897580624,
|
|
"rewards/accuracy_reward": 0.9062500447034836,
|
|
"step": 893
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.9085006713867,
|
|
"epoch": 4.559796437659033,
|
|
"grad_norm": 0.07626423571726532,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0007,
|
|
"reward": 0.8906250298023224,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.8906250298023224,
|
|
"step": 894
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 429.55135345458984,
|
|
"epoch": 4.564885496183206,
|
|
"grad_norm": 0.19714519917345338,
|
|
"kl": 0.032379150390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0075,
|
|
"reward": 0.8058036118745804,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.8058036118745804,
|
|
"step": 895
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.3192138671875,
|
|
"epoch": 4.569974554707379,
|
|
"grad_norm": 0.288336479461512,
|
|
"kl": 0.03863525390625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0023,
|
|
"reward": 0.7098214626312256,
|
|
"reward_std": 0.05822656024247408,
|
|
"rewards/accuracy_reward": 0.7098214626312256,
|
|
"step": 896
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.8660888671875,
|
|
"epoch": 4.575063613231552,
|
|
"grad_norm": 0.3599363849448018,
|
|
"kl": 0.04449462890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0061,
|
|
"reward": 0.7745535969734192,
|
|
"reward_std": 0.07652468606829643,
|
|
"rewards/accuracy_reward": 0.7745535969734192,
|
|
"step": 897
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.87278747558594,
|
|
"epoch": 4.580152671755725,
|
|
"grad_norm": 0.14609726333330406,
|
|
"kl": 0.032073974609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0019,
|
|
"reward": 0.8348214626312256,
|
|
"reward_std": 0.026025486178696156,
|
|
"rewards/accuracy_reward": 0.8348214626312256,
|
|
"step": 898
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 415.6205520629883,
|
|
"epoch": 4.585241730279899,
|
|
"grad_norm": 0.12916363110190998,
|
|
"kl": 0.037841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0025,
|
|
"reward": 0.7321428954601288,
|
|
"reward_std": 0.018442777916789055,
|
|
"rewards/accuracy_reward": 0.7321428954601288,
|
|
"step": 899
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.19420623779297,
|
|
"epoch": 4.590330788804071,
|
|
"grad_norm": 0.28617106879822374,
|
|
"kl": 0.0439453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0042,
|
|
"reward": 0.8638393133878708,
|
|
"reward_std": 0.07984782662242651,
|
|
"rewards/accuracy_reward": 0.8638393133878708,
|
|
"step": 900
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.13394927978516,
|
|
"epoch": 4.595419847328245,
|
|
"grad_norm": 0.10636725500810751,
|
|
"kl": 0.03497314453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0024,
|
|
"reward": 0.8705357313156128,
|
|
"reward_std": 0.03064152877777815,
|
|
"rewards/accuracy_reward": 0.8705357313156128,
|
|
"step": 901
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 383.54913330078125,
|
|
"epoch": 4.600508905852418,
|
|
"grad_norm": 0.24339070909937904,
|
|
"kl": 0.0423583984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0076,
|
|
"reward": 0.863839328289032,
|
|
"reward_std": 0.06494904682040215,
|
|
"rewards/accuracy_reward": 0.863839328289032,
|
|
"step": 902
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.40403747558594,
|
|
"epoch": 4.6055979643765905,
|
|
"grad_norm": 0.20227152112236466,
|
|
"kl": 0.033447265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.8727678805589676,
|
|
"reward_std": 0.030496877618134022,
|
|
"rewards/accuracy_reward": 0.8727678805589676,
|
|
"step": 903
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 416.9486770629883,
|
|
"epoch": 4.6106870229007635,
|
|
"grad_norm": 0.16361222563326544,
|
|
"kl": 0.03900146484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.9218750298023224,
|
|
"reward_std": 0.026785715483129025,
|
|
"rewards/accuracy_reward": 0.9218750298023224,
|
|
"step": 904
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.8951110839844,
|
|
"epoch": 4.6157760814249365,
|
|
"grad_norm": 0.2266848944786614,
|
|
"kl": 0.047119140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.6808036118745804,
|
|
"reward_std": 0.03898446634411812,
|
|
"rewards/accuracy_reward": 0.6808036118745804,
|
|
"step": 905
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.9018096923828,
|
|
"epoch": 4.620865139949109,
|
|
"grad_norm": 0.2179588960514838,
|
|
"kl": 0.0423583984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.8348214626312256,
|
|
"reward_std": 0.027226698584854603,
|
|
"rewards/accuracy_reward": 0.8348214626312256,
|
|
"step": 906
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.7745666503906,
|
|
"epoch": 4.625954198473282,
|
|
"grad_norm": 0.34057522082116987,
|
|
"kl": 0.042022705078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0029,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.049350948072969913,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 907
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.3973388671875,
|
|
"epoch": 4.631043256997455,
|
|
"grad_norm": 0.18572980686580934,
|
|
"kl": 0.04290771484375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.848214328289032,
|
|
"reward_std": 0.03552421182394028,
|
|
"rewards/accuracy_reward": 0.848214328289032,
|
|
"step": 908
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.79689025878906,
|
|
"epoch": 4.636132315521628,
|
|
"grad_norm": 0.19450772268488836,
|
|
"kl": 0.04345703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0031,
|
|
"reward": 0.707589328289032,
|
|
"reward_std": 0.037099236622452736,
|
|
"rewards/accuracy_reward": 0.707589328289032,
|
|
"step": 909
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.9955520629883,
|
|
"epoch": 4.641221374045801,
|
|
"grad_norm": 0.2464243407356205,
|
|
"kl": 0.032684326171875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0113,
|
|
"reward": 0.8705357313156128,
|
|
"reward_std": 0.056081379763782024,
|
|
"rewards/accuracy_reward": 0.8705357313156128,
|
|
"step": 910
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.4933166503906,
|
|
"epoch": 4.646310432569974,
|
|
"grad_norm": 0.19269356120770217,
|
|
"kl": 0.03448486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0031,
|
|
"reward": 0.7455357611179352,
|
|
"reward_std": 0.05256816744804382,
|
|
"rewards/accuracy_reward": 0.7455357611179352,
|
|
"step": 911
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.3482360839844,
|
|
"epoch": 4.651399491094148,
|
|
"grad_norm": 0.1859106335445496,
|
|
"kl": 0.037506103515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0026,
|
|
"reward": 0.7901786118745804,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.7901786118745804,
|
|
"step": 912
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.24332427978516,
|
|
"epoch": 4.65648854961832,
|
|
"grad_norm": 0.2648912200352862,
|
|
"kl": 0.038330078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0038,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.0806080549955368,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 913
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 428.8236846923828,
|
|
"epoch": 4.661577608142494,
|
|
"grad_norm": 0.1829319708419935,
|
|
"kl": 0.04278564453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0013,
|
|
"reward": 0.7812500447034836,
|
|
"reward_std": 0.047465717419981956,
|
|
"rewards/accuracy_reward": 0.7812500447034836,
|
|
"step": 914
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 384.9464416503906,
|
|
"epoch": 4.666666666666667,
|
|
"grad_norm": 0.2926283781432313,
|
|
"kl": 0.04461669921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0008,
|
|
"reward": 0.7366071790456772,
|
|
"reward_std": 0.09345444105565548,
|
|
"rewards/accuracy_reward": 0.7366071790456772,
|
|
"step": 915
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.5714416503906,
|
|
"epoch": 4.67175572519084,
|
|
"grad_norm": 0.22387212250904767,
|
|
"kl": 0.04425048828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0061,
|
|
"reward": 0.6138393133878708,
|
|
"reward_std": 0.057595500722527504,
|
|
"rewards/accuracy_reward": 0.6138393133878708,
|
|
"step": 916
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.3326110839844,
|
|
"epoch": 4.676844783715013,
|
|
"grad_norm": 0.26370251833787844,
|
|
"kl": 0.04241943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0024,
|
|
"reward": 0.821428582072258,
|
|
"reward_std": 0.07764968648552895,
|
|
"rewards/accuracy_reward": 0.821428582072258,
|
|
"step": 917
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.5446472167969,
|
|
"epoch": 4.681933842239186,
|
|
"grad_norm": 0.25080278769285724,
|
|
"kl": 0.03936767578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0012,
|
|
"reward": 0.7321428954601288,
|
|
"reward_std": 0.07716327719390392,
|
|
"rewards/accuracy_reward": 0.7321428954601288,
|
|
"step": 918
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.2611846923828,
|
|
"epoch": 4.687022900763359,
|
|
"grad_norm": 0.20208596526498823,
|
|
"kl": 0.03656005859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0037,
|
|
"reward": 0.8616071939468384,
|
|
"reward_std": 0.05162420030683279,
|
|
"rewards/accuracy_reward": 0.8616071939468384,
|
|
"step": 919
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 434.10716247558594,
|
|
"epoch": 4.692111959287532,
|
|
"grad_norm": 0.30049536325985265,
|
|
"kl": 0.042266845703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0022,
|
|
"reward": 0.7834821790456772,
|
|
"reward_std": 0.06885028537362814,
|
|
"rewards/accuracy_reward": 0.7834821790456772,
|
|
"step": 920
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 417.13841247558594,
|
|
"epoch": 4.697201017811705,
|
|
"grad_norm": 0.2683491526428907,
|
|
"kl": 0.0406494140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.7879464775323868,
|
|
"reward_std": 0.0731173912063241,
|
|
"rewards/accuracy_reward": 0.7879464775323868,
|
|
"step": 921
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.92859649658203,
|
|
"epoch": 4.702290076335878,
|
|
"grad_norm": 0.22634896088942552,
|
|
"kl": 0.0382080078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0002,
|
|
"reward": 0.8549107760190964,
|
|
"reward_std": 0.06764907389879227,
|
|
"rewards/accuracy_reward": 0.8549107760190964,
|
|
"step": 922
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.20091247558594,
|
|
"epoch": 4.707379134860051,
|
|
"grad_norm": 0.16277773296573514,
|
|
"kl": 0.037017822265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.003,
|
|
"reward": 0.7723214775323868,
|
|
"reward_std": 0.047465719282627106,
|
|
"rewards/accuracy_reward": 0.7723214775323868,
|
|
"step": 923
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 394.81251525878906,
|
|
"epoch": 4.712468193384224,
|
|
"grad_norm": 0.12720180950778365,
|
|
"kl": 0.034912109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.6763393133878708,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.6763393133878708,
|
|
"step": 924
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 412.18082427978516,
|
|
"epoch": 4.717557251908397,
|
|
"grad_norm": 0.11652032046572212,
|
|
"kl": 0.04071044921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0002,
|
|
"reward": 0.7879464477300644,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.7879464477300644,
|
|
"step": 925
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.50447845458984,
|
|
"epoch": 4.7226463104325695,
|
|
"grad_norm": 0.21457645604463352,
|
|
"kl": 0.035308837890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0047,
|
|
"reward": 0.95089291036129,
|
|
"reward_std": 0.03552421182394028,
|
|
"rewards/accuracy_reward": 0.95089291036129,
|
|
"step": 926
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 418.89957427978516,
|
|
"epoch": 4.727735368956743,
|
|
"grad_norm": 0.15558747985672983,
|
|
"kl": 0.03607177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.001,
|
|
"reward": 0.7388393133878708,
|
|
"reward_std": 0.024900486692786217,
|
|
"rewards/accuracy_reward": 0.7388393133878708,
|
|
"step": 927
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 387.4933166503906,
|
|
"epoch": 4.732824427480916,
|
|
"grad_norm": 0.1597714283291237,
|
|
"kl": 0.0367431640625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0055,
|
|
"reward": 0.776785746216774,
|
|
"reward_std": 0.03552421182394028,
|
|
"rewards/accuracy_reward": 0.776785746216774,
|
|
"step": 928
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.53795623779297,
|
|
"epoch": 4.737913486005089,
|
|
"grad_norm": 0.07468168970450847,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0043,
|
|
"reward": 0.8191964775323868,
|
|
"reward_std": 0.008928571827709675,
|
|
"rewards/accuracy_reward": 0.8191964775323868,
|
|
"step": 929
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 385.9375228881836,
|
|
"epoch": 4.743002544529262,
|
|
"grad_norm": 0.2934059460481899,
|
|
"kl": 0.0416259765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0021,
|
|
"reward": 0.658482164144516,
|
|
"reward_std": 0.06432593520730734,
|
|
"rewards/accuracy_reward": 0.658482164144516,
|
|
"step": 930
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 392.9442138671875,
|
|
"epoch": 4.748091603053435,
|
|
"grad_norm": 0.22240959310546268,
|
|
"kl": 0.0391845703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0026,
|
|
"reward": 0.9196428656578064,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.9196428656578064,
|
|
"step": 931
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.7120666503906,
|
|
"epoch": 4.753180661577608,
|
|
"grad_norm": 0.2358872098669444,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.8013393208384514,
|
|
"reward_std": 0.04715280793607235,
|
|
"rewards/accuracy_reward": 0.8013393208384514,
|
|
"step": 932
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 380.70760345458984,
|
|
"epoch": 4.758269720101781,
|
|
"grad_norm": 0.1334731560244657,
|
|
"kl": 0.03448486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0016,
|
|
"reward": 0.7633928954601288,
|
|
"reward_std": 0.032695015892386436,
|
|
"rewards/accuracy_reward": 0.7633928954601288,
|
|
"step": 933
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.9977798461914,
|
|
"epoch": 4.763358778625954,
|
|
"grad_norm": 0.31297726198591813,
|
|
"kl": 0.04241943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0057,
|
|
"reward": 0.8102678954601288,
|
|
"reward_std": 0.09022335056215525,
|
|
"rewards/accuracy_reward": 0.8102678954601288,
|
|
"step": 934
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.9174270629883,
|
|
"epoch": 4.768447837150127,
|
|
"grad_norm": 0.026221145898374452,
|
|
"kl": 0.0355224609375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.8214286267757416,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.8214286267757416,
|
|
"step": 935
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 370.58260345458984,
|
|
"epoch": 4.7735368956743,
|
|
"grad_norm": 0.168073179995655,
|
|
"kl": 0.03997802734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0015,
|
|
"reward": 0.8169643133878708,
|
|
"reward_std": 0.01785714365541935,
|
|
"rewards/accuracy_reward": 0.8169643133878708,
|
|
"step": 936
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 422.5223388671875,
|
|
"epoch": 4.778625954198473,
|
|
"grad_norm": 0.2884104922115341,
|
|
"kl": 0.037750244140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.7522321790456772,
|
|
"reward_std": 0.10550854075700045,
|
|
"rewards/accuracy_reward": 0.7522321790456772,
|
|
"step": 937
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.6272430419922,
|
|
"epoch": 4.783715012722646,
|
|
"grad_norm": 0.2598500667086591,
|
|
"kl": 0.0487060546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0013,
|
|
"reward": 0.714285746216774,
|
|
"reward_std": 0.05985454376786947,
|
|
"rewards/accuracy_reward": 0.714285746216774,
|
|
"step": 938
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.4955520629883,
|
|
"epoch": 4.788804071246819,
|
|
"grad_norm": 0.27851473283558814,
|
|
"kl": 0.03985595703125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.8883928954601288,
|
|
"reward_std": 0.08506523165851831,
|
|
"rewards/accuracy_reward": 0.8883928954601288,
|
|
"step": 939
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.6160888671875,
|
|
"epoch": 4.793893129770993,
|
|
"grad_norm": 0.21474023419633984,
|
|
"kl": 0.034393310546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0086,
|
|
"reward": 0.8772321790456772,
|
|
"reward_std": 0.055579494684934616,
|
|
"rewards/accuracy_reward": 0.8772321790456772,
|
|
"step": 940
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.59376525878906,
|
|
"epoch": 4.798982188295166,
|
|
"grad_norm": 0.028555917958620735,
|
|
"kl": 0.03741455078125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.7857143133878708,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.7857143133878708,
|
|
"step": 941
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.28126525878906,
|
|
"epoch": 4.804071246819339,
|
|
"grad_norm": 0.1607631720795229,
|
|
"kl": 0.04486083984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0039,
|
|
"reward": 0.8370536267757416,
|
|
"reward_std": 0.018298126757144928,
|
|
"rewards/accuracy_reward": 0.8370536267757416,
|
|
"step": 942
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.9643096923828,
|
|
"epoch": 4.809160305343512,
|
|
"grad_norm": 0.2168750544382541,
|
|
"kl": 0.04241943359375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0026,
|
|
"reward": 0.7901785969734192,
|
|
"reward_std": 0.03822423703968525,
|
|
"rewards/accuracy_reward": 0.7901785969734192,
|
|
"step": 943
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.9151916503906,
|
|
"epoch": 4.814249363867685,
|
|
"grad_norm": 0.1716971817309448,
|
|
"kl": 0.03436279296875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0059,
|
|
"reward": 0.6875000298023224,
|
|
"reward_std": 0.0357142873108387,
|
|
"rewards/accuracy_reward": 0.6875000298023224,
|
|
"step": 944
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 444.6250305175781,
|
|
"epoch": 4.819338422391858,
|
|
"grad_norm": 0.1343055994024979,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.8861607760190964,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.8861607760190964,
|
|
"step": 945
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.5245819091797,
|
|
"epoch": 4.824427480916031,
|
|
"grad_norm": 0.2720922671554191,
|
|
"kl": 0.03759765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0051,
|
|
"reward": 0.7187500298023224,
|
|
"reward_std": 0.0533813564106822,
|
|
"rewards/accuracy_reward": 0.7187500298023224,
|
|
"step": 946
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.68751525878906,
|
|
"epoch": 4.8295165394402035,
|
|
"grad_norm": 0.22572044314359166,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.004,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 947
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 410.9085006713867,
|
|
"epoch": 4.8346055979643765,
|
|
"grad_norm": 0.026522899248282012,
|
|
"kl": 0.03717041015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.8214286118745804,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.8214286118745804,
|
|
"step": 948
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.3125228881836,
|
|
"epoch": 4.8396946564885495,
|
|
"grad_norm": 0.29020971281239255,
|
|
"kl": 0.039581298828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0063,
|
|
"reward": 0.6941964626312256,
|
|
"reward_std": 0.07047826983034611,
|
|
"rewards/accuracy_reward": 0.6941964626312256,
|
|
"step": 949
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 388.8973388671875,
|
|
"epoch": 4.844783715012722,
|
|
"grad_norm": 0.1472612735975997,
|
|
"kl": 0.0362548828125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0001,
|
|
"reward": 0.8526785969734192,
|
|
"reward_std": 0.01785714365541935,
|
|
"rewards/accuracy_reward": 0.8526785969734192,
|
|
"step": 950
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 447.2388687133789,
|
|
"epoch": 4.849872773536895,
|
|
"grad_norm": 0.2380951682550517,
|
|
"kl": 0.03533935546875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0037,
|
|
"reward": 0.7678571790456772,
|
|
"reward_std": 0.07091925386339426,
|
|
"rewards/accuracy_reward": 0.7678571790456772,
|
|
"step": 951
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 409.0848388671875,
|
|
"epoch": 4.854961832061068,
|
|
"grad_norm": 0.2303177194221303,
|
|
"kl": 0.039794921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.002,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.0458906926214695,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 952
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.20538330078125,
|
|
"epoch": 4.860050890585242,
|
|
"grad_norm": 0.1625724698535417,
|
|
"kl": 0.03607177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0015,
|
|
"reward": 0.6651785969734192,
|
|
"reward_std": 0.026025486178696156,
|
|
"rewards/accuracy_reward": 0.6651785969734192,
|
|
"step": 953
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 388.0379638671875,
|
|
"epoch": 4.865139949109414,
|
|
"grad_norm": 0.1632224309248044,
|
|
"kl": 0.03607177734375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0033,
|
|
"reward": 0.870535746216774,
|
|
"reward_std": 0.027226698584854603,
|
|
"rewards/accuracy_reward": 0.870535746216774,
|
|
"step": 954
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 393.2098388671875,
|
|
"epoch": 4.870229007633588,
|
|
"grad_norm": 0.24119040391822338,
|
|
"kl": 0.036865234375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.6540178805589676,
|
|
"reward_std": 0.041494415141642094,
|
|
"rewards/accuracy_reward": 0.6540178805589676,
|
|
"step": 955
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 407.7031478881836,
|
|
"epoch": 4.875318066157761,
|
|
"grad_norm": 0.16184515316598308,
|
|
"kl": 0.03790283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0024,
|
|
"reward": 0.8705357313156128,
|
|
"reward_std": 0.04118260554969311,
|
|
"rewards/accuracy_reward": 0.8705357313156128,
|
|
"step": 956
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 382.06251525878906,
|
|
"epoch": 4.880407124681934,
|
|
"grad_norm": 0.29608545960046034,
|
|
"kl": 0.0374755859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.8348214775323868,
|
|
"reward_std": 0.07559729740023613,
|
|
"rewards/accuracy_reward": 0.8348214775323868,
|
|
"step": 957
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 402.70760345458984,
|
|
"epoch": 4.885496183206107,
|
|
"grad_norm": 0.3114491331748652,
|
|
"kl": 0.036529541015625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0006,
|
|
"reward": 0.7477678805589676,
|
|
"reward_std": 0.09519501589238644,
|
|
"rewards/accuracy_reward": 0.7477678805589676,
|
|
"step": 958
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 386.5893020629883,
|
|
"epoch": 4.89058524173028,
|
|
"grad_norm": 0.21215812775294332,
|
|
"kl": 0.038909912109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0112,
|
|
"reward": 0.7321428880095482,
|
|
"reward_std": 0.05055216047912836,
|
|
"rewards/accuracy_reward": 0.7321428880095482,
|
|
"step": 959
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 405.93082427978516,
|
|
"epoch": 4.895674300254453,
|
|
"grad_norm": 0.13748623377345526,
|
|
"kl": 0.037139892578125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.008,
|
|
"reward": 0.8526785969734192,
|
|
"reward_std": 0.01785714365541935,
|
|
"rewards/accuracy_reward": 0.8526785969734192,
|
|
"step": 960
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 414.26788330078125,
|
|
"epoch": 4.900763358778626,
|
|
"grad_norm": 0.20103315255542173,
|
|
"kl": 0.034149169921875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.7343750298023224,
|
|
"reward_std": 0.03709923755377531,
|
|
"rewards/accuracy_reward": 0.7343750298023224,
|
|
"step": 961
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 395.8504638671875,
|
|
"epoch": 4.905852417302799,
|
|
"grad_norm": 0.2960793778985056,
|
|
"kl": 0.037841796875,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0088,
|
|
"reward": 0.9196428805589676,
|
|
"reward_std": 0.08776525780558586,
|
|
"rewards/accuracy_reward": 0.9196428805589676,
|
|
"step": 962
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 400.05135345458984,
|
|
"epoch": 4.910941475826972,
|
|
"grad_norm": 0.5996619896909581,
|
|
"kl": 0.03887939453125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0062,
|
|
"reward": 0.7388392984867096,
|
|
"reward_std": 0.06764907389879227,
|
|
"rewards/accuracy_reward": 0.7388392984867096,
|
|
"step": 963
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 398.14510345458984,
|
|
"epoch": 4.916030534351145,
|
|
"grad_norm": 0.15519140037016188,
|
|
"kl": 0.031097412109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0032,
|
|
"reward": 0.8616071939468384,
|
|
"reward_std": 0.03382905758917332,
|
|
"rewards/accuracy_reward": 0.8616071939468384,
|
|
"step": 964
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 424.8817138671875,
|
|
"epoch": 4.921119592875318,
|
|
"grad_norm": 0.14356234993332598,
|
|
"kl": 0.0345458984375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0019,
|
|
"reward": 0.8593750447034836,
|
|
"reward_std": 0.02112732268869877,
|
|
"rewards/accuracy_reward": 0.8593750447034836,
|
|
"step": 965
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.2678756713867,
|
|
"epoch": 4.926208651399491,
|
|
"grad_norm": 0.20752419401112923,
|
|
"kl": 0.039215087890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.8258928954601288,
|
|
"reward_std": 0.050552159547805786,
|
|
"rewards/accuracy_reward": 0.8258928954601288,
|
|
"step": 966
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.64957427978516,
|
|
"epoch": 4.931297709923664,
|
|
"grad_norm": 0.17012701589914264,
|
|
"kl": 0.03399658203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0065,
|
|
"reward": 0.7098214626312256,
|
|
"reward_std": 0.030055894516408443,
|
|
"rewards/accuracy_reward": 0.7098214626312256,
|
|
"step": 967
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 423.7901916503906,
|
|
"epoch": 4.9363867684478375,
|
|
"grad_norm": 0.2558617450354285,
|
|
"kl": 0.03790283203125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.7968750447034836,
|
|
"reward_std": 0.06885028630495071,
|
|
"rewards/accuracy_reward": 0.7968750447034836,
|
|
"step": 968
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 403.5268020629883,
|
|
"epoch": 4.9414758269720105,
|
|
"grad_norm": 0.11205730650145966,
|
|
"kl": 0.03814697265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0009,
|
|
"reward": 0.8973214328289032,
|
|
"reward_std": 0.012198750860989094,
|
|
"rewards/accuracy_reward": 0.8973214328289032,
|
|
"step": 969
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.9085006713867,
|
|
"epoch": 4.9465648854961835,
|
|
"grad_norm": 0.15661667644129318,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.004,
|
|
"reward": 0.9017857313156128,
|
|
"reward_std": 0.026025486178696156,
|
|
"rewards/accuracy_reward": 0.9017857313156128,
|
|
"step": 970
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 401.1384048461914,
|
|
"epoch": 4.951653944020356,
|
|
"grad_norm": 0.23728140080143653,
|
|
"kl": 0.043212890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0032,
|
|
"reward": 0.7745536118745804,
|
|
"reward_std": 0.060614774003624916,
|
|
"rewards/accuracy_reward": 0.7745536118745804,
|
|
"step": 971
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 411.42635345458984,
|
|
"epoch": 4.956743002544529,
|
|
"grad_norm": 0.27996267425507504,
|
|
"kl": 0.037109375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0014,
|
|
"reward": 0.6830357313156128,
|
|
"reward_std": 0.08018255140632391,
|
|
"rewards/accuracy_reward": 0.6830357313156128,
|
|
"step": 972
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.0580596923828,
|
|
"epoch": 4.961832061068702,
|
|
"grad_norm": 0.24214198480984425,
|
|
"kl": 0.038848876953125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0011,
|
|
"reward": 0.745535746216774,
|
|
"reward_std": 0.0629409858956933,
|
|
"rewards/accuracy_reward": 0.745535746216774,
|
|
"step": 973
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 408.69644927978516,
|
|
"epoch": 4.966921119592875,
|
|
"grad_norm": 0.27715338649050936,
|
|
"kl": 0.0360107421875,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0062,
|
|
"reward": 0.7857143133878708,
|
|
"reward_std": 0.10380434338003397,
|
|
"rewards/accuracy_reward": 0.7857143133878708,
|
|
"step": 974
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 399.4107360839844,
|
|
"epoch": 4.972010178117048,
|
|
"grad_norm": 0.19920602212269978,
|
|
"kl": 0.037261962890625,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0004,
|
|
"reward": 0.8303571790456772,
|
|
"reward_std": 0.05728259403258562,
|
|
"rewards/accuracy_reward": 0.8303571790456772,
|
|
"step": 975
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 391.82144927978516,
|
|
"epoch": 4.977099236641221,
|
|
"grad_norm": 0.256930814820825,
|
|
"kl": 0.03521728515625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0044,
|
|
"reward": 0.848214328289032,
|
|
"reward_std": 0.04791303817182779,
|
|
"rewards/accuracy_reward": 0.848214328289032,
|
|
"step": 976
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 404.7165298461914,
|
|
"epoch": 4.982188295165394,
|
|
"grad_norm": 0.2157480024136974,
|
|
"kl": 0.034698486328125,
|
|
"learning_rate": 2e-06,
|
|
"loss": -0.0067,
|
|
"reward": 0.8928571790456772,
|
|
"reward_std": 0.054453397169709206,
|
|
"rewards/accuracy_reward": 0.8928571790456772,
|
|
"step": 977
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 430.3236770629883,
|
|
"epoch": 4.987277353689567,
|
|
"grad_norm": 0.23626104407159002,
|
|
"kl": 0.03369140625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0093,
|
|
"reward": 0.8214285969734192,
|
|
"reward_std": 0.09703753143548965,
|
|
"rewards/accuracy_reward": 0.8214285969734192,
|
|
"step": 978
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 397.3303756713867,
|
|
"epoch": 4.99236641221374,
|
|
"grad_norm": 0.21612155133967967,
|
|
"kl": 0.03619384765625,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0027,
|
|
"reward": 0.7410714477300644,
|
|
"reward_std": 0.03942544851452112,
|
|
"rewards/accuracy_reward": 0.7410714477300644,
|
|
"step": 979
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 406.9423294067383,
|
|
"epoch": 4.997455470737913,
|
|
"grad_norm": 0.19596386095038013,
|
|
"kl": 0.0380859375,
|
|
"learning_rate": 2e-06,
|
|
"loss": 0.0032,
|
|
"reward": 0.8883928656578064,
|
|
"reward_std": 0.038224236108362675,
|
|
"rewards/accuracy_reward": 0.8883928656578064,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 4.997455470737913,
|
|
"step": 980,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.0,
|
|
"train_runtime": 1.0788,
|
|
"train_samples_per_second": 25495.021,
|
|
"train_steps_per_second": 908.385
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 980,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 5,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 16,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|