{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.997455470737913,
  "eval_steps": 500,
  "global_step": 980,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 405.1964416503906,
      "epoch": 0.005089058524173028,
      "grad_norm": 0.5131905682335655,
      "kl": 0.0,
      "learning_rate": 2e-06,
      "loss": -0.0083,
      "reward": 0.3035714477300644,
      "reward_std": 0.3146933540701866,
      "rewards/accuracy_reward": 0.3035714477300644,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.7678756713867,
      "epoch": 0.010178117048346057,
      "grad_norm": 0.48550660503092874,
      "kl": 0.00038623809814453125,
      "learning_rate": 2e-06,
      "loss": -0.0022,
      "reward": 0.2812500074505806,
      "reward_std": 0.3097551055252552,
      "rewards/accuracy_reward": 0.2812500074505806,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.9174270629883,
      "epoch": 0.015267175572519083,
      "grad_norm": 0.4755285363526245,
      "kl": 0.0006351470947265625,
      "learning_rate": 2e-06,
      "loss": 0.0043,
      "reward": 0.3035714477300644,
      "reward_std": 0.31310277432203293,
      "rewards/accuracy_reward": 0.3035714477300644,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.6518020629883,
      "epoch": 0.020356234096692113,
      "grad_norm": 0.500511345750709,
      "kl": 0.00102996826171875,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.3660714402794838,
      "reward_std": 0.31569796800613403,
      "rewards/accuracy_reward": 0.3660714402794838,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.5335006713867,
      "epoch": 0.02544529262086514,
      "grad_norm": 0.5780194120079271,
      "kl": 0.001201629638671875,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.4062500149011612,
      "reward_std": 0.3942900151014328,
      "rewards/accuracy_reward": 0.4062500149011612,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.3236846923828,
      "epoch": 0.030534351145038167,
      "grad_norm": 0.5128731556949468,
      "kl": 0.00258636474609375,
      "learning_rate": 2e-06,
      "loss": -0.0005,
      "reward": 0.2566964440047741,
      "reward_std": 0.31038618460297585,
      "rewards/accuracy_reward": 0.2566964440047741,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.28572845458984,
      "epoch": 0.035623409669211195,
      "grad_norm": 0.5292397662077934,
      "kl": 0.0038604736328125,
      "learning_rate": 2e-06,
      "loss": 0.0214,
      "reward": 0.3058035895228386,
      "reward_std": 0.32114920020103455,
      "rewards/accuracy_reward": 0.3058035895228386,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.28126525878906,
      "epoch": 0.04071246819338423,
      "grad_norm": 0.4923284300392425,
      "kl": 0.003711700439453125,
      "learning_rate": 2e-06,
      "loss": -0.0064,
      "reward": 0.2901785895228386,
      "reward_std": 0.3053254038095474,
      "rewards/accuracy_reward": 0.2901785895228386,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.54689025878906,
      "epoch": 0.04580152671755725,
      "grad_norm": 0.5444443218780471,
      "kl": 0.0039520263671875,
      "learning_rate": 2e-06,
      "loss": 0.0072,
      "reward": 0.2857142947614193,
      "reward_std": 0.3841593265533447,
      "rewards/accuracy_reward": 0.2857142947614193,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.0937728881836,
      "epoch": 0.05089058524173028,
      "grad_norm": 1.5777669521612345,
      "kl": 0.00502777099609375,
      "learning_rate": 2e-06,
      "loss": -0.009,
      "reward": 0.2433035857975483,
      "reward_std": 0.3450405113399029,
      "rewards/accuracy_reward": 0.2433035857975483,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.0759048461914,
      "epoch": 0.05597964376590331,
      "grad_norm": 0.5543684439921044,
      "kl": 0.00626373291015625,
      "learning_rate": 2e-06,
      "loss": -0.0009,
      "reward": 0.3325893022119999,
      "reward_std": 0.29412248730659485,
      "rewards/accuracy_reward": 0.3325893022119999,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.1986770629883,
      "epoch": 0.061068702290076333,
      "grad_norm": 0.49762317088896774,
      "kl": 0.0060577392578125,
      "learning_rate": 2e-06,
      "loss": -0.0067,
      "reward": 0.3794643133878708,
      "reward_std": 0.30871570855379105,
      "rewards/accuracy_reward": 0.3794643133878708,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 350.9977798461914,
      "epoch": 0.06615776081424936,
      "grad_norm": 1714.086045615442,
      "kl": 0.655517578125,
      "learning_rate": 2e-06,
      "loss": 0.0215,
      "reward": 0.3638392984867096,
      "reward_std": 0.34932900965213776,
      "rewards/accuracy_reward": 0.3638392984867096,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 357.37947845458984,
      "epoch": 0.07124681933842239,
      "grad_norm": 0.5527423625236983,
      "kl": 0.0115203857421875,
      "learning_rate": 2e-06,
      "loss": 0.0088,
      "reward": 0.2700892984867096,
      "reward_std": 0.2972116284072399,
      "rewards/accuracy_reward": 0.2700892984867096,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 337.86608123779297,
      "epoch": 0.07633587786259542,
      "grad_norm": 0.6203881688811241,
      "kl": 0.0127716064453125,
      "learning_rate": 2e-06,
      "loss": 0.0117,
      "reward": 0.3727678619325161,
      "reward_std": 0.3377150446176529,
      "rewards/accuracy_reward": 0.3727678619325161,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 365.1651916503906,
      "epoch": 0.08142493638676845,
      "grad_norm": 0.4485230364798742,
      "kl": 0.0117340087890625,
      "learning_rate": 2e-06,
      "loss": -0.0035,
      "reward": 0.3236607275903225,
      "reward_std": 0.23699714615941048,
      "rewards/accuracy_reward": 0.3236607275903225,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 373.04689025878906,
      "epoch": 0.08651399491094147,
      "grad_norm": 0.5003998737790313,
      "kl": 0.01090240478515625,
      "learning_rate": 2e-06,
      "loss": -0.0165,
      "reward": 0.2343750074505806,
      "reward_std": 0.23511192575097084,
      "rewards/accuracy_reward": 0.2343750074505806,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.56697845458984,
      "epoch": 0.0916030534351145,
      "grad_norm": 0.5300248233778917,
      "kl": 0.0104217529296875,
      "learning_rate": 2e-06,
      "loss": 0.002,
      "reward": 0.2723214402794838,
      "reward_std": 0.2561025284230709,
      "rewards/accuracy_reward": 0.2723214402794838,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.7835006713867,
      "epoch": 0.09669211195928754,
      "grad_norm": 0.49467929686899487,
      "kl": 0.009246826171875,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.3482142984867096,
      "reward_std": 0.2795490361750126,
      "rewards/accuracy_reward": 0.3482142984867096,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.15404510498047,
      "epoch": 0.10178117048346055,
      "grad_norm": 0.5179413808478568,
      "kl": 0.0088653564453125,
      "learning_rate": 2e-06,
      "loss": 0.0286,
      "reward": 0.3526785969734192,
      "reward_std": 0.3307727947831154,
      "rewards/accuracy_reward": 0.3526785969734192,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.2834930419922,
      "epoch": 0.10687022900763359,
      "grad_norm": 0.48607254188968196,
      "kl": 0.011444091796875,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.3258928693830967,
      "reward_std": 0.21202195063233376,
      "rewards/accuracy_reward": 0.3258928693830967,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.3995666503906,
      "epoch": 0.11195928753180662,
      "grad_norm": 0.4940851576048368,
      "kl": 0.0086669921875,
      "learning_rate": 2e-06,
      "loss": -0.0179,
      "reward": 0.2299107238650322,
      "reward_std": 0.28535235673189163,
      "rewards/accuracy_reward": 0.2299107238650322,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.1585006713867,
      "epoch": 0.11704834605597965,
      "grad_norm": 0.5219084458151091,
      "kl": 0.00815582275390625,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.404017873108387,
      "reward_std": 0.3285277634859085,
      "rewards/accuracy_reward": 0.404017873108387,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 436.0357360839844,
      "epoch": 0.12213740458015267,
      "grad_norm": 0.4937449249945028,
      "kl": 0.00748443603515625,
      "learning_rate": 2e-06,
      "loss": 0.0023,
      "reward": 0.3995535969734192,
      "reward_std": 0.31111307069659233,
      "rewards/accuracy_reward": 0.3995535969734192,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.0312728881836,
      "epoch": 0.1272264631043257,
      "grad_norm": 0.5247292728268708,
      "kl": 0.00972747802734375,
      "learning_rate": 2e-06,
      "loss": 0.0027,
      "reward": 0.3325892984867096,
      "reward_std": 0.27157899737358093,
      "rewards/accuracy_reward": 0.3325892984867096,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 453.71653747558594,
      "epoch": 0.13231552162849872,
      "grad_norm": 0.5495977190025394,
      "kl": 0.0119781494140625,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.4107143059372902,
      "reward_std": 0.34284864366054535,
      "rewards/accuracy_reward": 0.4107143059372902,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.21876525878906,
      "epoch": 0.13740458015267176,
      "grad_norm": 0.47937723371612206,
      "kl": 0.009246826171875,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.31026787124574184,
      "reward_std": 0.27487244457006454,
      "rewards/accuracy_reward": 0.31026787124574184,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.98663330078125,
      "epoch": 0.14249363867684478,
      "grad_norm": 0.5276131424693773,
      "kl": 0.00921630859375,
      "learning_rate": 2e-06,
      "loss": 0.0092,
      "reward": 0.3281250223517418,
      "reward_std": 0.29099568352103233,
      "rewards/accuracy_reward": 0.3281250223517418,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.6852798461914,
      "epoch": 0.1475826972010178,
      "grad_norm": 0.5582899993904399,
      "kl": 0.0114593505859375,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.3660714477300644,
      "reward_std": 0.3422120660543442,
      "rewards/accuracy_reward": 0.3660714477300644,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.3638610839844,
      "epoch": 0.15267175572519084,
      "grad_norm": 0.5385443874335647,
      "kl": 0.0098114013671875,
      "learning_rate": 2e-06,
      "loss": -0.0095,
      "reward": 0.3861607387661934,
      "reward_std": 0.2970792390406132,
      "rewards/accuracy_reward": 0.3861607387661934,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.87500762939453,
      "epoch": 0.15776081424936386,
      "grad_norm": 0.5037518936326002,
      "kl": 0.0113983154296875,
      "learning_rate": 2e-06,
      "loss": -0.0173,
      "reward": 0.2812500111758709,
      "reward_std": 0.30366747453808784,
      "rewards/accuracy_reward": 0.2812500111758709,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.92859649658203,
      "epoch": 0.1628498727735369,
      "grad_norm": 0.4410514013813698,
      "kl": 0.0113983154296875,
      "learning_rate": 2e-06,
      "loss": 0.0067,
      "reward": 0.274553582072258,
      "reward_std": 0.2705678604543209,
      "rewards/accuracy_reward": 0.274553582072258,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.7968978881836,
      "epoch": 0.16793893129770993,
      "grad_norm": 0.5691294204541579,
      "kl": 0.012908935546875,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.3482142984867096,
      "reward_std": 0.3068159222602844,
      "rewards/accuracy_reward": 0.3482142984867096,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 359.5134048461914,
      "epoch": 0.17302798982188294,
      "grad_norm": 0.5562831699180847,
      "kl": 0.0130157470703125,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.3236607238650322,
      "reward_std": 0.3009896092116833,
      "rewards/accuracy_reward": 0.3236607238650322,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.4486770629883,
      "epoch": 0.178117048346056,
      "grad_norm": 0.46591620037609793,
      "kl": 0.0124053955078125,
      "learning_rate": 2e-06,
      "loss": -0.0005,
      "reward": 0.2924107238650322,
      "reward_std": 0.23107879608869553,
      "rewards/accuracy_reward": 0.2924107238650322,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.2343978881836,
      "epoch": 0.183206106870229,
      "grad_norm": 0.512032862767005,
      "kl": 0.012298583984375,
      "learning_rate": 2e-06,
      "loss": -0.0144,
      "reward": 0.3080357275903225,
      "reward_std": 0.2829487472772598,
      "rewards/accuracy_reward": 0.3080357275903225,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.73216247558594,
      "epoch": 0.18829516539440203,
      "grad_norm": 0.4691396176787084,
      "kl": 0.0124969482421875,
      "learning_rate": 2e-06,
      "loss": 0.0063,
      "reward": 0.2946428693830967,
      "reward_std": 0.27999019995331764,
      "rewards/accuracy_reward": 0.2946428693830967,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.99332427978516,
      "epoch": 0.19338422391857507,
      "grad_norm": 0.49734043814964074,
      "kl": 0.013275146484375,
      "learning_rate": 2e-06,
      "loss": 0.0126,
      "reward": 0.368303582072258,
      "reward_std": 0.37057018280029297,
      "rewards/accuracy_reward": 0.368303582072258,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.02903747558594,
      "epoch": 0.1984732824427481,
      "grad_norm": 0.42504957050666886,
      "kl": 0.01385498046875,
      "learning_rate": 2e-06,
      "loss": -0.0051,
      "reward": 0.377232164144516,
      "reward_std": 0.25631672516465187,
      "rewards/accuracy_reward": 0.377232164144516,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.84153747558594,
      "epoch": 0.2035623409669211,
      "grad_norm": 0.48761609257933625,
      "kl": 0.013336181640625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.2968750111758709,
      "reward_std": 0.30066392943263054,
      "rewards/accuracy_reward": 0.2968750111758709,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.01341247558594,
      "epoch": 0.20865139949109415,
      "grad_norm": 0.47812024733447134,
      "kl": 0.012115478515625,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.3147321566939354,
      "reward_std": 0.3126324266195297,
      "rewards/accuracy_reward": 0.3147321566939354,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.2321548461914,
      "epoch": 0.21374045801526717,
      "grad_norm": 0.43396375187819597,
      "kl": 0.0133819580078125,
      "learning_rate": 2e-06,
      "loss": 0.0067,
      "reward": 0.25000001676380634,
      "reward_std": 0.22224192321300507,
      "rewards/accuracy_reward": 0.25000001676380634,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 450.66519927978516,
      "epoch": 0.21882951653944022,
      "grad_norm": 0.4530485486842693,
      "kl": 0.0116119384765625,
      "learning_rate": 2e-06,
      "loss": 0.0046,
      "reward": 0.2968750149011612,
      "reward_std": 0.291314922273159,
      "rewards/accuracy_reward": 0.2968750149011612,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.6205520629883,
      "epoch": 0.22391857506361323,
      "grad_norm": 0.4882091945408692,
      "kl": 0.0125579833984375,
      "learning_rate": 2e-06,
      "loss": 0.0024,
      "reward": 0.357142873108387,
      "reward_std": 0.3270977512001991,
      "rewards/accuracy_reward": 0.357142873108387,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.82591247558594,
      "epoch": 0.22900763358778625,
      "grad_norm": 0.4906741745649979,
      "kl": 0.0128021240234375,
      "learning_rate": 2e-06,
      "loss": 0.0061,
      "reward": 0.4441964477300644,
      "reward_std": 0.2941880486905575,
      "rewards/accuracy_reward": 0.4441964477300644,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.5357360839844,
      "epoch": 0.2340966921119593,
      "grad_norm": 0.5026482980354947,
      "kl": 0.011474609375,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.4174107313156128,
      "reward_std": 0.3358287177979946,
      "rewards/accuracy_reward": 0.4174107313156128,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.8861846923828,
      "epoch": 0.23918575063613232,
      "grad_norm": 0.45279918851719525,
      "kl": 0.0128326416015625,
      "learning_rate": 2e-06,
      "loss": -0.0052,
      "reward": 0.2276785783469677,
      "reward_std": 0.25647593289613724,
      "rewards/accuracy_reward": 0.2276785783469677,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.22769927978516,
      "epoch": 0.24427480916030533,
      "grad_norm": 0.5454276173522247,
      "kl": 0.0137786865234375,
      "learning_rate": 2e-06,
      "loss": -0.0065,
      "reward": 0.330357164144516,
      "reward_std": 0.3306938484311104,
      "rewards/accuracy_reward": 0.330357164144516,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.2299346923828,
      "epoch": 0.24936386768447838,
      "grad_norm": 0.5348141948553139,
      "kl": 0.0132598876953125,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.4151785895228386,
      "reward_std": 0.3523707911372185,
      "rewards/accuracy_reward": 0.4151785895228386,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.0625228881836,
      "epoch": 0.2544529262086514,
      "grad_norm": 0.5085458953679328,
      "kl": 0.0128021240234375,
      "learning_rate": 2e-06,
      "loss": -0.0021,
      "reward": 0.522321455180645,
      "reward_std": 0.33058052510023117,
      "rewards/accuracy_reward": 0.522321455180645,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.6897506713867,
      "epoch": 0.2595419847328244,
      "grad_norm": 1518291951656.6533,
      "kl": 7079985152.011017,
      "learning_rate": 2e-06,
      "loss": 117960256.0,
      "reward": 0.3258928768336773,
      "reward_std": 0.2740046866238117,
      "rewards/accuracy_reward": 0.3258928768336773,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.23439025878906,
      "epoch": 0.26463104325699743,
      "grad_norm": 0.5514192754520774,
      "kl": 0.0154571533203125,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.223214291036129,
      "reward_std": 0.2859731763601303,
      "rewards/accuracy_reward": 0.223214291036129,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.78572845458984,
      "epoch": 0.2697201017811705,
      "grad_norm": 0.4865275565102642,
      "kl": 0.0140380859375,
      "learning_rate": 2e-06,
      "loss": -0.0111,
      "reward": 0.2812500074505806,
      "reward_std": 0.28903238475322723,
      "rewards/accuracy_reward": 0.2812500074505806,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.07814025878906,
      "epoch": 0.2748091603053435,
      "grad_norm": 0.5784731742120693,
      "kl": 0.015106201171875,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.3214285783469677,
      "reward_std": 0.33232294768095016,
      "rewards/accuracy_reward": 0.3214285783469677,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 361.86832427978516,
      "epoch": 0.27989821882951654,
      "grad_norm": 0.5517150744071869,
      "kl": 0.017242431640625,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.428571455180645,
      "reward_std": 0.34388555958867073,
      "rewards/accuracy_reward": 0.428571455180645,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.1629638671875,
      "epoch": 0.28498727735368956,
      "grad_norm": 0.4914644497587217,
      "kl": 0.01690673828125,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.3794643022119999,
      "reward_std": 0.27357952296733856,
      "rewards/accuracy_reward": 0.3794643022119999,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.62501525878906,
      "epoch": 0.2900763358778626,
      "grad_norm": 0.5305247627017694,
      "kl": 0.0151519775390625,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.3750000149011612,
      "reward_std": 0.36491215601563454,
      "rewards/accuracy_reward": 0.3750000149011612,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.9598388671875,
      "epoch": 0.2951653944020356,
      "grad_norm": 0.5355122953339883,
      "kl": 0.0160980224609375,
      "learning_rate": 2e-06,
      "loss": -0.0043,
      "reward": 0.3147321566939354,
      "reward_std": 0.33825405687093735,
      "rewards/accuracy_reward": 0.3147321566939354,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.5580520629883,
      "epoch": 0.30025445292620867,
      "grad_norm": 0.4984115312555115,
      "kl": 0.014129638671875,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.3750000260770321,
      "reward_std": 0.2658737301826477,
      "rewards/accuracy_reward": 0.3750000260770321,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.7991256713867,
      "epoch": 0.3053435114503817,
      "grad_norm": 0.48379704997835704,
      "kl": 0.0148162841796875,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.4040178656578064,
      "reward_std": 0.2832214683294296,
      "rewards/accuracy_reward": 0.4040178656578064,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.7276916503906,
      "epoch": 0.3104325699745547,
      "grad_norm": 0.5414065913568801,
      "kl": 0.013671875,
      "learning_rate": 2e-06,
      "loss": -0.0064,
      "reward": 0.4174107238650322,
      "reward_std": 0.32154662907123566,
      "rewards/accuracy_reward": 0.4174107238650322,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.85939025878906,
      "epoch": 0.3155216284987277,
      "grad_norm": 0.5719771747244955,
      "kl": 0.0153045654296875,
      "learning_rate": 2e-06,
      "loss": 0.014,
      "reward": 0.404017873108387,
      "reward_std": 0.3457542136311531,
      "rewards/accuracy_reward": 0.404017873108387,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.0736770629883,
      "epoch": 0.32061068702290074,
      "grad_norm": 0.5063681403358649,
      "kl": 0.016845703125,
      "learning_rate": 2e-06,
      "loss": 0.0128,
      "reward": 0.3035714440047741,
      "reward_std": 0.32448188215494156,
      "rewards/accuracy_reward": 0.3035714440047741,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.88172149658203,
      "epoch": 0.3256997455470738,
      "grad_norm": 0.4876071845124516,
      "kl": 0.015899658203125,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.2633928693830967,
      "reward_std": 0.27684302255511284,
      "rewards/accuracy_reward": 0.2633928693830967,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.57814025878906,
      "epoch": 0.33078880407124683,
      "grad_norm": 0.5032443746200898,
      "kl": 0.0156707763671875,
      "learning_rate": 2e-06,
      "loss": 0.0127,
      "reward": 0.3950892984867096,
      "reward_std": 0.3091568648815155,
      "rewards/accuracy_reward": 0.3950892984867096,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.8973388671875,
      "epoch": 0.33587786259541985,
      "grad_norm": 0.5211229249047339,
      "kl": 0.0162353515625,
      "learning_rate": 2e-06,
      "loss": 0.0062,
      "reward": 0.4553571492433548,
      "reward_std": 0.3269530087709427,
      "rewards/accuracy_reward": 0.4553571492433548,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.35716247558594,
      "epoch": 0.34096692111959287,
      "grad_norm": 0.48245726026970265,
      "kl": 0.01739501953125,
      "learning_rate": 2e-06,
      "loss": 0.0172,
      "reward": 0.379464291036129,
      "reward_std": 0.30845845863223076,
      "rewards/accuracy_reward": 0.379464291036129,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 354.5245666503906,
      "epoch": 0.3460559796437659,
      "grad_norm": 1531182.3047376974,
      "kl": 7968.012969970703,
      "learning_rate": 2e-06,
      "loss": 295.9489,
      "reward": 0.3236607238650322,
      "reward_std": 0.3228748217225075,
      "rewards/accuracy_reward": 0.3236607238650322,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 364.82591247558594,
      "epoch": 0.3511450381679389,
      "grad_norm": 0.4911830724591876,
      "kl": 0.0185394287109375,
      "learning_rate": 2e-06,
      "loss": 0.016,
      "reward": 0.3526785895228386,
      "reward_std": 0.24602409452199936,
      "rewards/accuracy_reward": 0.3526785895228386,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.0825958251953,
      "epoch": 0.356234096692112,
      "grad_norm": 0.49807944084033023,
      "kl": 0.0153961181640625,
      "learning_rate": 2e-06,
      "loss": 0.0054,
      "reward": 0.4017857387661934,
      "reward_std": 0.29982316493988037,
      "rewards/accuracy_reward": 0.4017857387661934,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 373.26564025878906,
      "epoch": 0.361323155216285,
      "grad_norm": 0.5036334041571832,
      "kl": 0.0206298828125,
      "learning_rate": 2e-06,
      "loss": 0.01,
      "reward": 0.3236607313156128,
      "reward_std": 0.29557934775948524,
      "rewards/accuracy_reward": 0.3236607313156128,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.5089416503906,
      "epoch": 0.366412213740458,
      "grad_norm": 0.39407939476751,
      "kl": 0.01800537109375,
      "learning_rate": 2e-06,
      "loss": 0.002,
      "reward": 0.33258930779993534,
      "reward_std": 0.21618272736668587,
      "rewards/accuracy_reward": 0.33258930779993534,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.0826110839844,
      "epoch": 0.37150127226463103,
      "grad_norm": 0.5130185552836011,
      "kl": 0.01806640625,
      "learning_rate": 2e-06,
      "loss": 0.0027,
      "reward": 0.33482143096625805,
      "reward_std": 0.3136202208697796,
      "rewards/accuracy_reward": 0.33482143096625805,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.50894927978516,
      "epoch": 0.37659033078880405,
      "grad_norm": 0.5177463812190622,
      "kl": 0.019622802734375,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.3169643096625805,
      "reward_std": 0.2695329450070858,
      "rewards/accuracy_reward": 0.3169643096625805,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.32592010498047,
      "epoch": 0.3816793893129771,
      "grad_norm": 0.5440731331635377,
      "kl": 0.02008056640625,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.3102678656578064,
      "reward_std": 0.2793062496930361,
      "rewards/accuracy_reward": 0.3102678656578064,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 376.77010345458984,
      "epoch": 0.38676844783715014,
      "grad_norm": 0.547175246797069,
      "kl": 0.01873779296875,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.3147321604192257,
      "reward_std": 0.2940896600484848,
      "rewards/accuracy_reward": 0.3147321604192257,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.57813262939453,
      "epoch": 0.39185750636132316,
      "grad_norm": 0.5229440696967318,
      "kl": 0.0159454345703125,
      "learning_rate": 2e-06,
      "loss": -0.0054,
      "reward": 0.3125000111758709,
      "reward_std": 0.3128367140889168,
      "rewards/accuracy_reward": 0.3125000111758709,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.7254638671875,
      "epoch": 0.3969465648854962,
      "grad_norm": 0.49808120455381655,
      "kl": 0.017059326171875,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.4598214477300644,
      "reward_std": 0.331941694021225,
      "rewards/accuracy_reward": 0.4598214477300644,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.88394927978516,
      "epoch": 0.4020356234096692,
      "grad_norm": 0.4672596874629748,
      "kl": 0.0201416015625,
      "learning_rate": 2e-06,
      "loss": 0.0093,
      "reward": 0.3214285895228386,
      "reward_std": 0.269159983843565,
      "rewards/accuracy_reward": 0.3214285895228386,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.1964569091797,
      "epoch": 0.4071246819338422,
      "grad_norm": 0.5054012743482283,
      "kl": 0.0167999267578125,
      "learning_rate": 2e-06,
      "loss": 0.0139,
      "reward": 0.4508928805589676,
      "reward_std": 0.3098481632769108,
      "rewards/accuracy_reward": 0.4508928805589676,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.5178756713867,
      "epoch": 0.4122137404580153,
      "grad_norm": 0.5139101758762243,
      "kl": 0.0169830322265625,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.33705358766019344,
      "reward_std": 0.2844383306801319,
      "rewards/accuracy_reward": 0.33705358766019344,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.6317138671875,
      "epoch": 0.4173027989821883,
      "grad_norm": 0.48587084836771205,
      "kl": 0.0179290771484375,
      "learning_rate": 2e-06,
      "loss": -0.0078,
      "reward": 0.2611607238650322,
      "reward_std": 0.3100224882364273,
      "rewards/accuracy_reward": 0.2611607238650322,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.8727798461914,
      "epoch": 0.4223918575063613,
      "grad_norm": 0.4928569542160694,
      "kl": 0.018310546875,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.3348214477300644,
      "reward_std": 0.3120606653392315,
      "rewards/accuracy_reward": 0.3348214477300644,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.92189025878906,
      "epoch": 0.42748091603053434,
      "grad_norm": 0.42531892065233795,
      "kl": 0.016448974609375,
      "learning_rate": 2e-06,
      "loss": 0.0082,
      "reward": 0.3816964402794838,
      "reward_std": 0.25463341921567917,
      "rewards/accuracy_reward": 0.3816964402794838,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.4174270629883,
      "epoch": 0.43256997455470736,
      "grad_norm": 0.5241773061730735,
      "kl": 0.019378662109375,
      "learning_rate": 2e-06,
      "loss": -0.0074,
      "reward": 0.3571428693830967,
      "reward_std": 0.2619221694767475,
      "rewards/accuracy_reward": 0.3571428693830967,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.58260345458984,
      "epoch": 0.43765903307888043,
      "grad_norm": 0.4781800627435873,
      "kl": 0.0172882080078125,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.3459821715950966,
      "reward_std": 0.2790309302508831,
      "rewards/accuracy_reward": 0.3459821715950966,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.4241256713867,
      "epoch": 0.44274809160305345,
      "grad_norm": 0.5003740783291474,
      "kl": 0.0181884765625,
      "learning_rate": 2e-06,
      "loss": -0.0166,
      "reward": 0.3147321492433548,
      "reward_std": 0.2965996153652668,
      "rewards/accuracy_reward": 0.3147321492433548,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.17188262939453,
      "epoch": 0.44783715012722647,
      "grad_norm": 0.5173256984473651,
      "kl": 0.01934814453125,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.341517873108387,
      "reward_std": 0.28107255324721336,
      "rewards/accuracy_reward": 0.341517873108387,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.31028747558594,
      "epoch": 0.4529262086513995,
      "grad_norm": 0.4509863293454374,
      "kl": 0.0174713134765625,
      "learning_rate": 2e-06,
      "loss": -0.007,
      "reward": 0.2500000186264515,
      "reward_std": 0.2574096694588661,
      "rewards/accuracy_reward": 0.2500000186264515,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.56251525878906,
      "epoch": 0.4580152671755725,
      "grad_norm": 0.4766616285716767,
      "kl": 0.01788330078125,
      "learning_rate": 2e-06,
      "loss": -0.0097,
      "reward": 0.19419643841683865,
      "reward_std": 0.2207874357700348,
      "rewards/accuracy_reward": 0.19419643841683865,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.45314025878906,
      "epoch": 0.4631043256997455,
      "grad_norm": 0.4959895156830065,
      "kl": 0.02020263671875,
      "learning_rate": 2e-06,
      "loss": 0.0128,
      "reward": 0.4129464402794838,
      "reward_std": 0.2791617177426815,
      "rewards/accuracy_reward": 0.4129464402794838,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.78126525878906,
      "epoch": 0.4681933842239186,
      "grad_norm": 0.4507057246808986,
      "kl": 0.0174560546875,
      "learning_rate": 2e-06,
      "loss": 0.0074,
      "reward": 0.337053582072258,
      "reward_std": 0.2914833575487137,
      "rewards/accuracy_reward": 0.337053582072258,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.45538330078125,
      "epoch": 0.4732824427480916,
      "grad_norm": 0.48879525342887997,
      "kl": 0.019073486328125,
      "learning_rate": 2e-06,
      "loss": 0.0083,
      "reward": 0.2633928656578064,
      "reward_std": 0.25695963576436043,
      "rewards/accuracy_reward": 0.2633928656578064,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.94422149658203,
      "epoch": 0.47837150127226463,
      "grad_norm": 0.5190734752226046,
      "kl": 0.018524169921875,
      "learning_rate": 2e-06,
      "loss": 0.0191,
      "reward": 0.3861607313156128,
      "reward_std": 0.36641061305999756,
      "rewards/accuracy_reward": 0.3861607313156128,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.5357360839844,
      "epoch": 0.48346055979643765,
      "grad_norm": 0.5370357401422521,
      "kl": 0.0185546875,
      "learning_rate": 2e-06,
      "loss": 0.0127,
      "reward": 0.3571428768336773,
      "reward_std": 0.2888643816113472,
      "rewards/accuracy_reward": 0.3571428768336773,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.84376525878906,
      "epoch": 0.48854961832061067,
      "grad_norm": 0.4674788354621914,
      "kl": 0.019195556640625,
      "learning_rate": 2e-06,
      "loss": 0.0152,
      "reward": 0.2767857201397419,
      "reward_std": 0.22408967837691307,
      "rewards/accuracy_reward": 0.2767857201397419,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.6049270629883,
      "epoch": 0.49363867684478374,
      "grad_norm": 0.5122986427613485,
      "kl": 0.01995849609375,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.3392857275903225,
      "reward_std": 0.29930707439780235,
      "rewards/accuracy_reward": 0.3392857275903225,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.98439025878906,
      "epoch": 0.49872773536895676,
      "grad_norm": 0.4555295504049037,
      "kl": 0.0169677734375,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.464285746216774,
      "reward_std": 0.2649185098707676,
      "rewards/accuracy_reward": 0.464285746216774,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.53572845458984,
      "epoch": 0.5038167938931297,
      "grad_norm": 0.4810120312253362,
      "kl": 0.0220947265625,
      "learning_rate": 2e-06,
      "loss": 0.0033,
      "reward": 0.3191964477300644,
      "reward_std": 0.2573278546333313,
      "rewards/accuracy_reward": 0.3191964477300644,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.0335006713867,
      "epoch": 0.5089058524173028,
      "grad_norm": 0.49415169467067754,
      "kl": 0.020294189453125,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.3214285857975483,
      "reward_std": 0.3164416328072548,
      "rewards/accuracy_reward": 0.3214285857975483,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.0424270629883,
      "epoch": 0.5139949109414759,
      "grad_norm": 0.5008631838848484,
      "kl": 0.020355224609375,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.3638392984867096,
      "reward_std": 0.312275193631649,
      "rewards/accuracy_reward": 0.3638392984867096,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.3527069091797,
      "epoch": 0.5190839694656488,
      "grad_norm": 0.4500688398225032,
      "kl": 0.021392822265625,
      "learning_rate": 2e-06,
      "loss": 0.0114,
      "reward": 0.247767873108387,
      "reward_std": 0.2502409480512142,
      "rewards/accuracy_reward": 0.247767873108387,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.0357360839844,
      "epoch": 0.5241730279898219,
      "grad_norm": 0.5522616500891663,
      "kl": 0.019317626953125,
      "learning_rate": 2e-06,
      "loss": -0.0139,
      "reward": 0.4531250223517418,
      "reward_std": 0.34994350373744965,
      "rewards/accuracy_reward": 0.4531250223517418,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 435.9174270629883,
      "epoch": 0.5292620865139949,
      "grad_norm": 0.5016605132656401,
      "kl": 0.020416259765625,
      "learning_rate": 2e-06,
      "loss": -0.0019,
      "reward": 0.4464285895228386,
      "reward_std": 0.3158109188079834,
      "rewards/accuracy_reward": 0.4464285895228386,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.1830520629883,
      "epoch": 0.5343511450381679,
      "grad_norm": 0.5508999015132893,
      "kl": 0.022491455078125,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.3593750260770321,
      "reward_std": 0.29125937074422836,
      "rewards/accuracy_reward": 0.3593750260770321,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.0067138671875,
      "epoch": 0.539440203562341,
      "grad_norm": 8.285535898929085,
      "kl": 0.1497802734375,
      "learning_rate": 2e-06,
      "loss": 0.0127,
      "reward": 0.3258928805589676,
      "reward_std": 0.31499721854925156,
      "rewards/accuracy_reward": 0.3258928805589676,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.1629638671875,
      "epoch": 0.544529262086514,
      "grad_norm": 0.560681049322606,
      "kl": 0.02166748046875,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.4107143059372902,
      "reward_std": 0.3652753680944443,
      "rewards/accuracy_reward": 0.4107143059372902,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.8951110839844,
      "epoch": 0.549618320610687,
      "grad_norm": 0.5057047183097904,
      "kl": 0.02301025390625,
      "learning_rate": 2e-06,
      "loss": -0.003,
      "reward": 0.3683035969734192,
      "reward_std": 0.2857242077589035,
      "rewards/accuracy_reward": 0.3683035969734192,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 433.0134048461914,
      "epoch": 0.55470737913486,
      "grad_norm": 0.5291985319332276,
      "kl": 0.0194854736328125,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.3415178619325161,
      "reward_std": 0.34176725894212723,
      "rewards/accuracy_reward": 0.3415178619325161,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.2611770629883,
      "epoch": 0.5597964376590331,
      "grad_norm": 0.5318844814949354,
      "kl": 0.023834228515625,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.3236607350409031,
      "reward_std": 0.2054804302752018,
      "rewards/accuracy_reward": 0.3236607350409031,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.0491256713867,
      "epoch": 0.5648854961832062,
      "grad_norm": 0.4732647168414915,
      "kl": 0.017913818359375,
      "learning_rate": 2e-06,
      "loss": 0.009,
      "reward": 0.3593750223517418,
      "reward_std": 0.3117176443338394,
      "rewards/accuracy_reward": 0.3593750223517418,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.0625228881836,
      "epoch": 0.5699745547073791,
      "grad_norm": 0.4403194436230484,
      "kl": 0.01910400390625,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.450892873108387,
      "reward_std": 0.29527556151151657,
      "rewards/accuracy_reward": 0.450892873108387,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.1942138671875,
      "epoch": 0.5750636132315522,
      "grad_norm": 0.585103280065545,
      "kl": 0.0179443359375,
      "learning_rate": 2e-06,
      "loss": 0.0097,
      "reward": 0.3147321492433548,
      "reward_std": 0.32731254771351814,
      "rewards/accuracy_reward": 0.3147321492433548,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.4107360839844,
      "epoch": 0.5801526717557252,
      "grad_norm": 0.499338643029717,
      "kl": 0.0187530517578125,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.3727678805589676,
      "reward_std": 0.30609316378831863,
      "rewards/accuracy_reward": 0.3727678805589676,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.26564025878906,
      "epoch": 0.5852417302798982,
      "grad_norm": 0.5229032264133338,
      "kl": 0.0181884765625,
      "learning_rate": 2e-06,
      "loss": 0.0128,
      "reward": 0.3950893059372902,
      "reward_std": 0.2910423092544079,
      "rewards/accuracy_reward": 0.3950893059372902,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.41295623779297,
      "epoch": 0.5903307888040712,
      "grad_norm": 0.46415048500586265,
      "kl": 0.0173492431640625,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.3482142984867096,
      "reward_std": 0.2729547992348671,
      "rewards/accuracy_reward": 0.3482142984867096,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.7611770629883,
      "epoch": 0.5954198473282443,
      "grad_norm": 0.5151821080636549,
      "kl": 0.017333984375,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.3236607350409031,
      "reward_std": 0.3345264159142971,
      "rewards/accuracy_reward": 0.3236607350409031,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.51564025878906,
      "epoch": 0.6005089058524173,
      "grad_norm": 0.5375586636437247,
      "kl": 0.018890380859375,
      "learning_rate": 2e-06,
      "loss": 0.0204,
      "reward": 0.4263393059372902,
      "reward_std": 0.35839200019836426,
      "rewards/accuracy_reward": 0.4263393059372902,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.0736770629883,
      "epoch": 0.6055979643765903,
      "grad_norm": 0.4459136396200577,
      "kl": 0.01971435546875,
      "learning_rate": 2e-06,
      "loss": 0.0028,
      "reward": 0.3392857275903225,
      "reward_std": 0.2719515413045883,
      "rewards/accuracy_reward": 0.3392857275903225,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.1272506713867,
      "epoch": 0.6106870229007634,
      "grad_norm": 0.44422677688912865,
      "kl": 0.01959228515625,
      "learning_rate": 2e-06,
      "loss": 0.0103,
      "reward": 0.2812500074505806,
      "reward_std": 0.24111849442124367,
      "rewards/accuracy_reward": 0.2812500074505806,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.32144927978516,
      "epoch": 0.6157760814249363,
      "grad_norm": 0.5333976032222449,
      "kl": 0.01995849609375,
      "learning_rate": 2e-06,
      "loss": 0.0184,
      "reward": 0.3861607313156128,
      "reward_std": 0.3275603875517845,
      "rewards/accuracy_reward": 0.3861607313156128,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.2768020629883,
      "epoch": 0.6208651399491094,
      "grad_norm": 0.5631596318868146,
      "kl": 0.020294189453125,
      "learning_rate": 2e-06,
      "loss": 0.015,
      "reward": 0.3794643059372902,
      "reward_std": 0.2962530180811882,
      "rewards/accuracy_reward": 0.3794643059372902,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 373.7366180419922,
      "epoch": 0.6259541984732825,
      "grad_norm": 0.5473422513057858,
      "kl": 0.02227783203125,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.3526785895228386,
      "reward_std": 0.3441708981990814,
      "rewards/accuracy_reward": 0.3526785895228386,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.9018096923828,
      "epoch": 0.6310432569974554,
      "grad_norm": 0.44013792020862513,
      "kl": 0.018707275390625,
      "learning_rate": 2e-06,
      "loss": -0.0034,
      "reward": 0.4575893133878708,
      "reward_std": 0.24109597876667976,
      "rewards/accuracy_reward": 0.4575893133878708,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.28572845458984,
      "epoch": 0.6361323155216285,
      "grad_norm": 0.481985000927126,
      "kl": 0.021728515625,
      "learning_rate": 2e-06,
      "loss": -0.0025,
      "reward": 0.352678582072258,
      "reward_std": 0.24602680280804634,
      "rewards/accuracy_reward": 0.352678582072258,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.2701110839844,
      "epoch": 0.6412213740458015,
      "grad_norm": 0.4547008231526985,
      "kl": 0.022247314453125,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.2968750111758709,
      "reward_std": 0.22678174823522568,
      "rewards/accuracy_reward": 0.2968750111758709,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.3192138671875,
      "epoch": 0.6463104325699746,
      "grad_norm": 0.8872411450857681,
      "kl": 0.01995849609375,
      "learning_rate": 2e-06,
      "loss": 0.0099,
      "reward": 0.2611607238650322,
      "reward_std": 0.26269694045186043,
      "rewards/accuracy_reward": 0.2611607238650322,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.0669860839844,
      "epoch": 0.6513994910941476,
      "grad_norm": 0.5408514885200469,
      "kl": 0.021697998046875,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.4308035895228386,
      "reward_std": 0.28540412336587906,
      "rewards/accuracy_reward": 0.4308035895228386,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.52457427978516,
      "epoch": 0.6564885496183206,
      "grad_norm": 0.5282900476478553,
      "kl": 0.023101806640625,
      "learning_rate": 2e-06,
      "loss": -0.0052,
      "reward": 0.345982164144516,
      "reward_std": 0.3362126648426056,
      "rewards/accuracy_reward": 0.345982164144516,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.7701110839844,
      "epoch": 0.6615776081424937,
      "grad_norm": 0.5250549946167662,
      "kl": 0.02166748046875,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.3013393022119999,
      "reward_std": 0.27612515166401863,
      "rewards/accuracy_reward": 0.3013393022119999,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.2768020629883,
      "epoch": 0.6666666666666666,
      "grad_norm": 0.500657457264774,
      "kl": 0.01947021484375,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.357142873108387,
      "reward_std": 0.31440506875514984,
      "rewards/accuracy_reward": 0.357142873108387,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.7098388671875,
      "epoch": 0.6717557251908397,
      "grad_norm": 0.5116768708151028,
      "kl": 0.018798828125,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.2790178656578064,
      "reward_std": 0.31369929388165474,
      "rewards/accuracy_reward": 0.2790178656578064,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.84154510498047,
      "epoch": 0.6768447837150128,
      "grad_norm": 0.5323305032149934,
      "kl": 0.021087646484375,
      "learning_rate": 2e-06,
      "loss": -0.0115,
      "reward": 0.3526785783469677,
      "reward_std": 0.324299618601799,
      "rewards/accuracy_reward": 0.3526785783469677,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.5491256713867,
      "epoch": 0.6819338422391857,
      "grad_norm": 0.49075673954093485,
      "kl": 0.021514892578125,
      "learning_rate": 2e-06,
      "loss": -0.0009,
      "reward": 0.2901785857975483,
      "reward_std": 0.31014423072338104,
      "rewards/accuracy_reward": 0.2901785857975483,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.2053756713867,
      "epoch": 0.6870229007633588,
      "grad_norm": 0.5048769982735202,
      "kl": 0.02349853515625,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.3035714402794838,
      "reward_std": 0.2917110025882721,
      "rewards/accuracy_reward": 0.3035714402794838,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.18082427978516,
      "epoch": 0.6921119592875318,
      "grad_norm": 0.4036350502302806,
      "kl": 0.02081298828125,
      "learning_rate": 2e-06,
      "loss": -0.0069,
      "reward": 0.3660714402794838,
      "reward_std": 0.22461625561118126,
      "rewards/accuracy_reward": 0.3660714402794838,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.10047149658203,
      "epoch": 0.6972010178117048,
      "grad_norm": 0.4490091972981054,
      "kl": 0.022430419921875,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.2924107275903225,
      "reward_std": 0.2619885765016079,
      "rewards/accuracy_reward": 0.2924107275903225,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.5714416503906,
      "epoch": 0.7022900763358778,
      "grad_norm": 0.4789993319261546,
      "kl": 0.023834228515625,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.4062500149011612,
      "reward_std": 0.22731737792491913,
      "rewards/accuracy_reward": 0.4062500149011612,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.3348388671875,
      "epoch": 0.7073791348600509,
      "grad_norm": 0.5687204248492608,
      "kl": 0.02569580078125,
      "learning_rate": 2e-06,
      "loss": -0.0093,
      "reward": 0.321428582072258,
      "reward_std": 0.26937170699238777,
      "rewards/accuracy_reward": 0.321428582072258,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.02010345458984,
      "epoch": 0.712468193384224,
      "grad_norm": 0.46015246488914435,
      "kl": 0.021484375,
      "learning_rate": 2e-06,
      "loss": 0.0171,
      "reward": 0.5089285895228386,
      "reward_std": 0.2650904916226864,
      "rewards/accuracy_reward": 0.5089285895228386,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.76788330078125,
      "epoch": 0.7175572519083969,
      "grad_norm": 0.5545229461003499,
      "kl": 0.022003173828125,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.3772321566939354,
      "reward_std": 0.3813677653670311,
      "rewards/accuracy_reward": 0.3772321566939354,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.89957427978516,
      "epoch": 0.72264631043257,
      "grad_norm": 0.6084536376433793,
      "kl": 0.025726318359375,
      "learning_rate": 2e-06,
      "loss": 0.0156,
      "reward": 0.3482143022119999,
      "reward_std": 0.3355560079216957,
      "rewards/accuracy_reward": 0.3482143022119999,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.20091247558594,
      "epoch": 0.727735368956743,
      "grad_norm": 0.4972764291068402,
      "kl": 0.024444580078125,
      "learning_rate": 2e-06,
      "loss": -0.0005,
      "reward": 0.3191964440047741,
      "reward_std": 0.3096605874598026,
      "rewards/accuracy_reward": 0.3191964440047741,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.8817138671875,
      "epoch": 0.732824427480916,
      "grad_norm": 0.5557362711506282,
      "kl": 0.027069091796875,
      "learning_rate": 2e-06,
      "loss": 0.0124,
      "reward": 0.3437500074505806,
      "reward_std": 0.258017111569643,
      "rewards/accuracy_reward": 0.3437500074505806,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.1428680419922,
      "epoch": 0.7379134860050891,
      "grad_norm": 0.5670857388133471,
      "kl": 0.026580810546875,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.3125000149011612,
      "reward_std": 0.33708399534225464,
      "rewards/accuracy_reward": 0.3125000149011612,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.0870666503906,
      "epoch": 0.7430025445292621,
      "grad_norm": 0.568961626764239,
      "kl": 0.026275634765625,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.4598214402794838,
      "reward_std": 0.315539114177227,
      "rewards/accuracy_reward": 0.4598214402794838,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.8147430419922,
      "epoch": 0.7480916030534351,
      "grad_norm": 0.4740036629407517,
      "kl": 0.026611328125,
      "learning_rate": 2e-06,
      "loss": 0.0062,
      "reward": 0.4754464477300644,
      "reward_std": 0.26761702448129654,
      "rewards/accuracy_reward": 0.4754464477300644,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.73885345458984,
      "epoch": 0.7531806615776081,
      "grad_norm": 0.4295908650666131,
      "kl": 0.02606201171875,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.4107143059372902,
      "reward_std": 0.21619067713618279,
      "rewards/accuracy_reward": 0.4107143059372902,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 350.50001525878906,
      "epoch": 0.7582697201017812,
      "grad_norm": 0.5712708207232068,
      "kl": 0.028656005859375,
      "learning_rate": 2e-06,
      "loss": 0.032,
      "reward": 0.3325892947614193,
      "reward_std": 0.3545599430799484,
      "rewards/accuracy_reward": 0.3325892947614193,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.7835006713867,
      "epoch": 0.7633587786259542,
      "grad_norm": 0.4762004238380701,
      "kl": 0.028167724609375,
      "learning_rate": 2e-06,
      "loss": 0.0093,
      "reward": 0.4151785895228386,
      "reward_std": 0.2631988301873207,
      "rewards/accuracy_reward": 0.4151785895228386,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 362.5602722167969,
      "epoch": 0.7684478371501272,
      "grad_norm": 0.5534272693869284,
      "kl": 0.0289306640625,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.2879464477300644,
      "reward_std": 0.3091566897928715,
      "rewards/accuracy_reward": 0.2879464477300644,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 323.91519927978516,
      "epoch": 0.7735368956743003,
      "grad_norm": 0.5997096089269098,
      "kl": 0.032501220703125,
      "learning_rate": 2e-06,
      "loss": -0.0006,
      "reward": 0.2790178656578064,
      "reward_std": 0.27878880500793457,
      "rewards/accuracy_reward": 0.2790178656578064,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 321.62947845458984,
      "epoch": 0.7786259541984732,
      "grad_norm": 0.660276718752937,
      "kl": 0.0316162109375,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.3616071566939354,
      "reward_std": 0.31992514431476593,
      "rewards/accuracy_reward": 0.3616071566939354,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 324.59822845458984,
      "epoch": 0.7837150127226463,
      "grad_norm": 0.5640062542576917,
      "kl": 0.03253173828125,
      "learning_rate": 2e-06,
      "loss": -0.011,
      "reward": 0.4464285969734192,
      "reward_std": 0.27692829072475433,
      "rewards/accuracy_reward": 0.4464285969734192,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 302.19866943359375,
      "epoch": 0.7888040712468194,
      "grad_norm": 0.5839205172399542,
      "kl": 0.03533935546875,
      "learning_rate": 2e-06,
      "loss": -0.0094,
      "reward": 0.4598214477300644,
      "reward_std": 0.2679207921028137,
      "rewards/accuracy_reward": 0.4598214477300644,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 323.50670623779297,
      "epoch": 0.7938931297709924,
      "grad_norm": 0.6104848835648137,
      "kl": 0.0283203125,
      "learning_rate": 2e-06,
      "loss": -0.0073,
      "reward": 0.377232164144516,
      "reward_std": 0.32067109644412994,
      "rewards/accuracy_reward": 0.377232164144516,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 313.3616180419922,
      "epoch": 0.7989821882951654,
      "grad_norm": 0.5720085283610616,
      "kl": 0.029266357421875,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.4196428805589676,
      "reward_std": 0.30029767006635666,
      "rewards/accuracy_reward": 0.4196428805589676,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 354.7410888671875,
      "epoch": 0.8040712468193384,
      "grad_norm": 0.521897463534276,
      "kl": 0.024322509765625,
      "learning_rate": 2e-06,
      "loss": -0.0084,
      "reward": 0.3325893022119999,
      "reward_std": 0.2593177370727062,
      "rewards/accuracy_reward": 0.3325893022119999,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.61832427978516,
      "epoch": 0.8091603053435115,
      "grad_norm": 0.45568937591045094,
      "kl": 0.024078369140625,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.3147321566939354,
      "reward_std": 0.2266555279493332,
      "rewards/accuracy_reward": 0.3147321566939354,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.91519927978516,
      "epoch": 0.8142493638676844,
      "grad_norm": 0.5556974208639448,
      "kl": 0.024993896484375,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.290178582072258,
      "reward_std": 0.32750868797302246,
      "rewards/accuracy_reward": 0.290178582072258,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.08260345458984,
      "epoch": 0.8193384223918575,
      "grad_norm": 0.47142602312686566,
      "kl": 0.02667236328125,
      "learning_rate": 2e-06,
      "loss": 0.0084,
      "reward": 0.3392857387661934,
      "reward_std": 0.2555834725499153,
      "rewards/accuracy_reward": 0.3392857387661934,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.03572845458984,
      "epoch": 0.8244274809160306,
      "grad_norm": 0.4937371352762309,
      "kl": 0.025634765625,
      "learning_rate": 2e-06,
      "loss": 0.0209,
      "reward": 0.3549107350409031,
      "reward_std": 0.26360073685646057,
      "rewards/accuracy_reward": 0.3549107350409031,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.8415298461914,
      "epoch": 0.8295165394402035,
      "grad_norm": 0.4831961543470371,
      "kl": 0.02435302734375,
      "learning_rate": 2e-06,
      "loss": 0.0242,
      "reward": 0.4799107238650322,
      "reward_std": 0.2922033295035362,
      "rewards/accuracy_reward": 0.4799107238650322,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 454.8080596923828,
      "epoch": 0.8346055979643766,
      "grad_norm": 0.4581637953094632,
      "kl": 0.024139404296875,
      "learning_rate": 2e-06,
      "loss": -0.0089,
      "reward": 0.383928582072258,
      "reward_std": 0.300465926527977,
      "rewards/accuracy_reward": 0.383928582072258,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.7745666503906,
      "epoch": 0.8396946564885496,
      "grad_norm": 0.412366914780704,
      "kl": 0.02703857421875,
      "learning_rate": 2e-06,
      "loss": -0.0046,
      "reward": 0.3236607275903225,
      "reward_std": 0.18380586430430412,
      "rewards/accuracy_reward": 0.3236607275903225,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 433.7857360839844,
      "epoch": 0.8447837150127226,
      "grad_norm": 0.4332577326939765,
      "kl": 0.02471923828125,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.4799107387661934,
      "reward_std": 0.27713901177048683,
      "rewards/accuracy_reward": 0.4799107387661934,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.2768096923828,
      "epoch": 0.8498727735368957,
      "grad_norm": 0.4934250138706889,
      "kl": 0.02606201171875,
      "learning_rate": 2e-06,
      "loss": -0.0074,
      "reward": 0.2879464365541935,
      "reward_std": 0.2666035257279873,
      "rewards/accuracy_reward": 0.2879464365541935,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.50672149658203,
      "epoch": 0.8549618320610687,
      "grad_norm": 0.5135332913276037,
      "kl": 0.027008056640625,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.4486607387661934,
      "reward_std": 0.34256667643785477,
      "rewards/accuracy_reward": 0.4486607387661934,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 445.5960006713867,
      "epoch": 0.8600508905852418,
      "grad_norm": 0.4428290082641468,
      "kl": 0.025360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0083,
      "reward": 0.310267873108387,
      "reward_std": 0.28904811665415764,
      "rewards/accuracy_reward": 0.310267873108387,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.2500228881836,
      "epoch": 0.8651399491094147,
      "grad_norm": 0.5258875996753243,
      "kl": 0.0260009765625,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.341517873108387,
      "reward_std": 0.32597115635871887,
      "rewards/accuracy_reward": 0.341517873108387,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.9598388671875,
      "epoch": 0.8702290076335878,
      "grad_norm": 0.45207686350373955,
      "kl": 0.02337646484375,
      "learning_rate": 2e-06,
      "loss": 0.0032,
      "reward": 0.4107143022119999,
      "reward_std": 0.2683451995253563,
      "rewards/accuracy_reward": 0.4107143022119999,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.3638610839844,
      "epoch": 0.8753180661577609,
      "grad_norm": 0.42381339196292467,
      "kl": 0.02447509765625,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.3839285895228386,
      "reward_std": 0.2377573773264885,
      "rewards/accuracy_reward": 0.3839285895228386,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.27903747558594,
      "epoch": 0.8804071246819338,
      "grad_norm": 0.35561623304524603,
      "kl": 0.023101806640625,
      "learning_rate": 2e-06,
      "loss": -0.0098,
      "reward": 0.3191964477300644,
      "reward_std": 0.1617190968245268,
      "rewards/accuracy_reward": 0.3191964477300644,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.92859649658203,
      "epoch": 0.8854961832061069,
      "grad_norm": 0.3927903996471991,
      "kl": 0.025299072265625,
      "learning_rate": 2e-06,
      "loss": 0.0118,
      "reward": 0.439732164144516,
      "reward_std": 0.20010346360504627,
      "rewards/accuracy_reward": 0.439732164144516,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.2879638671875,
      "epoch": 0.8905852417302799,
      "grad_norm": 0.430060342574078,
      "kl": 0.024322509765625,
      "learning_rate": 2e-06,
      "loss": 0.0059,
      "reward": 0.3616071715950966,
      "reward_std": 0.22979383915662766,
      "rewards/accuracy_reward": 0.3616071715950966,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.3817138671875,
      "epoch": 0.8956743002544529,
      "grad_norm": 0.4500570691242079,
      "kl": 0.026123046875,
      "learning_rate": 2e-06,
      "loss": -0.0077,
      "reward": 0.4642857313156128,
      "reward_std": 0.25467485189437866,
      "rewards/accuracy_reward": 0.4642857313156128,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.4330520629883,
      "epoch": 0.9007633587786259,
      "grad_norm": 0.4689466745754758,
      "kl": 0.024078369140625,
      "learning_rate": 2e-06,
      "loss": 0.0004,
      "reward": 0.3973214477300644,
      "reward_std": 0.30610737577080727,
      "rewards/accuracy_reward": 0.3973214477300644,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.4174270629883,
      "epoch": 0.905852417302799,
      "grad_norm": 0.4388691039751368,
      "kl": 0.02703857421875,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.424107164144516,
      "reward_std": 0.2609916552901268,
      "rewards/accuracy_reward": 0.424107164144516,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.7299270629883,
      "epoch": 0.910941475826972,
      "grad_norm": 0.4180145109965147,
      "kl": 0.02935791015625,
      "learning_rate": 2e-06,
      "loss": 0.0045,
      "reward": 0.3772321566939354,
      "reward_std": 0.25678665190935135,
      "rewards/accuracy_reward": 0.3772321566939354,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 433.75225830078125,
      "epoch": 0.916030534351145,
      "grad_norm": 0.45666481451744956,
      "kl": 0.0283203125,
      "learning_rate": 2e-06,
      "loss": 0.0062,
      "reward": 0.23660715110599995,
      "reward_std": 0.2571524307131767,
      "rewards/accuracy_reward": 0.23660715110599995,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.3705596923828,
      "epoch": 0.9211195928753181,
      "grad_norm": 0.4965736901194939,
      "kl": 0.02862548828125,
      "learning_rate": 2e-06,
      "loss": 0.0117,
      "reward": 0.3928571492433548,
      "reward_std": 0.32339557260274887,
      "rewards/accuracy_reward": 0.3928571492433548,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.8861770629883,
      "epoch": 0.926208651399491,
      "grad_norm": 0.4738101599027936,
      "kl": 0.0289306640625,
      "learning_rate": 2e-06,
      "loss": -0.003,
      "reward": 0.3392857238650322,
      "reward_std": 0.27634909749031067,
      "rewards/accuracy_reward": 0.3392857238650322,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.6607360839844,
      "epoch": 0.9312977099236641,
      "grad_norm": 0.44741536703750234,
      "kl": 0.028106689453125,
      "learning_rate": 2e-06,
      "loss": 0.0005,
      "reward": 0.424107164144516,
      "reward_std": 0.2569168284535408,
      "rewards/accuracy_reward": 0.424107164144516,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.1294860839844,
      "epoch": 0.9363867684478372,
      "grad_norm": 0.4993649877556307,
      "kl": 0.02801513671875,
      "learning_rate": 2e-06,
      "loss": 0.0058,
      "reward": 0.3638393059372902,
      "reward_std": 0.27742746472358704,
      "rewards/accuracy_reward": 0.3638393059372902,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.8080596923828,
      "epoch": 0.9414758269720102,
      "grad_norm": 0.4702756887679677,
      "kl": 0.029876708984375,
      "learning_rate": 2e-06,
      "loss": 0.009,
      "reward": 0.341517873108387,
      "reward_std": 0.31923749297857285,
      "rewards/accuracy_reward": 0.341517873108387,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.7098388671875,
      "epoch": 0.9465648854961832,
      "grad_norm": 0.5106143671517,
      "kl": 0.03033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.3549107350409031,
      "reward_std": 0.28500746190547943,
      "rewards/accuracy_reward": 0.3549107350409031,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.80359649658203,
      "epoch": 0.9516539440203562,
      "grad_norm": 0.5394363221726483,
      "kl": 0.03045654296875,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.4754464477300644,
      "reward_std": 0.3313308395445347,
      "rewards/accuracy_reward": 0.4754464477300644,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.23885345458984,
      "epoch": 0.9567430025445293,
      "grad_norm": 0.48598201018594284,
      "kl": 0.029998779296875,
      "learning_rate": 2e-06,
      "loss": -0.0042,
      "reward": 0.3125000111758709,
      "reward_std": 0.2686355970799923,
      "rewards/accuracy_reward": 0.3125000111758709,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.5580520629883,
      "epoch": 0.9618320610687023,
      "grad_norm": 0.577078574347706,
      "kl": 0.02972412109375,
      "learning_rate": 2e-06,
      "loss": 0.0117,
      "reward": 0.3058035895228386,
      "reward_std": 0.3343946039676666,
      "rewards/accuracy_reward": 0.3058035895228386,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.7835006713867,
      "epoch": 0.9669211195928753,
      "grad_norm": 0.5538549733946959,
      "kl": 0.03118896484375,
      "learning_rate": 2e-06,
      "loss": 0.0086,
      "reward": 0.3705357313156128,
      "reward_std": 0.31720490753650665,
      "rewards/accuracy_reward": 0.3705357313156128,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.1116180419922,
      "epoch": 0.9720101781170484,
      "grad_norm": 0.4602172125795691,
      "kl": 0.03082275390625,
      "learning_rate": 2e-06,
      "loss": 0.0045,
      "reward": 0.4419643133878708,
      "reward_std": 0.2720428891479969,
      "rewards/accuracy_reward": 0.4419643133878708,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.6964416503906,
      "epoch": 0.9770992366412213,
      "grad_norm": 0.44511046078232314,
      "kl": 0.0264892578125,
      "learning_rate": 2e-06,
      "loss": -0.0051,
      "reward": 0.4129464440047741,
      "reward_std": 0.204301031306386,
      "rewards/accuracy_reward": 0.4129464440047741,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.27233123779297,
      "epoch": 0.9821882951653944,
      "grad_norm": 0.6368128639064,
      "kl": 0.036834716796875,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.424107164144516,
      "reward_std": 0.3008757568895817,
      "rewards/accuracy_reward": 0.424107164144516,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.9107360839844,
      "epoch": 0.9872773536895675,
      "grad_norm": 0.42714163361676444,
      "kl": 0.029388427734375,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.37053573317825794,
      "reward_std": 0.19569924473762512,
      "rewards/accuracy_reward": 0.37053573317825794,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.43751525878906,
      "epoch": 0.9923664122137404,
      "grad_norm": 0.5126159949792717,
      "kl": 0.028045654296875,
      "learning_rate": 2e-06,
      "loss": 0.0046,
      "reward": 0.4933035932481289,
      "reward_std": 0.3095056377351284,
      "rewards/accuracy_reward": 0.4933035932481289,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 354.4423141479492,
      "epoch": 0.9974554707379135,
      "grad_norm": 0.477609157386284,
      "kl": 0.029541015625,
      "learning_rate": 2e-06,
      "loss": 0.012,
      "reward": 0.4218750223517418,
      "reward_std": 0.22727755084633827,
      "rewards/accuracy_reward": 0.4218750223517418,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 434.3928756713867,
      "epoch": 1.005089058524173,
      "grad_norm": 0.45629019648922187,
      "kl": 0.028839111328125,
      "learning_rate": 2e-06,
      "loss": 0.0129,
      "reward": 0.4151785895228386,
      "reward_std": 0.20347346365451813,
      "rewards/accuracy_reward": 0.4151785895228386,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.9397506713867,
      "epoch": 1.0101781170483461,
      "grad_norm": 0.4852725436095131,
      "kl": 0.03228759765625,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.5714285969734192,
      "reward_std": 0.24667690321803093,
      "rewards/accuracy_reward": 0.5714285969734192,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.6919860839844,
      "epoch": 1.015267175572519,
      "grad_norm": 0.46684070027513186,
      "kl": 0.03106689453125,
      "learning_rate": 2e-06,
      "loss": -0.0063,
      "reward": 0.3973214477300644,
      "reward_std": 0.24830005317926407,
      "rewards/accuracy_reward": 0.3973214477300644,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.1518020629883,
      "epoch": 1.020356234096692,
      "grad_norm": 0.47365885205499736,
      "kl": 0.0308837890625,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.3928571715950966,
      "reward_std": 0.2442369982600212,
      "rewards/accuracy_reward": 0.3928571715950966,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.6294860839844,
      "epoch": 1.025445292620865,
      "grad_norm": 0.5216914587175241,
      "kl": 0.03045654296875,
      "learning_rate": 2e-06,
      "loss": 0.0095,
      "reward": 0.5781250223517418,
      "reward_std": 0.24771291390061378,
      "rewards/accuracy_reward": 0.5781250223517418,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.7143020629883,
      "epoch": 1.0305343511450382,
      "grad_norm": 0.49886249418845297,
      "kl": 0.034149169921875,
      "learning_rate": 2e-06,
      "loss": 0.0167,
      "reward": 0.466517873108387,
      "reward_std": 0.26621802896261215,
      "rewards/accuracy_reward": 0.466517873108387,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.1049270629883,
      "epoch": 1.0356234096692112,
      "grad_norm": 0.49883207537024243,
      "kl": 0.035430908203125,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.4888393059372902,
      "reward_std": 0.29329938814044,
      "rewards/accuracy_reward": 0.4888393059372902,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.4710006713867,
      "epoch": 1.0407124681933841,
      "grad_norm": 0.4743305957167603,
      "kl": 0.0341796875,
      "learning_rate": 2e-06,
      "loss": 0.0058,
      "reward": 0.6093750223517418,
      "reward_std": 0.27687905728816986,
      "rewards/accuracy_reward": 0.6093750223517418,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.2366256713867,
      "epoch": 1.0458015267175573,
      "grad_norm": 0.41185213945826243,
      "kl": 0.031585693359375,
      "learning_rate": 2e-06,
      "loss": 0.005,
      "reward": 0.667410746216774,
      "reward_std": 0.1963009387254715,
      "rewards/accuracy_reward": 0.667410746216774,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.06251525878906,
      "epoch": 1.0508905852417303,
      "grad_norm": 0.4319517303591335,
      "kl": 0.029632568359375,
      "learning_rate": 2e-06,
      "loss": 0.0061,
      "reward": 0.542410746216774,
      "reward_std": 0.2033972479403019,
      "rewards/accuracy_reward": 0.542410746216774,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.9486846923828,
      "epoch": 1.0559796437659033,
      "grad_norm": 0.4428543206563001,
      "kl": 0.030517578125,
      "learning_rate": 2e-06,
      "loss": -0.0139,
      "reward": 0.5781250149011612,
      "reward_std": 0.21149570867419243,
      "rewards/accuracy_reward": 0.5781250149011612,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.04466247558594,
      "epoch": 1.0610687022900764,
      "grad_norm": 0.4095348359436766,
      "kl": 0.03240966796875,
      "learning_rate": 2e-06,
      "loss": 0.0137,
      "reward": 0.448660746216774,
      "reward_std": 0.18329654354602098,
      "rewards/accuracy_reward": 0.448660746216774,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.1138610839844,
      "epoch": 1.0661577608142494,
      "grad_norm": 0.4796397496935904,
      "kl": 0.030426025390625,
      "learning_rate": 2e-06,
      "loss": -0.004,
      "reward": 0.4732143059372902,
      "reward_std": 0.24544759839773178,
      "rewards/accuracy_reward": 0.4732143059372902,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.3973388671875,
      "epoch": 1.0712468193384224,
      "grad_norm": 0.9274552338240573,
      "kl": 0.039306640625,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.5982143059372902,
      "reward_std": 0.2766333222389221,
      "rewards/accuracy_reward": 0.5982143059372902,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.5335006713867,
      "epoch": 1.0763358778625953,
      "grad_norm": 0.4468713413539593,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": 0.0031,
      "reward": 0.562500037252903,
      "reward_std": 0.19667476043105125,
      "rewards/accuracy_reward": 0.562500037252903,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.85491943359375,
      "epoch": 1.0814249363867685,
      "grad_norm": 0.4865455605392445,
      "kl": 0.031768798828125,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.600446455180645,
      "reward_std": 0.27646007761359215,
      "rewards/accuracy_reward": 0.600446455180645,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 368.4665298461914,
      "epoch": 1.0865139949109415,
      "grad_norm": 0.4958354129620625,
      "kl": 0.033935546875,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.5334821715950966,
      "reward_std": 0.26529859378933907,
      "rewards/accuracy_reward": 0.5334821715950966,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.0424270629883,
      "epoch": 1.0916030534351144,
      "grad_norm": 0.47123769690556155,
      "kl": 0.03216552734375,
      "learning_rate": 2e-06,
      "loss": 0.0105,
      "reward": 0.4866071715950966,
      "reward_std": 0.22155790403485298,
      "rewards/accuracy_reward": 0.4866071715950966,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.37500762939453,
      "epoch": 1.0966921119592876,
      "grad_norm": 0.326585929171639,
      "kl": 0.03814697265625,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.642857164144516,
      "reward_std": 0.11197268962860107,
      "rewards/accuracy_reward": 0.642857164144516,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.30135345458984,
      "epoch": 1.1017811704834606,
      "grad_norm": 0.44026320719542633,
      "kl": 0.03521728515625,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.464285746216774,
      "reward_std": 0.28557220473885536,
      "rewards/accuracy_reward": 0.464285746216774,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.1540298461914,
      "epoch": 1.1068702290076335,
      "grad_norm": 0.446293170783821,
      "kl": 0.03741455078125,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.4910714477300644,
      "reward_std": 0.17842019442468882,
      "rewards/accuracy_reward": 0.4910714477300644,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.4352798461914,
      "epoch": 1.1119592875318065,
      "grad_norm": 0.3950797046488124,
      "kl": 0.036041259765625,
      "learning_rate": 2e-06,
      "loss": -0.0117,
      "reward": 0.5491071715950966,
      "reward_std": 0.18994246423244476,
      "rewards/accuracy_reward": 0.5491071715950966,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.73663330078125,
      "epoch": 1.1170483460559797,
      "grad_norm": 0.4317642165996371,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": 0.0164,
      "reward": 0.5000000149011612,
      "reward_std": 0.18737347051501274,
      "rewards/accuracy_reward": 0.5000000149011612,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.8727798461914,
      "epoch": 1.1221374045801527,
      "grad_norm": 0.48090345298021026,
      "kl": 0.037322998046875,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.5915178805589676,
      "reward_std": 0.1756830532103777,
      "rewards/accuracy_reward": 0.5915178805589676,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.70314025878906,
      "epoch": 1.1272264631043256,
      "grad_norm": 0.3540583574823124,
      "kl": 0.0330810546875,
      "learning_rate": 2e-06,
      "loss": -0.0116,
      "reward": 0.5803571864962578,
      "reward_std": 0.14323088619858027,
      "rewards/accuracy_reward": 0.5803571864962578,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.71876525878906,
      "epoch": 1.1323155216284988,
      "grad_norm": 0.4245122051677136,
      "kl": 0.033111572265625,
      "learning_rate": 2e-06,
      "loss": 0.0217,
      "reward": 0.502232164144516,
      "reward_std": 0.2278486266732216,
      "rewards/accuracy_reward": 0.502232164144516,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.1272506713867,
      "epoch": 1.1374045801526718,
      "grad_norm": 0.3779360649176341,
      "kl": 0.03619384765625,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.5312500298023224,
      "reward_std": 0.21440864354372025,
      "rewards/accuracy_reward": 0.5312500298023224,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.6942138671875,
      "epoch": 1.1424936386768447,
      "grad_norm": 0.3296850361604471,
      "kl": 0.032379150390625,
      "learning_rate": 2e-06,
      "loss": 0.0079,
      "reward": 0.6584821790456772,
      "reward_std": 0.11233820579946041,
      "rewards/accuracy_reward": 0.6584821790456772,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 435.2812728881836,
      "epoch": 1.1475826972010177,
      "grad_norm": 0.44279309621022384,
      "kl": 0.03387451171875,
      "learning_rate": 2e-06,
      "loss": -0.0073,
      "reward": 0.5000000074505806,
      "reward_std": 0.23925619572401047,
      "rewards/accuracy_reward": 0.5000000074505806,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 445.60269927978516,
      "epoch": 1.1526717557251909,
      "grad_norm": 0.3810375840986681,
      "kl": 0.03192138671875,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.435267873108387,
      "reward_std": 0.18524267710745335,
      "rewards/accuracy_reward": 0.435267873108387,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.0446548461914,
      "epoch": 1.1577608142493638,
      "grad_norm": 0.45017396993067865,
      "kl": 0.035614013671875,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.491071455180645,
      "reward_std": 0.2348989974707365,
      "rewards/accuracy_reward": 0.491071455180645,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.5513610839844,
      "epoch": 1.162849872773537,
      "grad_norm": 0.3586093802630662,
      "kl": 0.031402587890625,
      "learning_rate": 2e-06,
      "loss": 0.014,
      "reward": 0.5848214402794838,
      "reward_std": 0.2014264166355133,
      "rewards/accuracy_reward": 0.5848214402794838,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 450.0111846923828,
      "epoch": 1.16793893129771,
      "grad_norm": 0.3888403716091177,
      "kl": 0.029754638671875,
      "learning_rate": 2e-06,
      "loss": 0.0123,
      "reward": 0.6383928954601288,
      "reward_std": 0.16469183936715126,
      "rewards/accuracy_reward": 0.6383928954601288,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.9308166503906,
      "epoch": 1.173027989821883,
      "grad_norm": 0.3757730072446275,
      "kl": 0.035491943359375,
      "learning_rate": 2e-06,
      "loss": 0.006,
      "reward": 0.6205357387661934,
      "reward_std": 0.1877366341650486,
      "rewards/accuracy_reward": 0.6205357387661934,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.8995590209961,
      "epoch": 1.178117048346056,
      "grad_norm": 0.36476781267980535,
      "kl": 0.030426025390625,
      "learning_rate": 2e-06,
      "loss": 0.0085,
      "reward": 0.705357164144516,
      "reward_std": 0.16365888342261314,
      "rewards/accuracy_reward": 0.705357164144516,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.4085006713867,
      "epoch": 1.183206106870229,
      "grad_norm": 0.42247243573537296,
      "kl": 0.029266357421875,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.5089285895228386,
      "reward_std": 0.2098158746957779,
      "rewards/accuracy_reward": 0.5089285895228386,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.1183166503906,
      "epoch": 1.188295165394402,
      "grad_norm": 0.3372301931726151,
      "kl": 0.029815673828125,
      "learning_rate": 2e-06,
      "loss": -0.0018,
      "reward": 0.4933035969734192,
      "reward_std": 0.19880928099155426,
      "rewards/accuracy_reward": 0.4933035969734192,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.52457427978516,
      "epoch": 1.193384223918575,
      "grad_norm": 0.3984313133748342,
      "kl": 0.033966064453125,
      "learning_rate": 2e-06,
      "loss": -0.0109,
      "reward": 0.5223214626312256,
      "reward_std": 0.17286018654704094,
      "rewards/accuracy_reward": 0.5223214626312256,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.57591247558594,
      "epoch": 1.1984732824427482,
      "grad_norm": 0.4800689228399824,
      "kl": 0.0406494140625,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.5223214402794838,
      "reward_std": 0.2609461396932602,
      "rewards/accuracy_reward": 0.5223214402794838,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.08483123779297,
      "epoch": 1.2035623409669212,
      "grad_norm": 0.4084541086945262,
      "kl": 0.0341796875,
      "learning_rate": 2e-06,
      "loss": 0.0039,
      "reward": 0.5424107387661934,
      "reward_std": 0.17931898683309555,
      "rewards/accuracy_reward": 0.5424107387661934,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.9665298461914,
      "epoch": 1.2086513994910941,
      "grad_norm": 0.4019034905125619,
      "kl": 0.033935546875,
      "learning_rate": 2e-06,
      "loss": -0.0024,
      "reward": 0.647321455180645,
      "reward_std": 0.18874311447143555,
      "rewards/accuracy_reward": 0.647321455180645,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.82814025878906,
      "epoch": 1.213740458015267,
      "grad_norm": 0.48533269021127756,
      "kl": 0.0325927734375,
      "learning_rate": 2e-06,
      "loss": -0.0053,
      "reward": 0.5468750298023224,
      "reward_std": 0.254442248493433,
      "rewards/accuracy_reward": 0.5468750298023224,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.8214569091797,
      "epoch": 1.2188295165394403,
      "grad_norm": 0.3491733026298112,
      "kl": 0.029266357421875,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.517857164144516,
      "reward_std": 0.1553249955177307,
      "rewards/accuracy_reward": 0.517857164144516,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.12501525878906,
      "epoch": 1.2239185750636132,
      "grad_norm": 0.3738114254251958,
      "kl": 0.030853271484375,
      "learning_rate": 2e-06,
      "loss": -0.0029,
      "reward": 0.5200893208384514,
      "reward_std": 0.22691549360752106,
      "rewards/accuracy_reward": 0.5200893208384514,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 369.85045623779297,
      "epoch": 1.2290076335877862,
      "grad_norm": 0.4317542008409837,
      "kl": 0.034698486328125,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.4241071566939354,
      "reward_std": 0.19226149097085,
      "rewards/accuracy_reward": 0.4241071566939354,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.1227798461914,
      "epoch": 1.2340966921119594,
      "grad_norm": 0.39238353022547695,
      "kl": 0.03424072265625,
      "learning_rate": 2e-06,
      "loss": -0.008,
      "reward": 0.569196455180645,
      "reward_std": 0.17385854944586754,
      "rewards/accuracy_reward": 0.569196455180645,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.10716247558594,
      "epoch": 1.2391857506361323,
      "grad_norm": 0.43920936279747014,
      "kl": 0.029815673828125,
      "learning_rate": 2e-06,
      "loss": 0.0136,
      "reward": 0.3593750223517418,
      "reward_std": 0.24526547640562057,
      "rewards/accuracy_reward": 0.3593750223517418,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.65626525878906,
      "epoch": 1.2442748091603053,
      "grad_norm": 0.41848082095968575,
      "kl": 0.032684326171875,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.5825893133878708,
      "reward_std": 0.21210570260882378,
      "rewards/accuracy_reward": 0.5825893133878708,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.6562728881836,
      "epoch": 1.2493638676844783,
      "grad_norm": 0.5011391731279294,
      "kl": 0.03106689453125,
      "learning_rate": 2e-06,
      "loss": -0.0091,
      "reward": 0.5781250149011612,
      "reward_std": 0.26321787759661674,
      "rewards/accuracy_reward": 0.5781250149011612,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.5982360839844,
      "epoch": 1.2544529262086515,
      "grad_norm": 0.3997580217150404,
      "kl": 0.035614013671875,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.4687500074505806,
      "reward_std": 0.15454928297549486,
      "rewards/accuracy_reward": 0.4687500074505806,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.3995666503906,
      "epoch": 1.2595419847328244,
      "grad_norm": 0.487790323685233,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.0148,
      "reward": 0.511160746216774,
      "reward_std": 0.24315107986330986,
      "rewards/accuracy_reward": 0.511160746216774,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.16297149658203,
      "epoch": 1.2646310432569974,
      "grad_norm": 0.38761795177162306,
      "kl": 0.029388427734375,
      "learning_rate": 2e-06,
      "loss": 0.0152,
      "reward": 0.611607164144516,
      "reward_std": 0.21232472360134125,
      "rewards/accuracy_reward": 0.611607164144516,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.4442138671875,
      "epoch": 1.2697201017811706,
      "grad_norm": 0.49600390157546337,
      "kl": 0.031982421875,
      "learning_rate": 2e-06,
      "loss": 0.0028,
      "reward": 0.3750000149011612,
      "reward_std": 0.2584892399609089,
      "rewards/accuracy_reward": 0.3750000149011612,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.50001525878906,
      "epoch": 1.2748091603053435,
      "grad_norm": 0.42773230362935344,
      "kl": 0.036529541015625,
      "learning_rate": 2e-06,
      "loss": -0.0068,
      "reward": 0.4375000149011612,
      "reward_std": 0.2202574610710144,
      "rewards/accuracy_reward": 0.4375000149011612,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 352.59153747558594,
      "epoch": 1.2798982188295165,
      "grad_norm": 0.5330881236514794,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.618303582072258,
      "reward_std": 0.27262888848781586,
      "rewards/accuracy_reward": 0.618303582072258,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.53126525878906,
      "epoch": 1.2849872773536894,
      "grad_norm": 0.48866138610644344,
      "kl": 0.031524658203125,
      "learning_rate": 2e-06,
      "loss": 0.0105,
      "reward": 0.5848214626312256,
      "reward_std": 0.2368122972548008,
      "rewards/accuracy_reward": 0.5848214626312256,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.4620666503906,
      "epoch": 1.2900763358778626,
      "grad_norm": 0.3896532735098542,
      "kl": 0.03399658203125,
      "learning_rate": 2e-06,
      "loss": 0.0163,
      "reward": 0.7366071790456772,
      "reward_std": 0.18220902606844902,
      "rewards/accuracy_reward": 0.7366071790456772,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.18082427978516,
      "epoch": 1.2951653944020356,
      "grad_norm": 0.486253566989,
      "kl": 0.03790283203125,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.4933036044239998,
      "reward_std": 0.2729470394551754,
      "rewards/accuracy_reward": 0.4933036044239998,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.66966247558594,
      "epoch": 1.3002544529262088,
      "grad_norm": 0.45345429544975063,
      "kl": 0.0303955078125,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.4531250149011612,
      "reward_std": 0.2670621797442436,
      "rewards/accuracy_reward": 0.4531250149011612,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.8058319091797,
      "epoch": 1.3053435114503817,
      "grad_norm": 0.438097016293131,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.6450893133878708,
      "reward_std": 0.16849614307284355,
      "rewards/accuracy_reward": 0.6450893133878708,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.25447845458984,
      "epoch": 1.3104325699745547,
      "grad_norm": 0.359204988846788,
      "kl": 0.035003662109375,
      "learning_rate": 2e-06,
      "loss": 0.0191,
      "reward": 0.651785746216774,
      "reward_std": 0.16984091699123383,
      "rewards/accuracy_reward": 0.651785746216774,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.47322845458984,
      "epoch": 1.3155216284987277,
      "grad_norm": 0.46896489998912644,
      "kl": 0.034515380859375,
      "learning_rate": 2e-06,
      "loss": 0.0054,
      "reward": 0.5558035969734192,
      "reward_std": 0.2598639354109764,
      "rewards/accuracy_reward": 0.5558035969734192,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.02903747558594,
      "epoch": 1.3206106870229006,
      "grad_norm": 0.3417142397006384,
      "kl": 0.031768798828125,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.651785746216774,
      "reward_std": 0.15594056900590658,
      "rewards/accuracy_reward": 0.651785746216774,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.66072845458984,
      "epoch": 1.3256997455470738,
      "grad_norm": 0.4718016390791712,
      "kl": 0.03558349609375,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.5312500223517418,
      "reward_std": 0.22380899265408516,
      "rewards/accuracy_reward": 0.5312500223517418,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.72547149658203,
      "epoch": 1.3307888040712468,
      "grad_norm": 0.3798765837581581,
      "kl": 0.0338134765625,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.5044643059372902,
      "reward_std": 0.15109536796808243,
      "rewards/accuracy_reward": 0.5044643059372902,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.7143020629883,
      "epoch": 1.33587786259542,
      "grad_norm": 0.48318812132563077,
      "kl": 0.035064697265625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.5870536044239998,
      "reward_std": 0.2547951564192772,
      "rewards/accuracy_reward": 0.5870536044239998,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.0022506713867,
      "epoch": 1.340966921119593,
      "grad_norm": 0.39300607799916687,
      "kl": 0.031982421875,
      "learning_rate": 2e-06,
      "loss": 0.0131,
      "reward": 0.5825893133878708,
      "reward_std": 0.21780962124466896,
      "rewards/accuracy_reward": 0.5825893133878708,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.43528747558594,
      "epoch": 1.3460559796437659,
      "grad_norm": 0.490922251385516,
      "kl": 0.0289306640625,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.5089286044239998,
      "reward_std": 0.2536001205444336,
      "rewards/accuracy_reward": 0.5089286044239998,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.38394927978516,
      "epoch": 1.3511450381679388,
      "grad_norm": 0.3978566875964569,
      "kl": 0.03375244140625,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.6651786118745804,
      "reward_std": 0.17543086595833302,
      "rewards/accuracy_reward": 0.6651786118745804,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.9933166503906,
      "epoch": 1.356234096692112,
      "grad_norm": 0.4691451616822436,
      "kl": 0.033203125,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.5133928805589676,
      "reward_std": 0.2103098053485155,
      "rewards/accuracy_reward": 0.5133928805589676,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.1361846923828,
      "epoch": 1.361323155216285,
      "grad_norm": 0.371972857118148,
      "kl": 0.029083251953125,
      "learning_rate": 2e-06,
      "loss": 0.0142,
      "reward": 0.6718750298023224,
      "reward_std": 0.17460346966981888,
      "rewards/accuracy_reward": 0.6718750298023224,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.89734649658203,
      "epoch": 1.366412213740458,
      "grad_norm": 0.5071890470760458,
      "kl": 0.031951904296875,
      "learning_rate": 2e-06,
      "loss": -0.0029,
      "reward": 0.593750037252903,
      "reward_std": 0.30577272549271584,
      "rewards/accuracy_reward": 0.593750037252903,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.30582427978516,
      "epoch": 1.3715012722646311,
      "grad_norm": 0.3804687691660587,
      "kl": 0.02972412109375,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.573660746216774,
      "reward_std": 0.2071095071732998,
      "rewards/accuracy_reward": 0.573660746216774,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.37278747558594,
      "epoch": 1.376590330788804,
      "grad_norm": 0.4090213962618039,
      "kl": 0.0294189453125,
      "learning_rate": 2e-06,
      "loss": 0.0117,
      "reward": 0.5446428954601288,
      "reward_std": 0.1941103432327509,
      "rewards/accuracy_reward": 0.5446428954601288,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.1897506713867,
      "epoch": 1.381679389312977,
      "grad_norm": 0.48042069051097636,
      "kl": 0.0303955078125,
      "learning_rate": 2e-06,
      "loss": -0.0069,
      "reward": 0.5401785895228386,
      "reward_std": 0.27686627209186554,
      "rewards/accuracy_reward": 0.5401785895228386,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.62501525878906,
      "epoch": 1.38676844783715,
      "grad_norm": 0.45262121383079323,
      "kl": 0.02978515625,
      "learning_rate": 2e-06,
      "loss": 0.0141,
      "reward": 0.5111607313156128,
      "reward_std": 0.22661678865551949,
      "rewards/accuracy_reward": 0.5111607313156128,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.17635345458984,
      "epoch": 1.3918575063613232,
      "grad_norm": 0.4179025179046707,
      "kl": 0.030609130859375,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.5758928805589676,
      "reward_std": 0.19894640147686005,
      "rewards/accuracy_reward": 0.5758928805589676,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.39510345458984,
      "epoch": 1.3969465648854962,
      "grad_norm": 0.4180587963753563,
      "kl": 0.028350830078125,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.5803571790456772,
      "reward_std": 0.21554413810372353,
      "rewards/accuracy_reward": 0.5803571790456772,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.1718978881836,
      "epoch": 1.4020356234096691,
      "grad_norm": 0.4589957571854596,
      "kl": 0.032562255859375,
      "learning_rate": 2e-06,
      "loss": -0.0039,
      "reward": 0.5870536044239998,
      "reward_std": 0.2515076510608196,
      "rewards/accuracy_reward": 0.5870536044239998,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.12947845458984,
      "epoch": 1.4071246819338423,
      "grad_norm": 0.4488105381852421,
      "kl": 0.029541015625,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.5870536118745804,
      "reward_std": 0.23234201595187187,
      "rewards/accuracy_reward": 0.5870536118745804,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.8147506713867,
      "epoch": 1.4122137404580153,
      "grad_norm": 0.49935408777879425,
      "kl": 0.0369873046875,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.5290178805589676,
      "reward_std": 0.25455041602253914,
      "rewards/accuracy_reward": 0.5290178805589676,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.8549270629883,
      "epoch": 1.4173027989821882,
      "grad_norm": 0.45934001217436377,
      "kl": 0.03057861328125,
      "learning_rate": 2e-06,
      "loss": 0.0141,
      "reward": 0.589285746216774,
      "reward_std": 0.2629491277039051,
      "rewards/accuracy_reward": 0.589285746216774,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.23885345458984,
      "epoch": 1.4223918575063612,
      "grad_norm": 0.3803307352328088,
      "kl": 0.030548095703125,
      "learning_rate": 2e-06,
      "loss": 0.0155,
      "reward": 0.5200893059372902,
      "reward_std": 0.22422028705477715,
      "rewards/accuracy_reward": 0.5200893059372902,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.8772506713867,
      "epoch": 1.4274809160305344,
      "grad_norm": 0.40325345382090017,
      "kl": 0.03460693359375,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.6540178954601288,
      "reward_std": 0.15160241909325123,
      "rewards/accuracy_reward": 0.6540178954601288,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.95984649658203,
      "epoch": 1.4325699745547074,
      "grad_norm": 0.47073666845442763,
      "kl": 0.032440185546875,
      "learning_rate": 2e-06,
      "loss": -0.0036,
      "reward": 0.3883928805589676,
      "reward_std": 0.2049722783267498,
      "rewards/accuracy_reward": 0.3883928805589676,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.3616256713867,
      "epoch": 1.4376590330788805,
      "grad_norm": 0.4470282122223272,
      "kl": 0.02899169921875,
      "learning_rate": 2e-06,
      "loss": 0.012,
      "reward": 0.649553582072258,
      "reward_std": 0.1916458122432232,
      "rewards/accuracy_reward": 0.649553582072258,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.8638610839844,
      "epoch": 1.4427480916030535,
      "grad_norm": 0.39981447818567717,
      "kl": 0.03497314453125,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.6093750223517418,
      "reward_std": 0.17711340822279453,
      "rewards/accuracy_reward": 0.6093750223517418,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.58038330078125,
      "epoch": 1.4478371501272265,
      "grad_norm": 0.3820981361187511,
      "kl": 0.03021240234375,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.6205357387661934,
      "reward_std": 0.16524812299758196,
      "rewards/accuracy_reward": 0.6205357387661934,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.3236846923828,
      "epoch": 1.4529262086513994,
      "grad_norm": 0.37936483564842083,
      "kl": 0.0384521484375,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.5379464626312256,
      "reward_std": 0.1923013273626566,
      "rewards/accuracy_reward": 0.5379464626312256,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.3660888671875,
      "epoch": 1.4580152671755724,
      "grad_norm": 0.33842922303195444,
      "kl": 0.03204345703125,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.4308035895228386,
      "reward_std": 0.1204148530960083,
      "rewards/accuracy_reward": 0.4308035895228386,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.7768096923828,
      "epoch": 1.4631043256997456,
      "grad_norm": 0.43404556961180785,
      "kl": 0.029266357421875,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6406250298023224,
      "reward_std": 0.2344888113439083,
      "rewards/accuracy_reward": 0.6406250298023224,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.2835006713867,
      "epoch": 1.4681933842239185,
      "grad_norm": 0.3645532284578946,
      "kl": 0.029327392578125,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.5334821678698063,
      "reward_std": 0.17899974528700113,
      "rewards/accuracy_reward": 0.5334821678698063,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.8750305175781,
      "epoch": 1.4732824427480917,
      "grad_norm": 0.29925358250241385,
      "kl": 0.03021240234375,
      "learning_rate": 2e-06,
      "loss": 0.0059,
      "reward": 0.6763393133878708,
      "reward_std": 0.12090125773102045,
      "rewards/accuracy_reward": 0.6763393133878708,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.1741256713867,
      "epoch": 1.4783715012722647,
      "grad_norm": 0.41734257737695496,
      "kl": 0.03192138671875,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.7410714477300644,
      "reward_std": 0.17781273648142815,
      "rewards/accuracy_reward": 0.7410714477300644,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.84376525878906,
      "epoch": 1.4834605597964376,
      "grad_norm": 0.4388068708751415,
      "kl": 0.0333251953125,
      "learning_rate": 2e-06,
      "loss": 0.0123,
      "reward": 0.6450893208384514,
      "reward_std": 0.2453245185315609,
      "rewards/accuracy_reward": 0.6450893208384514,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.1785888671875,
      "epoch": 1.4885496183206106,
      "grad_norm": 0.3374416338879907,
      "kl": 0.027130126953125,
      "learning_rate": 2e-06,
      "loss": 0.0131,
      "reward": 0.6808035969734192,
      "reward_std": 0.15548420883715153,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.5960006713867,
      "epoch": 1.4936386768447838,
      "grad_norm": 0.3458868433278366,
      "kl": 0.031585693359375,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.5647321715950966,
      "reward_std": 0.1358319204300642,
      "rewards/accuracy_reward": 0.5647321715950966,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.8526916503906,
      "epoch": 1.4987277353689568,
      "grad_norm": 0.3211074271463654,
      "kl": 0.0347900390625,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.6562500447034836,
      "reward_std": 0.13926138915121555,
      "rewards/accuracy_reward": 0.6562500447034836,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.2120666503906,
      "epoch": 1.5038167938931297,
      "grad_norm": 0.38219937811282745,
      "kl": 0.0341796875,
      "learning_rate": 2e-06,
      "loss": 0.0079,
      "reward": 0.6674107387661934,
      "reward_std": 0.20987043622881174,
      "rewards/accuracy_reward": 0.6674107387661934,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.3593978881836,
      "epoch": 1.508905852417303,
      "grad_norm": 0.43162745395979,
      "kl": 0.0322265625,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.5513393059372902,
      "reward_std": 0.2469811663031578,
      "rewards/accuracy_reward": 0.5513393059372902,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.5714416503906,
      "epoch": 1.5139949109414759,
      "grad_norm": 0.4450368481965956,
      "kl": 0.03717041015625,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.6250000298023224,
      "reward_std": 0.2539636231958866,
      "rewards/accuracy_reward": 0.6250000298023224,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.12501525878906,
      "epoch": 1.5190839694656488,
      "grad_norm": 0.3218231617500364,
      "kl": 0.033233642578125,
      "learning_rate": 2e-06,
      "loss": -0.0039,
      "reward": 0.4888393059372902,
      "reward_std": 0.14116082433611155,
      "rewards/accuracy_reward": 0.4888393059372902,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.96653747558594,
      "epoch": 1.5241730279898218,
      "grad_norm": 0.41128833810114596,
      "kl": 0.03143310546875,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.517857164144516,
      "reward_std": 0.20962579175829887,
      "rewards/accuracy_reward": 0.517857164144516,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.5268096923828,
      "epoch": 1.5292620865139948,
      "grad_norm": 0.3854823268565017,
      "kl": 0.031005859375,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.4732143133878708,
      "reward_std": 0.21179921738803387,
      "rewards/accuracy_reward": 0.4732143133878708,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.1897506713867,
      "epoch": 1.534351145038168,
      "grad_norm": 0.3507539999839261,
      "kl": 0.03533935546875,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.7120536118745804,
      "reward_std": 0.18208618462085724,
      "rewards/accuracy_reward": 0.7120536118745804,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.68082427978516,
      "epoch": 1.5394402035623411,
      "grad_norm": 0.4084442329727315,
      "kl": 0.035064697265625,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.5825893133878708,
      "reward_std": 0.19158652145415545,
      "rewards/accuracy_reward": 0.5825893133878708,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 435.0580520629883,
      "epoch": 1.544529262086514,
      "grad_norm": 0.4381021420029986,
      "kl": 0.034912109375,
      "learning_rate": 2e-06,
      "loss": -0.0014,
      "reward": 0.5357142984867096,
      "reward_std": 0.21104642748832703,
      "rewards/accuracy_reward": 0.5357142984867096,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.93082427978516,
      "epoch": 1.549618320610687,
      "grad_norm": 0.4277718986497233,
      "kl": 0.03472900390625,
      "learning_rate": 2e-06,
      "loss": 0.0099,
      "reward": 0.6406250298023224,
      "reward_std": 0.19158761575818062,
      "rewards/accuracy_reward": 0.6406250298023224,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.15403747558594,
      "epoch": 1.55470737913486,
      "grad_norm": 0.3701582237907345,
      "kl": 0.03363037109375,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.6205357536673546,
      "reward_std": 0.1558861043304205,
      "rewards/accuracy_reward": 0.6205357536673546,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.51564025878906,
      "epoch": 1.559796437659033,
      "grad_norm": 0.3962037612138367,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.4642857387661934,
      "reward_std": 0.19843707978725433,
      "rewards/accuracy_reward": 0.4642857387661934,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.88172149658203,
      "epoch": 1.5648854961832062,
      "grad_norm": 0.3602054384066848,
      "kl": 0.031585693359375,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.4866071566939354,
      "reward_std": 0.17235196754336357,
      "rewards/accuracy_reward": 0.4866071566939354,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.9107360839844,
      "epoch": 1.5699745547073791,
      "grad_norm": 0.2773944269698253,
      "kl": 0.0299072265625,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.8147321790456772,
      "reward_std": 0.09125630464404821,
      "rewards/accuracy_reward": 0.8147321790456772,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.0178756713867,
      "epoch": 1.5750636132315523,
      "grad_norm": 0.4004195588759506,
      "kl": 0.031463623046875,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.4553571566939354,
      "reward_std": 0.1763141043484211,
      "rewards/accuracy_reward": 0.4553571566939354,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.7723388671875,
      "epoch": 1.5801526717557253,
      "grad_norm": 0.3571687177934301,
      "kl": 0.031951904296875,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.6205357313156128,
      "reward_std": 0.17585763707756996,
      "rewards/accuracy_reward": 0.6205357313156128,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.2812728881836,
      "epoch": 1.5852417302798982,
      "grad_norm": 0.3898645628909473,
      "kl": 0.0343017578125,
      "learning_rate": 2e-06,
      "loss": 0.0107,
      "reward": 0.455357164144516,
      "reward_std": 0.1789543218910694,
      "rewards/accuracy_reward": 0.455357164144516,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.9018020629883,
      "epoch": 1.5903307888040712,
      "grad_norm": 0.45925822888569257,
      "kl": 0.0361328125,
      "learning_rate": 2e-06,
      "loss": 0.0136,
      "reward": 0.5401785895228386,
      "reward_std": 0.27576345577836037,
      "rewards/accuracy_reward": 0.5401785895228386,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 376.39064025878906,
      "epoch": 1.5954198473282442,
      "grad_norm": 0.3952624210013506,
      "kl": 0.03875732421875,
      "learning_rate": 2e-06,
      "loss": -0.0172,
      "reward": 0.502232164144516,
      "reward_std": 0.128346499055624,
      "rewards/accuracy_reward": 0.502232164144516,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.2366256713867,
      "epoch": 1.6005089058524173,
      "grad_norm": 0.4302683829067842,
      "kl": 0.03704833984375,
      "learning_rate": 2e-06,
      "loss": 0.0154,
      "reward": 0.4687500149011612,
      "reward_std": 0.19160039350390434,
      "rewards/accuracy_reward": 0.4687500149011612,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.0759048461914,
      "epoch": 1.6055979643765903,
      "grad_norm": 0.3283633379052205,
      "kl": 0.034576416015625,
      "learning_rate": 2e-06,
      "loss": -0.0065,
      "reward": 0.6406250298023224,
      "reward_std": 0.1151136914268136,
      "rewards/accuracy_reward": 0.6406250298023224,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.1585006713867,
      "epoch": 1.6106870229007635,
      "grad_norm": 0.41211305415430904,
      "kl": 0.031707763671875,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.4330357313156128,
      "reward_std": 0.2064228653907776,
      "rewards/accuracy_reward": 0.4330357313156128,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.80804443359375,
      "epoch": 1.6157760814249365,
      "grad_norm": 0.4277863774804549,
      "kl": 0.034759521484375,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.562500037252903,
      "reward_std": 0.2106967382133007,
      "rewards/accuracy_reward": 0.562500037252903,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.3906478881836,
      "epoch": 1.6208651399491094,
      "grad_norm": 0.42432018902000207,
      "kl": 0.031585693359375,
      "learning_rate": 2e-06,
      "loss": -0.0022,
      "reward": 0.5781250298023224,
      "reward_std": 0.19926684722304344,
      "rewards/accuracy_reward": 0.5781250298023224,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.9263610839844,
      "epoch": 1.6259541984732824,
      "grad_norm": 0.41377970984118717,
      "kl": 0.03594970703125,
      "learning_rate": 2e-06,
      "loss": 0.0122,
      "reward": 0.620535746216774,
      "reward_std": 0.23870133608579636,
      "rewards/accuracy_reward": 0.620535746216774,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.1361770629883,
      "epoch": 1.6310432569974553,
      "grad_norm": 0.36151250654207095,
      "kl": 0.03594970703125,
      "learning_rate": 2e-06,
      "loss": 0.0074,
      "reward": 0.5669643133878708,
      "reward_std": 0.166221440769732,
      "rewards/accuracy_reward": 0.5669643133878708,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.2410888671875,
      "epoch": 1.6361323155216285,
      "grad_norm": 0.40267635839368887,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.4441964402794838,
      "reward_std": 0.17330226674675941,
      "rewards/accuracy_reward": 0.4441964402794838,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.63841247558594,
      "epoch": 1.6412213740458015,
      "grad_norm": 0.33240541791531786,
      "kl": 0.03179931640625,
      "learning_rate": 2e-06,
      "loss": 0.0109,
      "reward": 0.6049107387661934,
      "reward_std": 0.1530414316803217,
      "rewards/accuracy_reward": 0.6049107387661934,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.9665298461914,
      "epoch": 1.6463104325699747,
      "grad_norm": 0.3621088793133356,
      "kl": 0.0328369140625,
      "learning_rate": 2e-06,
      "loss": -0.0053,
      "reward": 0.6116071566939354,
      "reward_std": 0.1585872322320938,
      "rewards/accuracy_reward": 0.6116071566939354,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.8973388671875,
      "epoch": 1.6513994910941476,
      "grad_norm": 0.4354893144018653,
      "kl": 0.03662109375,
      "learning_rate": 2e-06,
      "loss": 0.0125,
      "reward": 0.5736607313156128,
      "reward_std": 0.2336120130494237,
      "rewards/accuracy_reward": 0.5736607313156128,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.6986846923828,
      "epoch": 1.6564885496183206,
      "grad_norm": 0.35881806451798626,
      "kl": 0.0347900390625,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.4575892984867096,
      "reward_std": 0.17096217907965183,
      "rewards/accuracy_reward": 0.4575892984867096,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.61163330078125,
      "epoch": 1.6615776081424936,
      "grad_norm": 0.4297659442878547,
      "kl": 0.0390625,
      "learning_rate": 2e-06,
      "loss": 0.0062,
      "reward": 0.6138393133878708,
      "reward_std": 0.22144446894526482,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.7924270629883,
      "epoch": 1.6666666666666665,
      "grad_norm": 0.34862166048080134,
      "kl": 0.033721923828125,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.4665178805589676,
      "reward_std": 0.14675204921513796,
      "rewards/accuracy_reward": 0.4665178805589676,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.4107360839844,
      "epoch": 1.6717557251908397,
      "grad_norm": 0.4076142663981743,
      "kl": 0.03118896484375,
      "learning_rate": 2e-06,
      "loss": 0.0119,
      "reward": 0.6339285969734192,
      "reward_std": 0.22718551196157932,
      "rewards/accuracy_reward": 0.6339285969734192,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.5826110839844,
      "epoch": 1.6768447837150129,
      "grad_norm": 0.35303215684775435,
      "kl": 0.03253173828125,
      "learning_rate": 2e-06,
      "loss": -0.0017,
      "reward": 0.620535746216774,
      "reward_std": 0.1566992960870266,
      "rewards/accuracy_reward": 0.620535746216774,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.8147506713867,
      "epoch": 1.6819338422391859,
      "grad_norm": 0.3803328513367462,
      "kl": 0.034515380859375,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.604910746216774,
      "reward_std": 0.19881199300289154,
      "rewards/accuracy_reward": 0.604910746216774,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.32591247558594,
      "epoch": 1.6870229007633588,
      "grad_norm": 0.4755914311292853,
      "kl": 0.034027099609375,
      "learning_rate": 2e-06,
      "loss": -0.007,
      "reward": 0.6205357387661934,
      "reward_std": 0.24791720509529114,
      "rewards/accuracy_reward": 0.6205357387661934,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.4241256713867,
      "epoch": 1.6921119592875318,
      "grad_norm": 0.4104178979199164,
      "kl": 0.03424072265625,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.6718750298023224,
      "reward_std": 0.17258745804429054,
      "rewards/accuracy_reward": 0.6718750298023224,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.7500228881836,
      "epoch": 1.6972010178117047,
      "grad_norm": 0.39424631438568525,
      "kl": 0.03424072265625,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.4709821604192257,
      "reward_std": 0.1996939592063427,
      "rewards/accuracy_reward": 0.4709821604192257,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.5022430419922,
      "epoch": 1.7022900763358777,
      "grad_norm": 0.3814444494372407,
      "kl": 0.03125,
      "learning_rate": 2e-06,
      "loss": 0.0105,
      "reward": 0.5959821492433548,
      "reward_std": 0.16038869880139828,
      "rewards/accuracy_reward": 0.5959821492433548,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.14733123779297,
      "epoch": 1.7073791348600509,
      "grad_norm": 0.41359950433869397,
      "kl": 0.032470703125,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.6138393133878708,
      "reward_std": 0.1900954283773899,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.3660888671875,
      "epoch": 1.712468193384224,
      "grad_norm": 0.40692091109165157,
      "kl": 0.033294677734375,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.6004464700818062,
      "reward_std": 0.16432827152311802,
      "rewards/accuracy_reward": 0.6004464700818062,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.2522506713867,
      "epoch": 1.717557251908397,
      "grad_norm": 0.4135208048065927,
      "kl": 0.03399658203125,
      "learning_rate": 2e-06,
      "loss": 0.0152,
      "reward": 0.5245535895228386,
      "reward_std": 0.18983528763055801,
      "rewards/accuracy_reward": 0.5245535895228386,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.0692138671875,
      "epoch": 1.72264631043257,
      "grad_norm": 0.29376874707478684,
      "kl": 0.03375244140625,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.5669643133878708,
      "reward_std": 0.12192143499851227,
      "rewards/accuracy_reward": 0.5669643133878708,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.49778747558594,
      "epoch": 1.727735368956743,
      "grad_norm": 0.3968883357490178,
      "kl": 0.032073974609375,
      "learning_rate": 2e-06,
      "loss": -0.0122,
      "reward": 0.4910714626312256,
      "reward_std": 0.1667480319738388,
      "rewards/accuracy_reward": 0.4910714626312256,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.55359649658203,
      "epoch": 1.732824427480916,
      "grad_norm": 0.4601899434981398,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.600446455180645,
      "reward_std": 0.23292654752731323,
      "rewards/accuracy_reward": 0.600446455180645,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 332.2879638671875,
      "epoch": 1.737913486005089,
      "grad_norm": 0.5129007500319527,
      "kl": 0.04302978515625,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.5267857387661934,
      "reward_std": 0.21882708743214607,
      "rewards/accuracy_reward": 0.5267857387661934,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.5268096923828,
      "epoch": 1.743002544529262,
      "grad_norm": 0.3902613328094504,
      "kl": 0.034698486328125,
      "learning_rate": 2e-06,
      "loss": -0.0071,
      "reward": 0.5223214477300644,
      "reward_std": 0.1862612385302782,
      "rewards/accuracy_reward": 0.5223214477300644,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.68082427978516,
      "epoch": 1.7480916030534353,
      "grad_norm": 0.43948939716465657,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.573660746216774,
      "reward_std": 0.21117087453603745,
      "rewards/accuracy_reward": 0.573660746216774,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.7165298461914,
      "epoch": 1.7531806615776082,
      "grad_norm": 0.4362451793291335,
      "kl": 0.033477783203125,
      "learning_rate": 2e-06,
      "loss": 0.0086,
      "reward": 0.5357143133878708,
      "reward_std": 0.19573705457150936,
      "rewards/accuracy_reward": 0.5357143133878708,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.72769927978516,
      "epoch": 1.7582697201017812,
      "grad_norm": 0.41031786835866074,
      "kl": 0.03466796875,
      "learning_rate": 2e-06,
      "loss": -0.0072,
      "reward": 0.5178571566939354,
      "reward_std": 0.14876011572778225,
      "rewards/accuracy_reward": 0.5178571566939354,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 365.25447845458984,
      "epoch": 1.7633587786259541,
      "grad_norm": 0.46595037952621376,
      "kl": 0.0374755859375,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.5781250223517418,
      "reward_std": 0.18289937265217304,
      "rewards/accuracy_reward": 0.5781250223517418,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.93082427978516,
      "epoch": 1.768447837150127,
      "grad_norm": 0.45774566704542136,
      "kl": 0.039154052734375,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.6049107536673546,
      "reward_std": 0.18452896922826767,
      "rewards/accuracy_reward": 0.6049107536673546,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 367.59822845458984,
      "epoch": 1.7735368956743003,
      "grad_norm": 2.591658172875442,
      "kl": 0.056793212890625,
      "learning_rate": 2e-06,
      "loss": 0.0083,
      "reward": 0.506696455180645,
      "reward_std": 0.15466941893100739,
      "rewards/accuracy_reward": 0.506696455180645,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.86385345458984,
      "epoch": 1.7786259541984732,
      "grad_norm": 0.37643096212783456,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.7187500447034836,
      "reward_std": 0.14254576992243528,
      "rewards/accuracy_reward": 0.7187500447034836,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.5736770629883,
      "epoch": 1.7837150127226464,
      "grad_norm": 0.3720669780083704,
      "kl": 0.032257080078125,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.6160714626312256,
      "reward_std": 0.15607618167996407,
      "rewards/accuracy_reward": 0.6160714626312256,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.3705520629883,
      "epoch": 1.7888040712468194,
      "grad_norm": 0.36898135098898155,
      "kl": 0.0330810546875,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.5424107313156128,
      "reward_std": 0.1427915208041668,
      "rewards/accuracy_reward": 0.5424107313156128,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.2120666503906,
      "epoch": 1.7938931297709924,
      "grad_norm": 0.34968780393525445,
      "kl": 0.03704833984375,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.5000000186264515,
      "reward_std": 0.11996633186936378,
      "rewards/accuracy_reward": 0.5000000186264515,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.6160888671875,
      "epoch": 1.7989821882951653,
      "grad_norm": 2599.9910718457472,
      "kl": 13.27606201171875,
      "learning_rate": 2e-06,
      "loss": 0.3824,
      "reward": 0.5022321715950966,
      "reward_std": 0.18969968520104885,
      "rewards/accuracy_reward": 0.5022321715950966,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.83260345458984,
      "epoch": 1.8040712468193383,
      "grad_norm": 0.34414783813611044,
      "kl": 0.039794921875,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6183035969734192,
      "reward_std": 0.16032780893146992,
      "rewards/accuracy_reward": 0.6183035969734192,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.9084930419922,
      "epoch": 1.8091603053435115,
      "grad_norm": 0.3857649031928157,
      "kl": 0.039215087890625,
      "learning_rate": 2e-06,
      "loss": -0.0128,
      "reward": 0.5937500149011612,
      "reward_std": 0.14901735074818134,
      "rewards/accuracy_reward": 0.5937500149011612,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.50672149658203,
      "epoch": 1.8142493638676844,
      "grad_norm": 0.349036494255612,
      "kl": 0.033050537109375,
      "learning_rate": 2e-06,
      "loss": -0.0025,
      "reward": 0.580357164144516,
      "reward_std": 0.1297006607055664,
      "rewards/accuracy_reward": 0.580357164144516,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.9174346923828,
      "epoch": 1.8193384223918576,
      "grad_norm": 0.27196929473992737,
      "kl": 0.034759521484375,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.5736607387661934,
      "reward_std": 0.08670743089169264,
      "rewards/accuracy_reward": 0.5736607387661934,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.54466247558594,
      "epoch": 1.8244274809160306,
      "grad_norm": 0.35237072076561626,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": -0.009,
      "reward": 0.6785714626312256,
      "reward_std": 0.11349314637482166,
      "rewards/accuracy_reward": 0.6785714626312256,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.69197845458984,
      "epoch": 1.8295165394402035,
      "grad_norm": 0.31748531000848407,
      "kl": 0.0367431640625,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.5424107387661934,
      "reward_std": 0.10300773568451405,
      "rewards/accuracy_reward": 0.5424107387661934,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.5401916503906,
      "epoch": 1.8346055979643765,
      "grad_norm": 0.3810081949243458,
      "kl": 0.0352783203125,
      "learning_rate": 2e-06,
      "loss": -0.0056,
      "reward": 0.81026791036129,
      "reward_std": 0.1514055710285902,
      "rewards/accuracy_reward": 0.81026791036129,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.8549270629883,
      "epoch": 1.8396946564885495,
      "grad_norm": 0.2877504412241006,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.424107164144516,
      "reward_std": 0.09726398717612028,
      "rewards/accuracy_reward": 0.424107164144516,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.46207427978516,
      "epoch": 1.8447837150127226,
      "grad_norm": 0.3646209313520994,
      "kl": 0.04241943359375,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.6093750074505806,
      "reward_std": 0.1521219816058874,
      "rewards/accuracy_reward": 0.6093750074505806,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.89734649658203,
      "epoch": 1.8498727735368958,
      "grad_norm": 0.4243928780122091,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": 0.0187,
      "reward": 0.4665178880095482,
      "reward_std": 0.20290330052375793,
      "rewards/accuracy_reward": 0.4665178880095482,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.5446548461914,
      "epoch": 1.8549618320610688,
      "grad_norm": 0.31994331064348525,
      "kl": 0.0322265625,
      "learning_rate": 2e-06,
      "loss": -0.0074,
      "reward": 0.6450893133878708,
      "reward_std": 0.12879652343690395,
      "rewards/accuracy_reward": 0.6450893133878708,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.03797149658203,
      "epoch": 1.8600508905852418,
      "grad_norm": 0.40152860734111306,
      "kl": 0.037322998046875,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.6004464775323868,
      "reward_std": 0.13713151961565018,
      "rewards/accuracy_reward": 0.6004464775323868,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.9241180419922,
      "epoch": 1.8651399491094147,
      "grad_norm": 0.2987453957740081,
      "kl": 0.029693603515625,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.589285746216774,
      "reward_std": 0.12920781783759594,
      "rewards/accuracy_reward": 0.589285746216774,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.2991256713867,
      "epoch": 1.8702290076335877,
      "grad_norm": 0.554465600930555,
      "kl": 0.0467529296875,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.6004464477300644,
      "reward_std": 0.2248956747353077,
      "rewards/accuracy_reward": 0.6004464477300644,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.3013610839844,
      "epoch": 1.8753180661577609,
      "grad_norm": 0.31516645881936106,
      "kl": 0.035675048828125,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.6071428954601288,
      "reward_std": 0.11709805019199848,
      "rewards/accuracy_reward": 0.6071428954601288,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.3504638671875,
      "epoch": 1.8804071246819338,
      "grad_norm": 0.29945925934244355,
      "kl": 0.043212890625,
      "learning_rate": 2e-06,
      "loss": -0.0081,
      "reward": 0.6138393208384514,
      "reward_std": 0.1219743937253952,
      "rewards/accuracy_reward": 0.6138393208384514,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.60717010498047,
      "epoch": 1.885496183206107,
      "grad_norm": 0.400533724930988,
      "kl": 0.05548095703125,
      "learning_rate": 2e-06,
      "loss": 0.0093,
      "reward": 0.5736607387661934,
      "reward_std": 0.15631252340972424,
      "rewards/accuracy_reward": 0.5736607387661934,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.2455520629883,
      "epoch": 1.89058524173028,
      "grad_norm": 0.3670173912293237,
      "kl": 0.0465087890625,
      "learning_rate": 2e-06,
      "loss": -0.0021,
      "reward": 0.6183035895228386,
      "reward_std": 0.13964036107063293,
      "rewards/accuracy_reward": 0.6183035895228386,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.30359649658203,
      "epoch": 1.895674300254453,
      "grad_norm": 0.4588507404224872,
      "kl": 0.04742431640625,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.502232164144516,
      "reward_std": 0.1481290515512228,
      "rewards/accuracy_reward": 0.502232164144516,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.11385345458984,
      "epoch": 1.900763358778626,
      "grad_norm": 39.86349118531167,
      "kl": 0.4259033203125,
      "learning_rate": 2e-06,
      "loss": 0.0122,
      "reward": 0.6093750149011612,
      "reward_std": 0.16022942773997784,
      "rewards/accuracy_reward": 0.6093750149011612,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.35939025878906,
      "epoch": 1.9058524173027989,
      "grad_norm": 0.33153842654680793,
      "kl": 0.0458984375,
      "learning_rate": 2e-06,
      "loss": -0.0035,
      "reward": 0.627232164144516,
      "reward_std": 0.14701843727380037,
      "rewards/accuracy_reward": 0.627232164144516,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.33707427978516,
      "epoch": 1.910941475826972,
      "grad_norm": 0.33889936958064354,
      "kl": 0.044921875,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.5245535969734192,
      "reward_std": 0.14769601821899414,
      "rewards/accuracy_reward": 0.5245535969734192,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.78350830078125,
      "epoch": 1.916030534351145,
      "grad_norm": 0.4102871349992259,
      "kl": 0.04376220703125,
      "learning_rate": 2e-06,
      "loss": 0.0196,
      "reward": 0.3928571566939354,
      "reward_std": 0.21786130033433437,
      "rewards/accuracy_reward": 0.3928571566939354,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.90403747558594,
      "epoch": 1.9211195928753182,
      "grad_norm": 0.33711468718878645,
      "kl": 0.044708251953125,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.4464285969734192,
      "reward_std": 0.14733134396374226,
      "rewards/accuracy_reward": 0.4464285969734192,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.5491256713867,
      "epoch": 1.9262086513994912,
      "grad_norm": 0.35638819766037244,
      "kl": 0.04266357421875,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.5156250223517418,
      "reward_std": 0.14308641105890274,
      "rewards/accuracy_reward": 0.5156250223517418,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.14064025878906,
      "epoch": 1.9312977099236641,
      "grad_norm": 0.35502262918683586,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": 0.0089,
      "reward": 0.5758928880095482,
      "reward_std": 0.09469473734498024,
      "rewards/accuracy_reward": 0.5758928880095482,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 455.4799270629883,
      "epoch": 1.936386768447837,
      "grad_norm": 0.330973980295281,
      "kl": 0.03887939453125,
      "learning_rate": 2e-06,
      "loss": 0.0061,
      "reward": 0.5691964477300644,
      "reward_std": 0.15365827456116676,
      "rewards/accuracy_reward": 0.5691964477300644,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.4643020629883,
      "epoch": 1.94147582697201,
      "grad_norm": 0.43447773535161155,
      "kl": 0.039306640625,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.537946455180645,
      "reward_std": 0.20215855166316032,
      "rewards/accuracy_reward": 0.537946455180645,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.75001525878906,
      "epoch": 1.9465648854961832,
      "grad_norm": 0.3709496721140309,
      "kl": 0.04852294921875,
      "learning_rate": 2e-06,
      "loss": -0.0046,
      "reward": 0.647321455180645,
      "reward_std": 0.19486423209309578,
      "rewards/accuracy_reward": 0.647321455180645,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.32144927978516,
      "epoch": 1.9516539440203562,
      "grad_norm": 0.33983340450443467,
      "kl": 0.04833984375,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.642857164144516,
      "reward_std": 0.13795265182852745,
      "rewards/accuracy_reward": 0.642857164144516,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.7812728881836,
      "epoch": 1.9567430025445294,
      "grad_norm": 0.3542287261755593,
      "kl": 0.0501708984375,
      "learning_rate": 2e-06,
      "loss": 0.016,
      "reward": 0.6026786044239998,
      "reward_std": 0.130963871255517,
      "rewards/accuracy_reward": 0.6026786044239998,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.20314025878906,
      "epoch": 1.9618320610687023,
      "grad_norm": 0.4141484086187222,
      "kl": 0.0703125,
      "learning_rate": 2e-06,
      "loss": 0.0165,
      "reward": 0.5111607387661934,
      "reward_std": 0.17906580865383148,
      "rewards/accuracy_reward": 0.5111607387661934,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.6138610839844,
      "epoch": 1.9669211195928753,
      "grad_norm": 0.5824860644956747,
      "kl": 0.067626953125,
      "learning_rate": 2e-06,
      "loss": 0.0072,
      "reward": 0.4977678805589676,
      "reward_std": 0.18197993654757738,
      "rewards/accuracy_reward": 0.4977678805589676,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.7143020629883,
      "epoch": 1.9720101781170483,
      "grad_norm": 0.4942465240724718,
      "kl": 0.062255859375,
      "learning_rate": 2e-06,
      "loss": -0.0053,
      "reward": 0.4575893133878708,
      "reward_std": 0.13631832972168922,
      "rewards/accuracy_reward": 0.4575893133878708,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.1785888671875,
      "epoch": 1.9770992366412212,
      "grad_norm": 0.3869605632287253,
      "kl": 0.04345703125,
      "learning_rate": 2e-06,
      "loss": 0.0074,
      "reward": 0.5825893208384514,
      "reward_std": 0.16959000378847122,
      "rewards/accuracy_reward": 0.5825893208384514,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.16072845458984,
      "epoch": 1.9821882951653944,
      "grad_norm": 0.37786287346057257,
      "kl": 0.03607177734375,
      "learning_rate": 2e-06,
      "loss": 0.0031,
      "reward": 0.595982164144516,
      "reward_std": 0.14545245841145515,
      "rewards/accuracy_reward": 0.595982164144516,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.7143096923828,
      "epoch": 1.9872773536895676,
      "grad_norm": 0.39575592064903486,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": 0.0182,
      "reward": 0.6718750149011612,
      "reward_std": 0.18826821073889732,
      "rewards/accuracy_reward": 0.6718750149011612,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.8772430419922,
      "epoch": 1.9923664122137406,
      "grad_norm": 0.4651636579207121,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.0161,
      "reward": 0.5625000298023224,
      "reward_std": 0.24423056095838547,
      "rewards/accuracy_reward": 0.5625000298023224,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.13463592529297,
      "epoch": 1.9974554707379135,
      "grad_norm": 0.44450093547546876,
      "kl": 0.036407470703125,
      "learning_rate": 2e-06,
      "loss": -0.0022,
      "reward": 0.6250000298023224,
      "reward_std": 0.2150175478309393,
      "rewards/accuracy_reward": 0.6250000298023224,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.3460006713867,
      "epoch": 2.005089058524173,
      "grad_norm": 0.2742009983878323,
      "kl": 0.041015625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.754464328289032,
      "reward_std": 0.1010371521115303,
      "rewards/accuracy_reward": 0.754464328289032,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.9598388671875,
      "epoch": 2.010178117048346,
      "grad_norm": 0.3472987389060835,
      "kl": 0.037567138671875,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.5111607313156128,
      "reward_std": 0.10763046145439148,
      "rewards/accuracy_reward": 0.5111607313156128,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.2410888671875,
      "epoch": 2.015267175572519,
      "grad_norm": 0.39088139408455136,
      "kl": 0.0390625,
      "learning_rate": 2e-06,
      "loss": 0.0172,
      "reward": 0.6138393133878708,
      "reward_std": 0.14116082713007927,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.7745666503906,
      "epoch": 2.0203562340966923,
      "grad_norm": 0.34232536414655745,
      "kl": 0.035797119140625,
      "learning_rate": 2e-06,
      "loss": -0.0124,
      "reward": 0.8482142984867096,
      "reward_std": 0.12868122477084398,
      "rewards/accuracy_reward": 0.8482142984867096,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 357.08484649658203,
      "epoch": 2.0254452926208653,
      "grad_norm": 0.399584855628817,
      "kl": 0.039398193359375,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.6941964477300644,
      "reward_std": 0.14294251147657633,
      "rewards/accuracy_reward": 0.6941964477300644,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.6897506713867,
      "epoch": 2.030534351145038,
      "grad_norm": 130.25491711312384,
      "kl": 4.02850341796875,
      "learning_rate": 2e-06,
      "loss": 0.1802,
      "reward": 0.7656250447034836,
      "reward_std": 0.11223897151648998,
      "rewards/accuracy_reward": 0.7656250447034836,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.0736846923828,
      "epoch": 2.035623409669211,
      "grad_norm": 0.46588574160831064,
      "kl": 0.0416259765625,
      "learning_rate": 2e-06,
      "loss": -0.01,
      "reward": 0.7008928805589676,
      "reward_std": 0.20006506331264973,
      "rewards/accuracy_reward": 0.7008928805589676,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.6339416503906,
      "epoch": 2.040712468193384,
      "grad_norm": 0.373123724241272,
      "kl": 0.0482177734375,
      "learning_rate": 2e-06,
      "loss": 0.0058,
      "reward": 0.7321428954601288,
      "reward_std": 0.13184583187103271,
      "rewards/accuracy_reward": 0.7321428954601288,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.9107360839844,
      "epoch": 2.045801526717557,
      "grad_norm": 0.31806823628981257,
      "kl": 0.04083251953125,
      "learning_rate": 2e-06,
      "loss": -0.0053,
      "reward": 0.7767857313156128,
      "reward_std": 0.09784962050616741,
      "rewards/accuracy_reward": 0.7767857313156128,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.2254638671875,
      "epoch": 2.05089058524173,
      "grad_norm": 0.42290855068697064,
      "kl": 0.0394287109375,
      "learning_rate": 2e-06,
      "loss": 0.0061,
      "reward": 0.6562500298023224,
      "reward_std": 0.19378305226564407,
      "rewards/accuracy_reward": 0.6562500298023224,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 426.7433319091797,
      "epoch": 2.0559796437659035,
      "grad_norm": 0.2605804750219208,
      "kl": 0.037017822265625,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.8035714626312256,
      "reward_std": 0.06577018182724714,
      "rewards/accuracy_reward": 0.8035714626312256,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.4776916503906,
      "epoch": 2.0610687022900764,
      "grad_norm": 0.280147550531787,
      "kl": 0.035491943359375,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.7455357611179352,
      "reward_std": 0.07104842457920313,
      "rewards/accuracy_reward": 0.7455357611179352,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.09822845458984,
      "epoch": 2.0661577608142494,
      "grad_norm": 0.3281689744499087,
      "kl": 0.03564453125,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.7611607611179352,
      "reward_std": 0.1360610779374838,
      "rewards/accuracy_reward": 0.7611607611179352,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.3593978881836,
      "epoch": 2.0712468193384224,
      "grad_norm": 0.31823988820677146,
      "kl": 0.0439453125,
      "learning_rate": 2e-06,
      "loss": -0.0,
      "reward": 0.7633928954601288,
      "reward_std": 0.12937422282993793,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.81251525878906,
      "epoch": 2.0763358778625953,
      "grad_norm": 0.3993553492680455,
      "kl": 0.04681396484375,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.7232143133878708,
      "reward_std": 0.09746193699538708,
      "rewards/accuracy_reward": 0.7232143133878708,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.1026916503906,
      "epoch": 2.0814249363867683,
      "grad_norm": 0.26287455186122644,
      "kl": 0.0369873046875,
      "learning_rate": 2e-06,
      "loss": 0.0089,
      "reward": 0.6250000298023224,
      "reward_std": 0.08987135346978903,
      "rewards/accuracy_reward": 0.6250000298023224,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.8459930419922,
      "epoch": 2.0865139949109412,
      "grad_norm": 0.3305582164075515,
      "kl": 0.04248046875,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.6964285969734192,
      "reward_std": 0.09424471203237772,
      "rewards/accuracy_reward": 0.6964285969734192,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.44644927978516,
      "epoch": 2.0916030534351147,
      "grad_norm": 0.3452528519941532,
      "kl": 0.0430908203125,
      "learning_rate": 2e-06,
      "loss": 0.0109,
      "reward": 0.7633928954601288,
      "reward_std": 0.13682775013148785,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.33707427978516,
      "epoch": 2.0966921119592876,
      "grad_norm": 0.3699552318627565,
      "kl": 0.0400390625,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.7433035969734192,
      "reward_std": 0.10905144643038511,
      "rewards/accuracy_reward": 0.7433035969734192,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.36832427978516,
      "epoch": 2.1017811704834606,
      "grad_norm": 0.44646836528989053,
      "kl": 0.04669189453125,
      "learning_rate": 2e-06,
      "loss": 0.0074,
      "reward": 0.6473214626312256,
      "reward_std": 0.16127971559762955,
      "rewards/accuracy_reward": 0.6473214626312256,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.11385345458984,
      "epoch": 2.1068702290076335,
      "grad_norm": 0.2947258485589255,
      "kl": 0.03851318359375,
      "learning_rate": 2e-06,
      "loss": -0.0006,
      "reward": 0.7008928954601288,
      "reward_std": 0.10946544446051121,
      "rewards/accuracy_reward": 0.7008928954601288,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.6986770629883,
      "epoch": 2.1119592875318065,
      "grad_norm": 0.3735043766143211,
      "kl": 0.0386962890625,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.7656250298023224,
      "reward_std": 0.13605837430804968,
      "rewards/accuracy_reward": 0.7656250298023224,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.44644927978516,
      "epoch": 2.1170483460559795,
      "grad_norm": 0.33044634292850333,
      "kl": 0.03399658203125,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.6941964626312256,
      "reward_std": 0.1353988889604807,
      "rewards/accuracy_reward": 0.6941964626312256,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.8504638671875,
      "epoch": 2.122137404580153,
      "grad_norm": 0.3482109526429795,
      "kl": 0.038238525390625,
      "learning_rate": 2e-06,
      "loss": 0.0099,
      "reward": 0.6651786118745804,
      "reward_std": 0.09537875186651945,
      "rewards/accuracy_reward": 0.6651786118745804,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.33707427978516,
      "epoch": 2.127226463104326,
      "grad_norm": 0.23540557666160633,
      "kl": 0.035552978515625,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.6473214477300644,
      "reward_std": 0.06424972135573626,
      "rewards/accuracy_reward": 0.6473214477300644,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.34153747558594,
      "epoch": 2.132315521628499,
      "grad_norm": 0.4010454738516299,
      "kl": 0.0411376953125,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.6049107313156128,
      "reward_std": 0.15023404452949762,
      "rewards/accuracy_reward": 0.6049107313156128,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.90403747558594,
      "epoch": 2.1374045801526718,
      "grad_norm": 0.3806691644319369,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": -0.0049,
      "reward": 0.6383928954601288,
      "reward_std": 0.10896879248321056,
      "rewards/accuracy_reward": 0.6383928954601288,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.0044860839844,
      "epoch": 2.1424936386768447,
      "grad_norm": 0.3365895191854068,
      "kl": 0.0411376953125,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.6785714700818062,
      "reward_std": 0.10474831517785788,
      "rewards/accuracy_reward": 0.6785714700818062,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.0714416503906,
      "epoch": 2.1475826972010177,
      "grad_norm": 0.3699404951725154,
      "kl": 0.037261962890625,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.6406250149011612,
      "reward_std": 0.1640634946525097,
      "rewards/accuracy_reward": 0.6406250149011612,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 361.0401916503906,
      "epoch": 2.1526717557251906,
      "grad_norm": 0.3745712794228524,
      "kl": 0.04376220703125,
      "learning_rate": 2e-06,
      "loss": 0.0056,
      "reward": 0.7477678954601288,
      "reward_std": 0.14180846884846687,
      "rewards/accuracy_reward": 0.7477678954601288,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.10716247558594,
      "epoch": 2.157760814249364,
      "grad_norm": 0.23307581680227343,
      "kl": 0.03656005859375,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.745535746216774,
      "reward_std": 0.06262173783034086,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.1875228881836,
      "epoch": 2.162849872773537,
      "grad_norm": 0.38264157640860647,
      "kl": 0.04461669921875,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.6383928805589676,
      "reward_std": 0.15253328159451485,
      "rewards/accuracy_reward": 0.6383928805589676,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.47322845458984,
      "epoch": 2.16793893129771,
      "grad_norm": 0.30949443469371074,
      "kl": 0.034393310546875,
      "learning_rate": 2e-06,
      "loss": 0.0079,
      "reward": 0.6406250260770321,
      "reward_std": 0.10763046704232693,
      "rewards/accuracy_reward": 0.6406250260770321,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.3437728881836,
      "epoch": 2.173027989821883,
      "grad_norm": 0.34019617897471943,
      "kl": 0.0382080078125,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.5825893059372902,
      "reward_std": 0.10456457268446684,
      "rewards/accuracy_reward": 0.5825893059372902,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.52010345458984,
      "epoch": 2.178117048346056,
      "grad_norm": 0.252142055366362,
      "kl": 0.0340576171875,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.6808036118745804,
      "reward_std": 0.07311739027500153,
      "rewards/accuracy_reward": 0.6808036118745804,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.3526916503906,
      "epoch": 2.183206106870229,
      "grad_norm": 0.2591769614280603,
      "kl": 0.0418701171875,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.761160746216774,
      "reward_std": 0.09067067224532366,
      "rewards/accuracy_reward": 0.761160746216774,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.3326110839844,
      "epoch": 2.188295165394402,
      "grad_norm": 0.3599011585237677,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.6785714477300644,
      "reward_std": 0.13360299356281757,
      "rewards/accuracy_reward": 0.6785714477300644,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 368.3281478881836,
      "epoch": 2.1933842239185752,
      "grad_norm": 0.3769927627686171,
      "kl": 0.04144287109375,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.5781250298023224,
      "reward_std": 0.09783685021102428,
      "rewards/accuracy_reward": 0.5781250298023224,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.1473388671875,
      "epoch": 2.198473282442748,
      "grad_norm": 0.2943189171062689,
      "kl": 0.035247802734375,
      "learning_rate": 2e-06,
      "loss": 0.0046,
      "reward": 0.6540178880095482,
      "reward_std": 0.10128805786371231,
      "rewards/accuracy_reward": 0.6540178880095482,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.0513610839844,
      "epoch": 2.203562340966921,
      "grad_norm": 0.34144583657086436,
      "kl": 0.03643798828125,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.7767857313156128,
      "reward_std": 0.12260545510798693,
      "rewards/accuracy_reward": 0.7767857313156128,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.0870819091797,
      "epoch": 2.208651399491094,
      "grad_norm": 0.3126396945263014,
      "kl": 0.0384521484375,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.6138393133878708,
      "reward_std": 0.0747014582157135,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.0446548461914,
      "epoch": 2.213740458015267,
      "grad_norm": 0.3709590910879604,
      "kl": 0.03729248046875,
      "learning_rate": 2e-06,
      "loss": 0.0125,
      "reward": 0.8102678954601288,
      "reward_std": 0.16197794489562511,
      "rewards/accuracy_reward": 0.8102678954601288,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.2076110839844,
      "epoch": 2.21882951653944,
      "grad_norm": 0.28491003800984027,
      "kl": 0.0362548828125,
      "learning_rate": 2e-06,
      "loss": 0.0095,
      "reward": 0.6071428805589676,
      "reward_std": 0.08733848761767149,
      "rewards/accuracy_reward": 0.6071428805589676,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.9710006713867,
      "epoch": 2.223918575063613,
      "grad_norm": 0.2349347321988743,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.689732164144516,
      "reward_std": 0.06589301396161318,
      "rewards/accuracy_reward": 0.689732164144516,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.72769927978516,
      "epoch": 2.2290076335877864,
      "grad_norm": 0.29500242270003374,
      "kl": 0.03863525390625,
      "learning_rate": 2e-06,
      "loss": 0.0115,
      "reward": 0.6294643133878708,
      "reward_std": 0.07482158578932285,
      "rewards/accuracy_reward": 0.6294643133878708,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.10716247558594,
      "epoch": 2.2340966921119594,
      "grad_norm": 0.26301039338953314,
      "kl": 0.039031982421875,
      "learning_rate": 2e-06,
      "loss": -0.0066,
      "reward": 0.7611607611179352,
      "reward_std": 0.07249427866190672,
      "rewards/accuracy_reward": 0.7611607611179352,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.20538330078125,
      "epoch": 2.2391857506361323,
      "grad_norm": 0.3019924103632674,
      "kl": 0.038818359375,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.6808035969734192,
      "reward_std": 0.09883901290595531,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.13172149658203,
      "epoch": 2.2442748091603053,
      "grad_norm": 0.17822494728690552,
      "kl": 0.04278564453125,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.6986607611179352,
      "reward_std": 0.0267857164144516,
      "rewards/accuracy_reward": 0.6986607611179352,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.02233123779297,
      "epoch": 2.2493638676844783,
      "grad_norm": 0.21469670399697338,
      "kl": 0.036468505859375,
      "learning_rate": 2e-06,
      "loss": -0.0006,
      "reward": 0.7254464626312256,
      "reward_std": 0.03772235009819269,
      "rewards/accuracy_reward": 0.7254464626312256,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.6116180419922,
      "epoch": 2.2544529262086512,
      "grad_norm": 0.16764568530225557,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": 0.0082,
      "reward": 0.6250000149011612,
      "reward_std": 0.04935094714164734,
      "rewards/accuracy_reward": 0.6250000149011612,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.14288330078125,
      "epoch": 2.2595419847328246,
      "grad_norm": 0.21281068247363297,
      "kl": 0.036956787109375,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.7098214626312256,
      "reward_std": 0.05728259403258562,
      "rewards/accuracy_reward": 0.7098214626312256,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.5178756713867,
      "epoch": 2.2646310432569976,
      "grad_norm": 0.9965814898124155,
      "kl": 0.0435791015625,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.5111607350409031,
      "reward_std": 0.0895366258919239,
      "rewards/accuracy_reward": 0.5111607350409031,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.6986770629883,
      "epoch": 2.2697201017811706,
      "grad_norm": 0.3309671635916835,
      "kl": 0.0333251953125,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.7209821939468384,
      "reward_std": 0.10700734984129667,
      "rewards/accuracy_reward": 0.7209821939468384,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.07144927978516,
      "epoch": 2.2748091603053435,
      "grad_norm": 0.4272193039446874,
      "kl": 0.04412841796875,
      "learning_rate": 2e-06,
      "loss": 0.017,
      "reward": 0.7834821790456772,
      "reward_std": 0.08104903623461723,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.18082427978516,
      "epoch": 2.2798982188295165,
      "grad_norm": 0.3000200900552537,
      "kl": 0.037628173828125,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.6294643208384514,
      "reward_std": 0.07771168742328882,
      "rewards/accuracy_reward": 0.6294643208384514,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.3526916503906,
      "epoch": 2.2849872773536894,
      "grad_norm": 1.937837118819002,
      "kl": 0.0416259765625,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.5558036044239998,
      "reward_std": 0.1322879195213318,
      "rewards/accuracy_reward": 0.5558036044239998,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.02903747558594,
      "epoch": 2.2900763358778624,
      "grad_norm": 0.7869042879630593,
      "kl": 0.0377197265625,
      "learning_rate": 2e-06,
      "loss": -0.0101,
      "reward": 0.7723214626312256,
      "reward_std": 0.08922641910612583,
      "rewards/accuracy_reward": 0.7723214626312256,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.6027069091797,
      "epoch": 2.2951653944020354,
      "grad_norm": 0.2914751051443095,
      "kl": 0.03363037109375,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.7678571790456772,
      "reward_std": 0.10282399691641331,
      "rewards/accuracy_reward": 0.7678571790456772,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.5424270629883,
      "epoch": 2.300254452926209,
      "grad_norm": 0.35247194342747756,
      "kl": 0.03515625,
      "learning_rate": 2e-06,
      "loss": -0.0068,
      "reward": 0.689732164144516,
      "reward_std": 0.1558098942041397,
      "rewards/accuracy_reward": 0.689732164144516,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.58260345458984,
      "epoch": 2.3053435114503817,
      "grad_norm": 0.31912449668655113,
      "kl": 0.034912109375,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.7678571939468384,
      "reward_std": 0.12644578889012337,
      "rewards/accuracy_reward": 0.7678571939468384,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.71653747558594,
      "epoch": 2.3104325699745547,
      "grad_norm": 0.6029598774597014,
      "kl": 0.033966064453125,
      "learning_rate": 2e-06,
      "loss": -0.0036,
      "reward": 0.756696455180645,
      "reward_std": 0.12330368347465992,
      "rewards/accuracy_reward": 0.756696455180645,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.4620666503906,
      "epoch": 2.3155216284987277,
      "grad_norm": 0.3081410698912104,
      "kl": 0.03985595703125,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.683035746216774,
      "reward_std": 0.11600309796631336,
      "rewards/accuracy_reward": 0.683035746216774,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.7879638671875,
      "epoch": 2.3206106870229006,
      "grad_norm": 0.30355954965410276,
      "kl": 0.038818359375,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.7834821790456772,
      "reward_std": 0.11273291427642107,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.8995666503906,
      "epoch": 2.325699745547074,
      "grad_norm": 0.2773728066737305,
      "kl": 0.03631591796875,
      "learning_rate": 2e-06,
      "loss": -0.0061,
      "reward": 0.7745536118745804,
      "reward_std": 0.11543403938412666,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.0960006713867,
      "epoch": 2.330788804071247,
      "grad_norm": 0.2871216023789764,
      "kl": 0.032318115234375,
      "learning_rate": 2e-06,
      "loss": -0.0005,
      "reward": 0.6250000298023224,
      "reward_std": 0.09111165534704924,
      "rewards/accuracy_reward": 0.6250000298023224,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.71876525878906,
      "epoch": 2.33587786259542,
      "grad_norm": 0.34728748119133596,
      "kl": 0.033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.6808035969734192,
      "reward_std": 0.16387071087956429,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.1629638671875,
      "epoch": 2.340966921119593,
      "grad_norm": 0.2710432422591856,
      "kl": 0.03680419921875,
      "learning_rate": 2e-06,
      "loss": -0.0041,
      "reward": 0.6450893133878708,
      "reward_std": 0.08534066285938025,
      "rewards/accuracy_reward": 0.6450893133878708,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 453.3393020629883,
      "epoch": 2.346055979643766,
      "grad_norm": 0.2531748418736068,
      "kl": 0.03033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.6428571939468384,
      "reward_std": 0.06011178996413946,
      "rewards/accuracy_reward": 0.6428571939468384,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.7924346923828,
      "epoch": 2.351145038167939,
      "grad_norm": 0.29212336637874164,
      "kl": 0.033905029296875,
      "learning_rate": 2e-06,
      "loss": 0.0043,
      "reward": 0.7901786118745804,
      "reward_std": 0.06558010634034872,
      "rewards/accuracy_reward": 0.7901786118745804,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.77457427978516,
      "epoch": 2.356234096692112,
      "grad_norm": 0.2529114798782207,
      "kl": 0.031982421875,
      "learning_rate": 2e-06,
      "loss": 0.0119,
      "reward": 0.6607143059372902,
      "reward_std": 0.09752393420785666,
      "rewards/accuracy_reward": 0.6607143059372902,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.6183166503906,
      "epoch": 2.3613231552162848,
      "grad_norm": 0.28781993386561655,
      "kl": 0.036651611328125,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.7500000447034836,
      "reward_std": 0.0739385262131691,
      "rewards/accuracy_reward": 0.7500000447034836,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.93975830078125,
      "epoch": 2.366412213740458,
      "grad_norm": 0.25389227469639425,
      "kl": 0.0308837890625,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.6808035969734192,
      "reward_std": 0.05684161093086004,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.21653747558594,
      "epoch": 2.371501272264631,
      "grad_norm": 0.4598666607693008,
      "kl": 0.03839111328125,
      "learning_rate": 2e-06,
      "loss": -0.0082,
      "reward": 0.620535746216774,
      "reward_std": 0.17757731303572655,
      "rewards/accuracy_reward": 0.620535746216774,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.4241256713867,
      "epoch": 2.376590330788804,
      "grad_norm": 0.24305152529088678,
      "kl": 0.036865234375,
      "learning_rate": 2e-06,
      "loss": -0.005,
      "reward": 0.770089328289032,
      "reward_std": 0.04526757914572954,
      "rewards/accuracy_reward": 0.770089328289032,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.6741256713867,
      "epoch": 2.381679389312977,
      "grad_norm": 0.25200191074624484,
      "kl": 0.035003662109375,
      "learning_rate": 2e-06,
      "loss": -0.0083,
      "reward": 0.6227678805589676,
      "reward_std": 0.05827951803803444,
      "rewards/accuracy_reward": 0.6227678805589676,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.37501525878906,
      "epoch": 2.38676844783715,
      "grad_norm": 0.2410195169582811,
      "kl": 0.033538818359375,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.7790178954601288,
      "reward_std": 0.0782818403095007,
      "rewards/accuracy_reward": 0.7790178954601288,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.60939025878906,
      "epoch": 2.391857506361323,
      "grad_norm": 0.2660262521334539,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": 0.0117,
      "reward": 0.7589286118745804,
      "reward_std": 0.07602170202881098,
      "rewards/accuracy_reward": 0.7589286118745804,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.2210006713867,
      "epoch": 2.3969465648854964,
      "grad_norm": 0.38122609024199494,
      "kl": 0.034332275390625,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.7433035969734192,
      "reward_std": 0.14660858921706676,
      "rewards/accuracy_reward": 0.7433035969734192,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 435.5268096923828,
      "epoch": 2.4020356234096694,
      "grad_norm": 0.23998608078259098,
      "kl": 0.035552978515625,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.6540178805589676,
      "reward_std": 0.0509259719401598,
      "rewards/accuracy_reward": 0.6540178805589676,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.6919860839844,
      "epoch": 2.4071246819338423,
      "grad_norm": 0.3133690114987392,
      "kl": 0.03448486328125,
      "learning_rate": 2e-06,
      "loss": -0.0145,
      "reward": 0.6808035969734192,
      "reward_std": 0.13319170102477074,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.46207427978516,
      "epoch": 2.4122137404580153,
      "grad_norm": 0.2293450539149541,
      "kl": 0.03485107421875,
      "learning_rate": 2e-06,
      "loss": -0.0026,
      "reward": 0.848214328289032,
      "reward_std": 0.05338135547935963,
      "rewards/accuracy_reward": 0.848214328289032,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.0669860839844,
      "epoch": 2.4173027989821882,
      "grad_norm": 0.28646665317742065,
      "kl": 0.04327392578125,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.6473214477300644,
      "reward_std": 0.09079350251704454,
      "rewards/accuracy_reward": 0.6473214477300644,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.1138610839844,
      "epoch": 2.422391857506361,
      "grad_norm": 0.24344233501139234,
      "kl": 0.035247802734375,
      "learning_rate": 2e-06,
      "loss": 0.0095,
      "reward": 0.8816964626312256,
      "reward_std": 0.07695145718753338,
      "rewards/accuracy_reward": 0.8816964626312256,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.2812728881836,
      "epoch": 2.427480916030534,
      "grad_norm": 0.19091344253106,
      "kl": 0.0390625,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.823660746216774,
      "reward_std": 0.04893965553492308,
      "rewards/accuracy_reward": 0.823660746216774,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.5424270629883,
      "epoch": 2.432569974554707,
      "grad_norm": 0.2162042112233782,
      "kl": 0.034881591796875,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.7656250298023224,
      "reward_std": 0.05684161186218262,
      "rewards/accuracy_reward": 0.7656250298023224,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.49778747558594,
      "epoch": 2.4376590330788805,
      "grad_norm": 0.2279534507206969,
      "kl": 0.0379638671875,
      "learning_rate": 2e-06,
      "loss": -0.0038,
      "reward": 0.7232143133878708,
      "reward_std": 0.07123849913477898,
      "rewards/accuracy_reward": 0.7232143133878708,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.0960006713867,
      "epoch": 2.4427480916030535,
      "grad_norm": 0.3931866378264231,
      "kl": 0.041046142578125,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.6450893133878708,
      "reward_std": 0.08632101211696863,
      "rewards/accuracy_reward": 0.6450893133878708,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.54466247558594,
      "epoch": 2.4478371501272265,
      "grad_norm": 0.3263765017270065,
      "kl": 0.037872314453125,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.7299107611179352,
      "reward_std": 0.09125630278140306,
      "rewards/accuracy_reward": 0.7299107611179352,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.89064025878906,
      "epoch": 2.4529262086513994,
      "grad_norm": 0.37816173334724634,
      "kl": 0.03729248046875,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.5312500223517418,
      "reward_std": 0.18900035321712494,
      "rewards/accuracy_reward": 0.5312500223517418,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.5089569091797,
      "epoch": 2.4580152671755724,
      "grad_norm": 0.31647115041719376,
      "kl": 0.0355224609375,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.6406250298023224,
      "reward_std": 0.13283337000757456,
      "rewards/accuracy_reward": 0.6406250298023224,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.91297149658203,
      "epoch": 2.4631043256997454,
      "grad_norm": 0.44899756852157036,
      "kl": 0.0477294921875,
      "learning_rate": 2e-06,
      "loss": -0.0079,
      "reward": 0.8258928805589676,
      "reward_std": 0.09242038615047932,
      "rewards/accuracy_reward": 0.8258928805589676,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 456.3527069091797,
      "epoch": 2.4681933842239188,
      "grad_norm": 0.3007865981492443,
      "kl": 0.03271484375,
      "learning_rate": 2e-06,
      "loss": 0.0084,
      "reward": 0.776785746216774,
      "reward_std": 0.1100874561816454,
      "rewards/accuracy_reward": 0.776785746216774,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.5625228881836,
      "epoch": 2.4732824427480917,
      "grad_norm": 0.3841981197397463,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.7053571715950966,
      "reward_std": 0.14869811479002237,
      "rewards/accuracy_reward": 0.7053571715950966,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.3526916503906,
      "epoch": 2.4783715012722647,
      "grad_norm": 0.21849881479643907,
      "kl": 0.0345458984375,
      "learning_rate": 2e-06,
      "loss": 0.0036,
      "reward": 0.6629464626312256,
      "reward_std": 0.0834527313709259,
      "rewards/accuracy_reward": 0.6629464626312256,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 371.7410888671875,
      "epoch": 2.4834605597964376,
      "grad_norm": 0.320815641821296,
      "kl": 0.04010009765625,
      "learning_rate": 2e-06,
      "loss": 0.017,
      "reward": 0.7187500447034836,
      "reward_std": 0.12377672828733921,
      "rewards/accuracy_reward": 0.7187500447034836,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.64733123779297,
      "epoch": 2.4885496183206106,
      "grad_norm": 0.295315154149744,
      "kl": 0.038330078125,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.5892857387661934,
      "reward_std": 0.1196470819413662,
      "rewards/accuracy_reward": 0.5892857387661934,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.5044860839844,
      "epoch": 2.4936386768447836,
      "grad_norm": 0.30493045133475927,
      "kl": 0.0389404296875,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.707589328289032,
      "reward_std": 0.09578065201640129,
      "rewards/accuracy_reward": 0.707589328289032,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.64288330078125,
      "epoch": 2.4987277353689565,
      "grad_norm": 0.36220255206982693,
      "kl": 0.03564453125,
      "learning_rate": 2e-06,
      "loss": 0.0107,
      "reward": 0.593750037252903,
      "reward_std": 0.10374343954026699,
      "rewards/accuracy_reward": 0.593750037252903,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.23439025878906,
      "epoch": 2.5038167938931295,
      "grad_norm": 0.2994940841618887,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": 0.0056,
      "reward": 0.7455357611179352,
      "reward_std": 0.12250707671046257,
      "rewards/accuracy_reward": 0.7455357611179352,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.5736846923828,
      "epoch": 2.508905852417303,
      "grad_norm": 0.3568713253372041,
      "kl": 0.0406494140625,
      "learning_rate": 2e-06,
      "loss": 0.0119,
      "reward": 0.553571455180645,
      "reward_std": 0.14193019922822714,
      "rewards/accuracy_reward": 0.553571455180645,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.1428680419922,
      "epoch": 2.513994910941476,
      "grad_norm": 0.31931457684348474,
      "kl": 0.03009033203125,
      "learning_rate": 2e-06,
      "loss": -0.0032,
      "reward": 0.6830357313156128,
      "reward_std": 0.12449068203568459,
      "rewards/accuracy_reward": 0.6830357313156128,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.0402069091797,
      "epoch": 2.519083969465649,
      "grad_norm": 0.3240174542764656,
      "kl": 0.03570556640625,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.7522321790456772,
      "reward_std": 0.12789164297282696,
      "rewards/accuracy_reward": 0.7522321790456772,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.61385345458984,
      "epoch": 2.524173027989822,
      "grad_norm": 0.3173437085118972,
      "kl": 0.04254150390625,
      "learning_rate": 2e-06,
      "loss": -0.0039,
      "reward": 0.7232143133878708,
      "reward_std": 0.10991919878870249,
      "rewards/accuracy_reward": 0.7232143133878708,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.8303756713867,
      "epoch": 2.5292620865139948,
      "grad_norm": 0.36127386983772714,
      "kl": 0.036865234375,
      "learning_rate": 2e-06,
      "loss": -0.0017,
      "reward": 0.7098214700818062,
      "reward_std": 0.12185426987707615,
      "rewards/accuracy_reward": 0.7098214700818062,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.06697845458984,
      "epoch": 2.534351145038168,
      "grad_norm": 0.36061772993304764,
      "kl": 0.034820556640625,
      "learning_rate": 2e-06,
      "loss": 0.0056,
      "reward": 0.754464328289032,
      "reward_std": 0.15109536796808243,
      "rewards/accuracy_reward": 0.754464328289032,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.91966247558594,
      "epoch": 2.539440203562341,
      "grad_norm": 0.39722756084611466,
      "kl": 0.0340576171875,
      "learning_rate": 2e-06,
      "loss": 0.002,
      "reward": 0.6562500447034836,
      "reward_std": 0.1960136517882347,
      "rewards/accuracy_reward": 0.6562500447034836,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.82592010498047,
      "epoch": 2.544529262086514,
      "grad_norm": 0.31069158155189774,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.6093750298023224,
      "reward_std": 0.10550853796303272,
      "rewards/accuracy_reward": 0.6093750298023224,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.2321548461914,
      "epoch": 2.549618320610687,
      "grad_norm": 0.4584225175807347,
      "kl": 0.045654296875,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.7433036118745804,
      "reward_std": 0.12795093655586243,
      "rewards/accuracy_reward": 0.7433036118745804,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.7857360839844,
      "epoch": 2.55470737913486,
      "grad_norm": 0.28617880733955425,
      "kl": 0.033416748046875,
      "learning_rate": 2e-06,
      "loss": 0.0143,
      "reward": 0.6674107387661934,
      "reward_std": 0.11140253208577633,
      "rewards/accuracy_reward": 0.6674107387661934,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.3281478881836,
      "epoch": 2.559796437659033,
      "grad_norm": 0.3015933954740273,
      "kl": 0.032470703125,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.7254464626312256,
      "reward_std": 0.13475124537944794,
      "rewards/accuracy_reward": 0.7254464626312256,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.11609649658203,
      "epoch": 2.564885496183206,
      "grad_norm": 0.26912659206068457,
      "kl": 0.033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.7321428954601288,
      "reward_std": 0.07387762051075697,
      "rewards/accuracy_reward": 0.7321428954601288,
      "step": 503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.60939025878906,
      "epoch": 2.569974554707379,
      "grad_norm": 0.27967517952321175,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.7857143133878708,
      "reward_std": 0.09173476323485374,
      "rewards/accuracy_reward": 0.7857143133878708,
      "step": 504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.6250228881836,
      "epoch": 2.5750636132315523,
      "grad_norm": 0.22205355273321656,
      "kl": 0.031585693359375,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.7343750298023224,
      "reward_std": 0.08394023403525352,
      "rewards/accuracy_reward": 0.7343750298023224,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.39510345458984,
      "epoch": 2.5801526717557253,
      "grad_norm": 0.2898121744066452,
      "kl": 0.0379638671875,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.620535746216774,
      "reward_std": 0.10770668089389801,
      "rewards/accuracy_reward": 0.620535746216774,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.64510345458984,
      "epoch": 2.5852417302798982,
      "grad_norm": 0.3362396947209023,
      "kl": 0.036376953125,
      "learning_rate": 2e-06,
      "loss": 0.0043,
      "reward": 0.7968750298023224,
      "reward_std": 0.16273310966789722,
      "rewards/accuracy_reward": 0.7968750298023224,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.9218978881836,
      "epoch": 2.590330788804071,
      "grad_norm": 0.23902970790337014,
      "kl": 0.030181884765625,
      "learning_rate": 2e-06,
      "loss": 0.0054,
      "reward": 0.7366071790456772,
      "reward_std": 0.07940684352070093,
      "rewards/accuracy_reward": 0.7366071790456772,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.7544860839844,
      "epoch": 2.595419847328244,
      "grad_norm": 0.29514999944220416,
      "kl": 0.03887939453125,
      "learning_rate": 2e-06,
      "loss": 0.0033,
      "reward": 0.6584821715950966,
      "reward_std": 0.07890495471656322,
      "rewards/accuracy_reward": 0.6584821715950966,
      "step": 509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.8660888671875,
      "epoch": 2.6005089058524176,
      "grad_norm": 0.3798953413187013,
      "kl": 0.03662109375,
      "learning_rate": 2e-06,
      "loss": 0.0122,
      "reward": 0.6696428805589676,
      "reward_std": 0.19469597935676575,
      "rewards/accuracy_reward": 0.6696428805589676,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.5000228881836,
      "epoch": 2.6055979643765905,
      "grad_norm": 0.33955130967501823,
      "kl": 0.03558349609375,
      "learning_rate": 2e-06,
      "loss": -0.0025,
      "reward": 0.5491071715950966,
      "reward_std": 0.12097746971994638,
      "rewards/accuracy_reward": 0.5491071715950966,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.5692138671875,
      "epoch": 2.6106870229007635,
      "grad_norm": 0.24669263696065316,
      "kl": 0.0352783203125,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.6674107387661934,
      "reward_std": 0.08934654574841261,
      "rewards/accuracy_reward": 0.6674107387661934,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.66519927978516,
      "epoch": 2.6157760814249365,
      "grad_norm": 0.2859664489787216,
      "kl": 0.0347900390625,
      "learning_rate": 2e-06,
      "loss": 0.0009,
      "reward": 0.7209821790456772,
      "reward_std": 0.10619255620986223,
      "rewards/accuracy_reward": 0.7209821790456772,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.8549270629883,
      "epoch": 2.6208651399491094,
      "grad_norm": 0.2572719508539601,
      "kl": 0.03143310546875,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.7790178954601288,
      "reward_std": 0.12461351975798607,
      "rewards/accuracy_reward": 0.7790178954601288,
      "step": 514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.76341247558594,
      "epoch": 2.6259541984732824,
      "grad_norm": 0.3449533451220129,
      "kl": 0.03955078125,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.7544643133878708,
      "reward_std": 0.10766920261085033,
      "rewards/accuracy_reward": 0.7544643133878708,
      "step": 515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.7835006713867,
      "epoch": 2.6310432569974553,
      "grad_norm": 0.632385436314343,
      "kl": 0.032257080078125,
      "learning_rate": 2e-06,
      "loss": -0.0098,
      "reward": 0.8058035969734192,
      "reward_std": 0.13097807671874762,
      "rewards/accuracy_reward": 0.8058035969734192,
      "step": 516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.59154510498047,
      "epoch": 2.6361323155216283,
      "grad_norm": 0.33196555497200964,
      "kl": 0.0389404296875,
      "learning_rate": 2e-06,
      "loss": 0.0069,
      "reward": 0.587053582072258,
      "reward_std": 0.11103776469826698,
      "rewards/accuracy_reward": 0.587053582072258,
      "step": 517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.5513610839844,
      "epoch": 2.6412213740458013,
      "grad_norm": 0.3451468883461782,
      "kl": 0.03460693359375,
      "learning_rate": 2e-06,
      "loss": 0.0149,
      "reward": 0.5937500149011612,
      "reward_std": 0.14383834693580866,
      "rewards/accuracy_reward": 0.5937500149011612,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.51788330078125,
      "epoch": 2.6463104325699747,
      "grad_norm": 0.28800529153503623,
      "kl": 0.034454345703125,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.5959821790456772,
      "reward_std": 0.12700967863202095,
      "rewards/accuracy_reward": 0.5959821790456772,
      "step": 519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.31251525878906,
      "epoch": 2.6513994910941476,
      "grad_norm": 0.25559387172202164,
      "kl": 0.033294677734375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6339285969734192,
      "reward_std": 0.11485719867050648,
      "rewards/accuracy_reward": 0.6339285969734192,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.74109649658203,
      "epoch": 2.6564885496183206,
      "grad_norm": 0.2536722799441574,
      "kl": 0.03643798828125,
      "learning_rate": 2e-06,
      "loss": 0.0132,
      "reward": 0.4910714328289032,
      "reward_std": 0.08318000193685293,
      "rewards/accuracy_reward": 0.4910714328289032,
      "step": 521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.70983123779297,
      "epoch": 2.6615776081424936,
      "grad_norm": 0.2221512357164997,
      "kl": 0.03692626953125,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.683035746216774,
      "reward_std": 0.07241806481033564,
      "rewards/accuracy_reward": 0.683035746216774,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.91297149658203,
      "epoch": 2.6666666666666665,
      "grad_norm": 0.3046176456837757,
      "kl": 0.033477783203125,
      "learning_rate": 2e-06,
      "loss": -0.0055,
      "reward": 0.7321428805589676,
      "reward_std": 0.0882824556902051,
      "rewards/accuracy_reward": 0.7321428805589676,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 372.9174270629883,
      "epoch": 2.67175572519084,
      "grad_norm": 0.2409606452831004,
      "kl": 0.04193115234375,
      "learning_rate": 2e-06,
      "loss": -0.004,
      "reward": 0.7098214477300644,
      "reward_std": 0.04791303910315037,
      "rewards/accuracy_reward": 0.7098214477300644,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.5781478881836,
      "epoch": 2.676844783715013,
      "grad_norm": 0.27818205005551117,
      "kl": 0.03411865234375,
      "learning_rate": 2e-06,
      "loss": -0.0025,
      "reward": 0.6294643133878708,
      "reward_std": 0.07959691435098648,
      "rewards/accuracy_reward": 0.6294643133878708,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.12278747558594,
      "epoch": 2.681933842239186,
      "grad_norm": 0.3064833397677243,
      "kl": 0.04248046875,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.595982164144516,
      "reward_std": 0.08645018469542265,
      "rewards/accuracy_reward": 0.595982164144516,
      "step": 526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.58707427978516,
      "epoch": 2.687022900763359,
      "grad_norm": 0.1439587126531661,
      "kl": 0.037139892578125,
      "learning_rate": 2e-06,
      "loss": -0.0036,
      "reward": 0.6718750298023224,
      "reward_std": 0.029295665211975574,
      "rewards/accuracy_reward": 0.6718750298023224,
      "step": 527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.4710006713867,
      "epoch": 2.6921119592875318,
      "grad_norm": 0.19355861305380997,
      "kl": 0.036376953125,
      "learning_rate": 2e-06,
      "loss": -0.008,
      "reward": 0.6785714700818062,
      "reward_std": 0.05042298696935177,
      "rewards/accuracy_reward": 0.6785714700818062,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.09153747558594,
      "epoch": 2.6972010178117047,
      "grad_norm": 0.2527678447303911,
      "kl": 0.03192138671875,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.7053571939468384,
      "reward_std": 0.06765811704099178,
      "rewards/accuracy_reward": 0.7053571939468384,
      "step": 529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.0491256713867,
      "epoch": 2.7022900763358777,
      "grad_norm": 0.331200966018847,
      "kl": 0.03302001953125,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.6852678880095482,
      "reward_std": 0.09613898955285549,
      "rewards/accuracy_reward": 0.6852678880095482,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.2701110839844,
      "epoch": 2.7073791348600507,
      "grad_norm": 0.29291114397156326,
      "kl": 0.03558349609375,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.6071428954601288,
      "reward_std": 0.11480188090354204,
      "rewards/accuracy_reward": 0.6071428954601288,
      "step": 531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.6607360839844,
      "epoch": 2.712468193384224,
      "grad_norm": 0.31804500261647317,
      "kl": 0.03912353515625,
      "learning_rate": 2e-06,
      "loss": 0.009,
      "reward": 0.7834821790456772,
      "reward_std": 0.1205065455287695,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.7589416503906,
      "epoch": 2.717557251908397,
      "grad_norm": 0.26914663178766046,
      "kl": 0.040771484375,
      "learning_rate": 2e-06,
      "loss": 0.0155,
      "reward": 0.676339328289032,
      "reward_std": 0.07469875272363424,
      "rewards/accuracy_reward": 0.676339328289032,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.51563262939453,
      "epoch": 2.72264631043257,
      "grad_norm": 0.36988046687006165,
      "kl": 0.04022216796875,
      "learning_rate": 2e-06,
      "loss": 0.0112,
      "reward": 0.6517857387661934,
      "reward_std": 0.14681287854909897,
      "rewards/accuracy_reward": 0.6517857387661934,
      "step": 534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.5245666503906,
      "epoch": 2.727735368956743,
      "grad_norm": 0.3936767345121069,
      "kl": 0.03948974609375,
      "learning_rate": 2e-06,
      "loss": -0.009,
      "reward": 0.7544643133878708,
      "reward_std": 0.1561136618256569,
      "rewards/accuracy_reward": 0.7544643133878708,
      "step": 535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.8482360839844,
      "epoch": 2.732824427480916,
      "grad_norm": 0.24488324139826242,
      "kl": 0.037109375,
      "learning_rate": 2e-06,
      "loss": 0.013,
      "reward": 0.6562500223517418,
      "reward_std": 0.07999247591942549,
      "rewards/accuracy_reward": 0.6562500223517418,
      "step": 536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.2924270629883,
      "epoch": 2.7379134860050893,
      "grad_norm": 0.09820479925311285,
      "kl": 0.034423828125,
      "learning_rate": 2e-06,
      "loss": 0.0004,
      "reward": 0.8125000298023224,
      "reward_std": 0.015971913933753967,
      "rewards/accuracy_reward": 0.8125000298023224,
      "step": 537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.64064025878906,
      "epoch": 2.7430025445292623,
      "grad_norm": 0.286300366081529,
      "kl": 0.03802490234375,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.816964328289032,
      "reward_std": 0.09291432984173298,
      "rewards/accuracy_reward": 0.816964328289032,
      "step": 538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.6518020629883,
      "epoch": 2.7480916030534353,
      "grad_norm": 0.3399892125694863,
      "kl": 0.036865234375,
      "learning_rate": 2e-06,
      "loss": -0.0024,
      "reward": 0.636160746216774,
      "reward_std": 0.11417082604020834,
      "rewards/accuracy_reward": 0.636160746216774,
      "step": 539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.64510345458984,
      "epoch": 2.753180661577608,
      "grad_norm": 0.2047451118377167,
      "kl": 0.0401611328125,
      "learning_rate": 2e-06,
      "loss": -0.0034,
      "reward": 0.7566964402794838,
      "reward_std": 0.07594658620655537,
      "rewards/accuracy_reward": 0.7566964402794838,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.5424270629883,
      "epoch": 2.758269720101781,
      "grad_norm": 0.20144452404013666,
      "kl": 0.0401611328125,
      "learning_rate": 2e-06,
      "loss": -0.0043,
      "reward": 0.645089328289032,
      "reward_std": 0.058865154162049294,
      "rewards/accuracy_reward": 0.645089328289032,
      "step": 541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.56029510498047,
      "epoch": 2.763358778625954,
      "grad_norm": 0.2343635165306646,
      "kl": 0.03564453125,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.6651786118745804,
      "reward_std": 0.08738510962575674,
      "rewards/accuracy_reward": 0.6651786118745804,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.4085006713867,
      "epoch": 2.768447837150127,
      "grad_norm": 0.3373893144181298,
      "kl": 0.0377197265625,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.6584821790456772,
      "reward_std": 0.13745236676186323,
      "rewards/accuracy_reward": 0.6584821790456772,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.9843978881836,
      "epoch": 2.7735368956743,
      "grad_norm": 0.3204217362971684,
      "kl": 0.0377197265625,
      "learning_rate": 2e-06,
      "loss": -0.0091,
      "reward": 0.660714328289032,
      "reward_std": 0.13499699160456657,
      "rewards/accuracy_reward": 0.660714328289032,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.93751525878906,
      "epoch": 2.778625954198473,
      "grad_norm": 0.3300048793679498,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.6830357611179352,
      "reward_std": 0.14531533233821392,
      "rewards/accuracy_reward": 0.6830357611179352,
      "step": 545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.7366256713867,
      "epoch": 2.7837150127226464,
      "grad_norm": 0.2780528336605366,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.6517857536673546,
      "reward_std": 0.07582645677030087,
      "rewards/accuracy_reward": 0.6517857536673546,
      "step": 546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.80582427978516,
      "epoch": 2.7888040712468194,
      "grad_norm": 0.21568216254912498,
      "kl": 0.033203125,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.7633928954601288,
      "reward_std": 0.05859133042395115,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.2299270629883,
      "epoch": 2.7938931297709924,
      "grad_norm": 0.3650557249780628,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.8370536118745804,
      "reward_std": 0.1332291793078184,
      "rewards/accuracy_reward": 0.8370536118745804,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.74332427978516,
      "epoch": 2.7989821882951653,
      "grad_norm": 0.284772071971351,
      "kl": 0.03265380859375,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.7745535969734192,
      "reward_std": 0.07155140861868858,
      "rewards/accuracy_reward": 0.7745535969734192,
      "step": 549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.8884048461914,
      "epoch": 2.8040712468193383,
      "grad_norm": 0.377128397877783,
      "kl": 0.034881591796875,
      "learning_rate": 2e-06,
      "loss": 0.0067,
      "reward": 0.6540178805589676,
      "reward_std": 0.1443558856844902,
      "rewards/accuracy_reward": 0.6540178805589676,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.68975830078125,
      "epoch": 2.8091603053435117,
      "grad_norm": 0.2440057694514428,
      "kl": 0.04638671875,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.7723214477300644,
      "reward_std": 0.06154969800263643,
      "rewards/accuracy_reward": 0.7723214477300644,
      "step": 551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.14957427978516,
      "epoch": 2.8142493638676847,
      "grad_norm": 0.30153622558736576,
      "kl": 0.035064697265625,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.6852678954601288,
      "reward_std": 0.08264064136892557,
      "rewards/accuracy_reward": 0.6852678954601288,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.56028747558594,
      "epoch": 2.8193384223918576,
      "grad_norm": 0.337655409367331,
      "kl": 0.04473876953125,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.5915178954601288,
      "reward_std": 0.10053416807204485,
      "rewards/accuracy_reward": 0.5915178954601288,
      "step": 553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.39510345458984,
      "epoch": 2.8244274809160306,
      "grad_norm": 0.30254905149794203,
      "kl": 0.0377197265625,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.7834821939468384,
      "reward_std": 0.10977564752101898,
      "rewards/accuracy_reward": 0.7834821939468384,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.9397506713867,
      "epoch": 2.8295165394402035,
      "grad_norm": 0.2980175473217978,
      "kl": 0.033172607421875,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.6026786118745804,
      "reward_std": 0.10644346475601196,
      "rewards/accuracy_reward": 0.6026786118745804,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.2656478881836,
      "epoch": 2.8346055979643765,
      "grad_norm": 0.30986857343738305,
      "kl": 0.0374755859375,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.651785746216774,
      "reward_std": 0.11615678668022156,
      "rewards/accuracy_reward": 0.651785746216774,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.85491943359375,
      "epoch": 2.8396946564885495,
      "grad_norm": 0.4046743261194774,
      "kl": 0.03515625,
      "learning_rate": 2e-06,
      "loss": 0.0084,
      "reward": 0.6383928880095482,
      "reward_std": 0.16278479434549809,
      "rewards/accuracy_reward": 0.6383928880095482,
      "step": 557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.50447845458984,
      "epoch": 2.8447837150127224,
      "grad_norm": 0.33955198158693556,
      "kl": 0.031982421875,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.6406250223517418,
      "reward_std": 0.14977919682860374,
      "rewards/accuracy_reward": 0.6406250223517418,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 445.9486770629883,
      "epoch": 2.849872773536896,
      "grad_norm": 0.5235292122720531,
      "kl": 0.04193115234375,
      "learning_rate": 2e-06,
      "loss": 0.005,
      "reward": 0.6785714477300644,
      "reward_std": 0.13369468972086906,
      "rewards/accuracy_reward": 0.6785714477300644,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.21653747558594,
      "epoch": 2.854961832061069,
      "grad_norm": 0.3647830766982642,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.5580357313156128,
      "reward_std": 0.09375077206641436,
      "rewards/accuracy_reward": 0.5580357313156128,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.5781478881836,
      "epoch": 2.8600508905852418,
      "grad_norm": 0.2915846428107292,
      "kl": 0.03924560546875,
      "learning_rate": 2e-06,
      "loss": -0.0101,
      "reward": 0.7500000447034836,
      "reward_std": 0.09638989623636007,
      "rewards/accuracy_reward": 0.7500000447034836,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.76341247558594,
      "epoch": 2.8651399491094147,
      "grad_norm": 2.5118673372186335,
      "kl": 0.06280517578125,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.74776791036129,
      "reward_std": 0.14617369696497917,
      "rewards/accuracy_reward": 0.74776791036129,
      "step": 562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.06922149658203,
      "epoch": 2.8702290076335877,
      "grad_norm": 0.3476480288065041,
      "kl": 0.031097412109375,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.631696455180645,
      "reward_std": 0.1722291260957718,
      "rewards/accuracy_reward": 0.631696455180645,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.5580520629883,
      "epoch": 2.875318066157761,
      "grad_norm": 0.317993366086719,
      "kl": 0.04376220703125,
      "learning_rate": 2e-06,
      "loss": 0.0116,
      "reward": 0.5178571715950966,
      "reward_std": 0.09537875838577747,
      "rewards/accuracy_reward": 0.5178571715950966,
      "step": 564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.98216247558594,
      "epoch": 2.880407124681934,
      "grad_norm": 0.2902885029430109,
      "kl": 0.036529541015625,
      "learning_rate": 2e-06,
      "loss": 0.0118,
      "reward": 0.6250000298023224,
      "reward_std": 0.07891290169209242,
      "rewards/accuracy_reward": 0.6250000298023224,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.84376525878906,
      "epoch": 2.885496183206107,
      "grad_norm": 0.32453080996916994,
      "kl": 0.03338623046875,
      "learning_rate": 2e-06,
      "loss": 0.0128,
      "reward": 0.7053571715950966,
      "reward_std": 0.12533627077937126,
      "rewards/accuracy_reward": 0.7053571715950966,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.2991256713867,
      "epoch": 2.89058524173028,
      "grad_norm": 0.3087751087174545,
      "kl": 0.041015625,
      "learning_rate": 2e-06,
      "loss": 0.0077,
      "reward": 0.5691964477300644,
      "reward_std": 0.11318294238299131,
      "rewards/accuracy_reward": 0.5691964477300644,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.6428680419922,
      "epoch": 2.895674300254453,
      "grad_norm": 0.4055341932266634,
      "kl": 0.04046630859375,
      "learning_rate": 2e-06,
      "loss": -0.017,
      "reward": 0.6294643208384514,
      "reward_std": 0.1625868445262313,
      "rewards/accuracy_reward": 0.6294643208384514,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.8750228881836,
      "epoch": 2.900763358778626,
      "grad_norm": 0.3847610873766707,
      "kl": 0.036651611328125,
      "learning_rate": 2e-06,
      "loss": -0.0055,
      "reward": 0.8102678805589676,
      "reward_std": 0.2118864431977272,
      "rewards/accuracy_reward": 0.8102678805589676,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.62947845458984,
      "epoch": 2.905852417302799,
      "grad_norm": 0.41563935712517536,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.6964285969734192,
      "reward_std": 0.1858063880354166,
      "rewards/accuracy_reward": 0.6964285969734192,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.64957427978516,
      "epoch": 2.910941475826972,
      "grad_norm": 0.44069690199594175,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": -0.0047,
      "reward": 0.6897321790456772,
      "reward_std": 0.20478853583335876,
      "rewards/accuracy_reward": 0.6897321790456772,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 373.8035888671875,
      "epoch": 2.916030534351145,
      "grad_norm": 0.3411320971221695,
      "kl": 0.035247802734375,
      "learning_rate": 2e-06,
      "loss": 0.0237,
      "reward": 0.7745535969734192,
      "reward_std": 0.12431609351187944,
      "rewards/accuracy_reward": 0.7745535969734192,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.82814025878906,
      "epoch": 2.921119592875318,
      "grad_norm": 0.46650189630733896,
      "kl": 0.04046630859375,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.6852678805589676,
      "reward_std": 0.2135436087846756,
      "rewards/accuracy_reward": 0.6852678805589676,
      "step": 573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.54689025878906,
      "epoch": 2.926208651399491,
      "grad_norm": 0.3633625043570078,
      "kl": 0.0404052734375,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.6651785969734192,
      "reward_std": 0.13997509516775608,
      "rewards/accuracy_reward": 0.6651785969734192,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.51342010498047,
      "epoch": 2.931297709923664,
      "grad_norm": 0.3486662295054925,
      "kl": 0.03570556640625,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.6651786118745804,
      "reward_std": 0.1588444784283638,
      "rewards/accuracy_reward": 0.6651786118745804,
      "step": 575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.35939025878906,
      "epoch": 2.936386768447837,
      "grad_norm": 0.2602516130746799,
      "kl": 0.03900146484375,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.6875000298023224,
      "reward_std": 0.0915446886792779,
      "rewards/accuracy_reward": 0.6875000298023224,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.9620666503906,
      "epoch": 2.94147582697201,
      "grad_norm": 0.35960628442279924,
      "kl": 0.034393310546875,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.7388393133878708,
      "reward_std": 0.1947804857045412,
      "rewards/accuracy_reward": 0.7388393133878708,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.1986770629883,
      "epoch": 2.9465648854961835,
      "grad_norm": 0.374525516444503,
      "kl": 0.030975341796875,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.6227678880095482,
      "reward_std": 0.17119617387652397,
      "rewards/accuracy_reward": 0.6227678880095482,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.1183319091797,
      "epoch": 2.9516539440203564,
      "grad_norm": 0.2972914056458539,
      "kl": 0.03167724609375,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.6495536118745804,
      "reward_std": 0.10004776064306498,
      "rewards/accuracy_reward": 0.6495536118745804,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.02234649658203,
      "epoch": 2.9567430025445294,
      "grad_norm": 0.2602697339274284,
      "kl": 0.0377197265625,
      "learning_rate": 2e-06,
      "loss": 0.0121,
      "reward": 0.5669643208384514,
      "reward_std": 0.08922642096877098,
      "rewards/accuracy_reward": 0.5669643208384514,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.30358123779297,
      "epoch": 2.9618320610687023,
      "grad_norm": 0.2904891691402829,
      "kl": 0.03411865234375,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.8035714775323868,
      "reward_std": 0.09128634724766016,
      "rewards/accuracy_reward": 0.8035714775323868,
      "step": 581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.07814025878906,
      "epoch": 2.9669211195928753,
      "grad_norm": 0.3715192106038954,
      "kl": 0.03704833984375,
      "learning_rate": 2e-06,
      "loss": 0.0121,
      "reward": 0.5825893059372902,
      "reward_std": 0.14649997185915709,
      "rewards/accuracy_reward": 0.5825893059372902,
      "step": 582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.96207427978516,
      "epoch": 2.9720101781170483,
      "grad_norm": 0.37146966248031055,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.7187500298023224,
      "reward_std": 0.18226324021816254,
      "rewards/accuracy_reward": 0.7187500298023224,
      "step": 583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.5424270629883,
      "epoch": 2.9770992366412212,
      "grad_norm": 0.3870491567373662,
      "kl": 0.034027099609375,
      "learning_rate": 2e-06,
      "loss": 0.0108,
      "reward": 0.5758928880095482,
      "reward_std": 0.1732867918908596,
      "rewards/accuracy_reward": 0.5758928880095482,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.3437728881836,
      "epoch": 2.982188295165394,
      "grad_norm": 0.33433589615968345,
      "kl": 0.036346435546875,
      "learning_rate": 2e-06,
      "loss": -0.0018,
      "reward": 0.6651785969734192,
      "reward_std": 0.11502275057137012,
      "rewards/accuracy_reward": 0.6651785969734192,
      "step": 585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.1852798461914,
      "epoch": 2.9872773536895676,
      "grad_norm": 0.386757527349345,
      "kl": 0.03863525390625,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.7388393133878708,
      "reward_std": 0.10420894343405962,
      "rewards/accuracy_reward": 0.7388393133878708,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.43528747558594,
      "epoch": 2.9923664122137406,
      "grad_norm": 0.365220944421569,
      "kl": 0.034271240234375,
      "learning_rate": 2e-06,
      "loss": -0.0052,
      "reward": 0.6294643208384514,
      "reward_std": 0.16460285522043705,
      "rewards/accuracy_reward": 0.6294643208384514,
      "step": 587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 343.00001525878906,
      "epoch": 2.9974554707379135,
      "grad_norm": 0.40809859576380375,
      "kl": 0.040771484375,
      "learning_rate": 2e-06,
      "loss": -0.0079,
      "reward": 0.651785746216774,
      "reward_std": 0.21456751227378845,
      "rewards/accuracy_reward": 0.651785746216774,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.8236770629883,
      "epoch": 3.005089058524173,
      "grad_norm": 0.5054490077348739,
      "kl": 0.04595947265625,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.7745536118745804,
      "reward_std": 0.11722088698297739,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.66966247558594,
      "epoch": 3.010178117048346,
      "grad_norm": 0.29671769278277893,
      "kl": 0.04132080078125,
      "learning_rate": 2e-06,
      "loss": 0.0086,
      "reward": 0.8281250447034836,
      "reward_std": 0.08452587015926838,
      "rewards/accuracy_reward": 0.8281250447034836,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.2656478881836,
      "epoch": 3.015267175572519,
      "grad_norm": 0.3553393765638119,
      "kl": 0.034576416015625,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.7053571939468384,
      "reward_std": 0.11977626010775566,
      "rewards/accuracy_reward": 0.7053571939468384,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.9285888671875,
      "epoch": 3.0203562340966923,
      "grad_norm": 0.32713588955017314,
      "kl": 0.038970947265625,
      "learning_rate": 2e-06,
      "loss": 0.0084,
      "reward": 0.645089328289032,
      "reward_std": 0.07450868003070354,
      "rewards/accuracy_reward": 0.645089328289032,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 364.0178680419922,
      "epoch": 3.0254452926208653,
      "grad_norm": 0.44394381709580466,
      "kl": 0.049652099609375,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.6718750149011612,
      "reward_std": 0.14425750821828842,
      "rewards/accuracy_reward": 0.6718750149011612,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.5134048461914,
      "epoch": 3.030534351145038,
      "grad_norm": 0.33120930284551425,
      "kl": 0.0418701171875,
      "learning_rate": 2e-06,
      "loss": 0.0054,
      "reward": 0.6852678805589676,
      "reward_std": 0.11204889789223671,
      "rewards/accuracy_reward": 0.6852678805589676,
      "step": 594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.0535888671875,
      "epoch": 3.035623409669211,
      "grad_norm": 0.26027634818081746,
      "kl": 0.0386962890625,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.770089328289032,
      "reward_std": 0.0650099515914917,
      "rewards/accuracy_reward": 0.770089328289032,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.56028747558594,
      "epoch": 3.040712468193384,
      "grad_norm": 0.35247817317411145,
      "kl": 0.04168701171875,
      "learning_rate": 2e-06,
      "loss": -0.0044,
      "reward": 0.7098214775323868,
      "reward_std": 0.14996132999658585,
      "rewards/accuracy_reward": 0.7098214775323868,
      "step": 596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.66295623779297,
      "epoch": 3.045801526717557,
      "grad_norm": 0.27449168635641785,
      "kl": 0.04217529296875,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.6830357313156128,
      "reward_std": 0.08047888334840536,
      "rewards/accuracy_reward": 0.6830357313156128,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.22322845458984,
      "epoch": 3.05089058524173,
      "grad_norm": 0.31346122011455496,
      "kl": 0.04010009765625,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.7968750298023224,
      "reward_std": 0.10267934296280146,
      "rewards/accuracy_reward": 0.7968750298023224,
      "step": 598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.3616256713867,
      "epoch": 3.0559796437659035,
      "grad_norm": 0.3091595921253676,
      "kl": 0.0404052734375,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.6919643133878708,
      "reward_std": 0.09394084382802248,
      "rewards/accuracy_reward": 0.6919643133878708,
      "step": 599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.9375228881836,
      "epoch": 3.0610687022900764,
      "grad_norm": 0.2692371232740381,
      "kl": 0.03826904296875,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.707589328289032,
      "reward_std": 0.10160730499774218,
      "rewards/accuracy_reward": 0.707589328289032,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.5625228881836,
      "epoch": 3.0661577608142494,
      "grad_norm": 0.3397450164084543,
      "kl": 0.0394287109375,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.7254464626312256,
      "reward_std": 0.11211089976131916,
      "rewards/accuracy_reward": 0.7254464626312256,
      "step": 601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.9129638671875,
      "epoch": 3.0712468193384224,
      "grad_norm": 0.27695285703523814,
      "kl": 0.042236328125,
      "learning_rate": 2e-06,
      "loss": 0.0087,
      "reward": 0.6316964626312256,
      "reward_std": 0.11891603656113148,
      "rewards/accuracy_reward": 0.6316964626312256,
      "step": 602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.9107360839844,
      "epoch": 3.0763358778625953,
      "grad_norm": 0.35187548455133666,
      "kl": 0.04229736328125,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.7991071790456772,
      "reward_std": 0.14297254383563995,
      "rewards/accuracy_reward": 0.7991071790456772,
      "step": 603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 449.98439025878906,
      "epoch": 3.0814249363867683,
      "grad_norm": 181.87528038860185,
      "kl": 7.3677978515625,
      "learning_rate": 2e-06,
      "loss": 0.3632,
      "reward": 0.6093750298023224,
      "reward_std": 0.03898446634411812,
      "rewards/accuracy_reward": 0.6093750298023224,
      "step": 604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.67413330078125,
      "epoch": 3.0865139949109412,
      "grad_norm": 0.2238747470587776,
      "kl": 0.0469970703125,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.8370536267757416,
      "reward_std": 0.07155031058937311,
      "rewards/accuracy_reward": 0.8370536267757416,
      "step": 605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.4620666503906,
      "epoch": 3.0916030534351147,
      "grad_norm": 0.29689243902093276,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.7968750149011612,
      "reward_std": 0.09425375983119011,
      "rewards/accuracy_reward": 0.7968750149011612,
      "step": 606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.0201110839844,
      "epoch": 3.0966921119592876,
      "grad_norm": 0.29530092564323684,
      "kl": 0.0400390625,
      "learning_rate": 2e-06,
      "loss": -0.0047,
      "reward": 0.6964286044239998,
      "reward_std": 0.08155202306807041,
      "rewards/accuracy_reward": 0.6964286044239998,
      "step": 607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.64064025878906,
      "epoch": 3.1017811704834606,
      "grad_norm": 0.3836516877093382,
      "kl": 0.0430908203125,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.7901786118745804,
      "reward_std": 0.15968108735978603,
      "rewards/accuracy_reward": 0.7901786118745804,
      "step": 608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.1428680419922,
      "epoch": 3.1068702290076335,
      "grad_norm": 0.31082000201458926,
      "kl": 0.044677734375,
      "learning_rate": 2e-06,
      "loss": 0.0085,
      "reward": 0.79464291036129,
      "reward_std": 0.08997760713100433,
      "rewards/accuracy_reward": 0.79464291036129,
      "step": 609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.3393096923828,
      "epoch": 3.1119592875318065,
      "grad_norm": 0.3296703166146194,
      "kl": 0.0462646484375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6406250298023224,
      "reward_std": 0.12349375803023577,
      "rewards/accuracy_reward": 0.6406250298023224,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.76341247558594,
      "epoch": 3.1170483460559795,
      "grad_norm": 0.27837557288999537,
      "kl": 0.03546142578125,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.816964328289032,
      "reward_std": 0.09958386048674583,
      "rewards/accuracy_reward": 0.816964328289032,
      "step": 611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.76116943359375,
      "epoch": 3.122137404580153,
      "grad_norm": 0.16182405484609294,
      "kl": 0.036376953125,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.8169643133878708,
      "reward_std": 0.03539504110813141,
      "rewards/accuracy_reward": 0.8169643133878708,
      "step": 612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.9397506713867,
      "epoch": 3.127226463104326,
      "grad_norm": 0.29466200983917096,
      "kl": 0.04132080078125,
      "learning_rate": 2e-06,
      "loss": 0.0079,
      "reward": 0.8191964626312256,
      "reward_std": 0.06608309131115675,
      "rewards/accuracy_reward": 0.8191964626312256,
      "step": 613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.18528747558594,
      "epoch": 3.132315521628499,
      "grad_norm": 0.37028790033354736,
      "kl": 0.040283203125,
      "learning_rate": 2e-06,
      "loss": 0.0059,
      "reward": 0.6785714626312256,
      "reward_std": 0.14488856680691242,
      "rewards/accuracy_reward": 0.6785714626312256,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.8326110839844,
      "epoch": 3.1374045801526718,
      "grad_norm": 0.24426900672685442,
      "kl": 0.04193115234375,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.720982164144516,
      "reward_std": 0.08595624193549156,
      "rewards/accuracy_reward": 0.720982164144516,
      "step": 615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.8616256713867,
      "epoch": 3.1424936386768447,
      "grad_norm": 0.2820551616395041,
      "kl": 0.04302978515625,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.8281250447034836,
      "reward_std": 0.0801670728251338,
      "rewards/accuracy_reward": 0.8281250447034836,
      "step": 616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.2522506713867,
      "epoch": 3.1475826972010177,
      "grad_norm": 0.26225698657668917,
      "kl": 0.04449462890625,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.6116071715950966,
      "reward_std": 0.07501166500151157,
      "rewards/accuracy_reward": 0.6116071715950966,
      "step": 617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 346.5401916503906,
      "epoch": 3.1526717557251906,
      "grad_norm": 0.2630733654921422,
      "kl": 0.0440673828125,
      "learning_rate": 2e-06,
      "loss": 0.0024,
      "reward": 0.870535746216774,
      "reward_std": 0.06388495210558176,
      "rewards/accuracy_reward": 0.870535746216774,
      "step": 618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.43751525878906,
      "epoch": 3.157760814249364,
      "grad_norm": 0.21666299827365587,
      "kl": 0.0408935546875,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.832589328289032,
      "reward_std": 0.05401241313666105,
      "rewards/accuracy_reward": 0.832589328289032,
      "step": 619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.2835006713867,
      "epoch": 3.162849872773537,
      "grad_norm": 0.23041363980945273,
      "kl": 0.0338134765625,
      "learning_rate": 2e-06,
      "loss": -0.0029,
      "reward": 0.5937500223517418,
      "reward_std": 0.07110932841897011,
      "rewards/accuracy_reward": 0.5937500223517418,
      "step": 620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.21654510498047,
      "epoch": 3.16793893129771,
      "grad_norm": 0.31624338409159386,
      "kl": 0.04266357421875,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.723214328289032,
      "reward_std": 0.12588467728346586,
      "rewards/accuracy_reward": 0.723214328289032,
      "step": 621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.23885345458984,
      "epoch": 3.173027989821883,
      "grad_norm": 0.23240927253886212,
      "kl": 0.038848876953125,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.8080357611179352,
      "reward_std": 0.06388495303690434,
      "rewards/accuracy_reward": 0.8080357611179352,
      "step": 622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.7611846923828,
      "epoch": 3.178117048346056,
      "grad_norm": 0.25495663006769903,
      "kl": 0.04083251953125,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.7834821790456772,
      "reward_std": 0.062309928238391876,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.2500228881836,
      "epoch": 3.183206106870229,
      "grad_norm": 0.2430907572264551,
      "kl": 0.03887939453125,
      "learning_rate": 2e-06,
      "loss": 0.0077,
      "reward": 0.7455357611179352,
      "reward_std": 0.04791303817182779,
      "rewards/accuracy_reward": 0.7455357611179352,
      "step": 624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.83260345458984,
      "epoch": 3.188295165394402,
      "grad_norm": 0.18846428669679094,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.7410714626312256,
      "reward_std": 0.03441469371318817,
      "rewards/accuracy_reward": 0.7410714626312256,
      "step": 625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.01341247558594,
      "epoch": 3.1933842239185752,
      "grad_norm": 0.25244327963870355,
      "kl": 0.0400390625,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.7522321790456772,
      "reward_std": 0.058729544281959534,
      "rewards/accuracy_reward": 0.7522321790456772,
      "step": 626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.61163330078125,
      "epoch": 3.198473282442748,
      "grad_norm": 0.35993623996595636,
      "kl": 0.0455322265625,
      "learning_rate": 2e-06,
      "loss": -0.0009,
      "reward": 0.6964285969734192,
      "reward_std": 0.15657011978328228,
      "rewards/accuracy_reward": 0.6964285969734192,
      "step": 627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.1785888671875,
      "epoch": 3.203562340966921,
      "grad_norm": 0.270880235517583,
      "kl": 0.03558349609375,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.6852678805589676,
      "reward_std": 0.11927327513694763,
      "rewards/accuracy_reward": 0.6852678805589676,
      "step": 628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.5491256713867,
      "epoch": 3.208651399491094,
      "grad_norm": 0.17017632831466686,
      "kl": 0.041748046875,
      "learning_rate": 2e-06,
      "loss": 0.0024,
      "reward": 0.7611607611179352,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.7611607611179352,
      "step": 629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.70760345458984,
      "epoch": 3.213740458015267,
      "grad_norm": 0.2563735068076232,
      "kl": 0.04339599609375,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.698660746216774,
      "reward_std": 0.07047826889902353,
      "rewards/accuracy_reward": 0.698660746216774,
      "step": 630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.7053756713867,
      "epoch": 3.21882951653944,
      "grad_norm": 0.28587291112904484,
      "kl": 0.0439453125,
      "learning_rate": 2e-06,
      "loss": -0.0032,
      "reward": 0.745535746216774,
      "reward_std": 0.09940916765481234,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 443.2723388671875,
      "epoch": 3.223918575063613,
      "grad_norm": 0.23957626775845092,
      "kl": 0.033935546875,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.6808035969734192,
      "reward_std": 0.053694264963269234,
      "rewards/accuracy_reward": 0.6808035969734192,
      "step": 632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.56251525878906,
      "epoch": 3.2290076335877864,
      "grad_norm": 0.24762671660341598,
      "kl": 0.041046142578125,
      "learning_rate": 2e-06,
      "loss": -0.0018,
      "reward": 0.6986607611179352,
      "reward_std": 0.07905755192041397,
      "rewards/accuracy_reward": 0.6986607611179352,
      "step": 633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.37278747558594,
      "epoch": 3.2340966921119594,
      "grad_norm": 0.21421616813622704,
      "kl": 0.0419921875,
      "learning_rate": 2e-06,
      "loss": -0.0018,
      "reward": 0.926339328289032,
      "reward_std": 0.07376392930746078,
      "rewards/accuracy_reward": 0.926339328289032,
      "step": 634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.86832427978516,
      "epoch": 3.2391857506361323,
      "grad_norm": 0.29100501812736695,
      "kl": 0.036956787109375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.7477678954601288,
      "reward_std": 0.0895366258919239,
      "rewards/accuracy_reward": 0.7477678954601288,
      "step": 635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.35939025878906,
      "epoch": 3.2442748091603053,
      "grad_norm": 0.2336682314464919,
      "kl": 0.042816162109375,
      "learning_rate": 2e-06,
      "loss": -0.0002,
      "reward": 0.783482164144516,
      "reward_std": 0.07658668980002403,
      "rewards/accuracy_reward": 0.783482164144516,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.7924270629883,
      "epoch": 3.2493638676844783,
      "grad_norm": 0.3409330327011361,
      "kl": 0.04486083984375,
      "learning_rate": 2e-06,
      "loss": 0.012,
      "reward": 0.714285746216774,
      "reward_std": 0.12598120048642159,
      "rewards/accuracy_reward": 0.714285746216774,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.5960006713867,
      "epoch": 3.2544529262086512,
      "grad_norm": 0.2721761181692073,
      "kl": 0.036407470703125,
      "learning_rate": 2e-06,
      "loss": 0.0129,
      "reward": 0.676339328289032,
      "reward_std": 0.09084536600857973,
      "rewards/accuracy_reward": 0.676339328289032,
      "step": 638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.20091247558594,
      "epoch": 3.2595419847328246,
      "grad_norm": 0.2254530976772047,
      "kl": 0.0411376953125,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.863839328289032,
      "reward_std": 0.08098186552524567,
      "rewards/accuracy_reward": 0.863839328289032,
      "step": 639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.75894927978516,
      "epoch": 3.2646310432569976,
      "grad_norm": 0.23090982114457095,
      "kl": 0.034576416015625,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.6562500298023224,
      "reward_std": 0.058849673718214035,
      "rewards/accuracy_reward": 0.6562500298023224,
      "step": 640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.1986770629883,
      "epoch": 3.2697201017811706,
      "grad_norm": 0.20970869776093362,
      "kl": 0.038055419921875,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.7433036118745804,
      "reward_std": 0.056394290179014206,
      "rewards/accuracy_reward": 0.7433036118745804,
      "step": 641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.8504638671875,
      "epoch": 3.2748091603053435,
      "grad_norm": 0.3028783447423427,
      "kl": 0.04852294921875,
      "learning_rate": 2e-06,
      "loss": 0.005,
      "reward": 0.7589286118745804,
      "reward_std": 0.08613727707415819,
      "rewards/accuracy_reward": 0.7589286118745804,
      "step": 642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.9643020629883,
      "epoch": 3.2798982188295165,
      "grad_norm": 0.25492374875740365,
      "kl": 0.037567138671875,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.895089328289032,
      "reward_std": 0.0625000037252903,
      "rewards/accuracy_reward": 0.895089328289032,
      "step": 643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.77010345458984,
      "epoch": 3.2849872773536894,
      "grad_norm": 0.31517195871528625,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.814732164144516,
      "reward_std": 0.10004021972417831,
      "rewards/accuracy_reward": 0.814732164144516,
      "step": 644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.52010345458984,
      "epoch": 3.2900763358778624,
      "grad_norm": 0.2955813186190452,
      "kl": 0.0379638671875,
      "learning_rate": 2e-06,
      "loss": 0.013,
      "reward": 0.7656250447034836,
      "reward_std": 0.056651534512639046,
      "rewards/accuracy_reward": 0.7656250447034836,
      "step": 645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.21207427978516,
      "epoch": 3.2951653944020354,
      "grad_norm": 0.23760306636953452,
      "kl": 0.039703369140625,
      "learning_rate": 2e-06,
      "loss": 0.0101,
      "reward": 0.6696428805589676,
      "reward_std": 0.0654509337618947,
      "rewards/accuracy_reward": 0.6696428805589676,
      "step": 646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.0669708251953,
      "epoch": 3.300254452926209,
      "grad_norm": 0.3550598292123022,
      "kl": 0.0421142578125,
      "learning_rate": 2e-06,
      "loss": 0.0031,
      "reward": 0.8995536267757416,
      "reward_std": 0.1182012278586626,
      "rewards/accuracy_reward": 0.8995536267757416,
      "step": 647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.19197845458984,
      "epoch": 3.3053435114503817,
      "grad_norm": 0.2785541822365125,
      "kl": 0.0462646484375,
      "learning_rate": 2e-06,
      "loss": -0.0018,
      "reward": 0.7366071790456772,
      "reward_std": 0.0814228504896164,
      "rewards/accuracy_reward": 0.7366071790456772,
      "step": 648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.6607360839844,
      "epoch": 3.3104325699745547,
      "grad_norm": 0.2349895923734611,
      "kl": 0.0372314453125,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.7410714700818062,
      "reward_std": 0.061055757105350494,
      "rewards/accuracy_reward": 0.7410714700818062,
      "step": 649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 367.2835006713867,
      "epoch": 3.3155216284987277,
      "grad_norm": 0.22215560396533796,
      "kl": 0.04229736328125,
      "learning_rate": 2e-06,
      "loss": 0.0039,
      "reward": 0.6629464626312256,
      "reward_std": 0.05118321720510721,
      "rewards/accuracy_reward": 0.6629464626312256,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.7745666503906,
      "epoch": 3.3206106870229006,
      "grad_norm": 0.283289296655935,
      "kl": 0.0408935546875,
      "learning_rate": 2e-06,
      "loss": -0.0051,
      "reward": 0.8973214775323868,
      "reward_std": 0.07374844886362553,
      "rewards/accuracy_reward": 0.8973214775323868,
      "step": 651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.4285888671875,
      "epoch": 3.325699745547074,
      "grad_norm": 0.11145182451237014,
      "kl": 0.03375244140625,
      "learning_rate": 2e-06,
      "loss": 0.0028,
      "reward": 0.8995536118745804,
      "reward_std": 0.03225403279066086,
      "rewards/accuracy_reward": 0.8995536118745804,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.3616256713867,
      "epoch": 3.330788804071247,
      "grad_norm": 0.2587207892795744,
      "kl": 0.039306640625,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.7254464775323868,
      "reward_std": 0.06777824554592371,
      "rewards/accuracy_reward": 0.7254464775323868,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.97769927978516,
      "epoch": 3.33587786259542,
      "grad_norm": 0.17074145757257464,
      "kl": 0.035491943359375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.9218750447034836,
      "reward_std": 0.03332607354968786,
      "rewards/accuracy_reward": 0.9218750447034836,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.5669860839844,
      "epoch": 3.340966921119593,
      "grad_norm": 0.22564362087366385,
      "kl": 0.04010009765625,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.7879464775323868,
      "reward_std": 0.05759550258517265,
      "rewards/accuracy_reward": 0.7879464775323868,
      "step": 655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.0491256713867,
      "epoch": 3.346055979643766,
      "grad_norm": 0.28956084194362786,
      "kl": 0.039215087890625,
      "learning_rate": 2e-06,
      "loss": -0.0044,
      "reward": 0.7834821790456772,
      "reward_std": 0.0777943367138505,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.8772506713867,
      "epoch": 3.351145038167939,
      "grad_norm": 0.25403898238670936,
      "kl": 0.038360595703125,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.8169643133878708,
      "reward_std": 0.06450806744396687,
      "rewards/accuracy_reward": 0.8169643133878708,
      "step": 657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.0580596923828,
      "epoch": 3.356234096692112,
      "grad_norm": 0.27920868291769374,
      "kl": 0.04071044921875,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.7500000298023224,
      "reward_std": 0.0655801072716713,
      "rewards/accuracy_reward": 0.7500000298023224,
      "step": 658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.95314025878906,
      "epoch": 3.3613231552162848,
      "grad_norm": 0.1555965359034967,
      "kl": 0.0413818359375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.877232164144516,
      "reward_std": 0.04162358958274126,
      "rewards/accuracy_reward": 0.877232164144516,
      "step": 659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.2455596923828,
      "epoch": 3.366412213740458,
      "grad_norm": 0.282783654593396,
      "kl": 0.03851318359375,
      "learning_rate": 2e-06,
      "loss": -0.0047,
      "reward": 0.7187500298023224,
      "reward_std": 0.07644847221672535,
      "rewards/accuracy_reward": 0.7187500298023224,
      "step": 660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.3214416503906,
      "epoch": 3.371501272264631,
      "grad_norm": 0.3083253632524364,
      "kl": 0.044281005859375,
      "learning_rate": 2e-06,
      "loss": 0.0064,
      "reward": 0.7566964775323868,
      "reward_std": 0.1052577355876565,
      "rewards/accuracy_reward": 0.7566964775323868,
      "step": 661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.37501525878906,
      "epoch": 3.376590330788804,
      "grad_norm": 0.26469576861324823,
      "kl": 0.0411376953125,
      "learning_rate": 2e-06,
      "loss": 0.0113,
      "reward": 0.77901791036129,
      "reward_std": 0.03898446634411812,
      "rewards/accuracy_reward": 0.77901791036129,
      "step": 662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.6986846923828,
      "epoch": 3.381679389312977,
      "grad_norm": 0.2627735047617527,
      "kl": 0.04168701171875,
      "learning_rate": 2e-06,
      "loss": -0.0063,
      "reward": 0.7901785969734192,
      "reward_std": 0.0760837011039257,
      "rewards/accuracy_reward": 0.7901785969734192,
      "step": 663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.10939025878906,
      "epoch": 3.38676844783715,
      "grad_norm": 0.18452333139172977,
      "kl": 0.03875732421875,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.7232143133878708,
      "reward_std": 0.05419615097343922,
      "rewards/accuracy_reward": 0.7232143133878708,
      "step": 664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.3750228881836,
      "epoch": 3.391857506361323,
      "grad_norm": 0.2461374771338689,
      "kl": 0.03839111328125,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.8772321939468384,
      "reward_std": 0.05495638120919466,
      "rewards/accuracy_reward": 0.8772321939468384,
      "step": 665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.0558166503906,
      "epoch": 3.3969465648854964,
      "grad_norm": 0.2725249106011546,
      "kl": 0.03228759765625,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.6495536044239998,
      "reward_std": 0.05011117737740278,
      "rewards/accuracy_reward": 0.6495536044239998,
      "step": 666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.5000228881836,
      "epoch": 3.4020356234096694,
      "grad_norm": 0.3329227921759514,
      "kl": 0.037139892578125,
      "learning_rate": 2e-06,
      "loss": 0.0064,
      "reward": 0.6517857313156128,
      "reward_std": 0.13027985487133265,
      "rewards/accuracy_reward": 0.6517857313156128,
      "step": 667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.6138610839844,
      "epoch": 3.4071246819338423,
      "grad_norm": 0.17960047933009235,
      "kl": 0.04449462890625,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.8861607611179352,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.8861607611179352,
      "step": 668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.6808166503906,
      "epoch": 3.4122137404580153,
      "grad_norm": 0.28774083314625865,
      "kl": 0.03741455078125,
      "learning_rate": 2e-06,
      "loss": -0.0056,
      "reward": 0.8191964477300644,
      "reward_std": 0.08094278071075678,
      "rewards/accuracy_reward": 0.8191964477300644,
      "step": 669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.6428680419922,
      "epoch": 3.4173027989821882,
      "grad_norm": 0.1745721965526644,
      "kl": 0.035064697265625,
      "learning_rate": 2e-06,
      "loss": 0.0036,
      "reward": 0.7053571678698063,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.7053571678698063,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.78572845458984,
      "epoch": 3.422391857506361,
      "grad_norm": 0.3028643599835002,
      "kl": 0.03973388671875,
      "learning_rate": 2e-06,
      "loss": 0.0043,
      "reward": 0.7879464626312256,
      "reward_std": 0.0633819680660963,
      "rewards/accuracy_reward": 0.7879464626312256,
      "step": 671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.3616256713867,
      "epoch": 3.427480916030534,
      "grad_norm": 0.1785298830645011,
      "kl": 0.0374755859375,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.7254464477300644,
      "reward_std": 0.042757630348205566,
      "rewards/accuracy_reward": 0.7254464477300644,
      "step": 672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.73439025878906,
      "epoch": 3.432569974554707,
      "grad_norm": 0.1509571326952281,
      "kl": 0.03564453125,
      "learning_rate": 2e-06,
      "loss": 0.0046,
      "reward": 0.720982164144516,
      "reward_std": 0.035539694130420685,
      "rewards/accuracy_reward": 0.720982164144516,
      "step": 673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.36832427978516,
      "epoch": 3.4376590330788805,
      "grad_norm": 0.27893651104570416,
      "kl": 0.04473876953125,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.8437500298023224,
      "reward_std": 0.09173476230353117,
      "rewards/accuracy_reward": 0.8437500298023224,
      "step": 674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.1942138671875,
      "epoch": 3.4427480916030535,
      "grad_norm": 0.26786490204804564,
      "kl": 0.04034423828125,
      "learning_rate": 2e-06,
      "loss": -0.0028,
      "reward": 0.7142857611179352,
      "reward_std": 0.0712384982034564,
      "rewards/accuracy_reward": 0.7142857611179352,
      "step": 675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.5111770629883,
      "epoch": 3.4478371501272265,
      "grad_norm": 0.1557097339950374,
      "kl": 0.037353515625,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.7991071939468384,
      "reward_std": 0.036299921572208405,
      "rewards/accuracy_reward": 0.7991071939468384,
      "step": 676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.59153747558594,
      "epoch": 3.4529262086513994,
      "grad_norm": 0.22808365057294916,
      "kl": 0.038299560546875,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.7343750447034836,
      "reward_std": 0.05011117737740278,
      "rewards/accuracy_reward": 0.7343750447034836,
      "step": 677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.03797149658203,
      "epoch": 3.4580152671755724,
      "grad_norm": 0.13826976815213973,
      "kl": 0.038330078125,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.7142857536673546,
      "reward_std": 0.01785714365541935,
      "rewards/accuracy_reward": 0.7142857536673546,
      "step": 678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.4308166503906,
      "epoch": 3.4631043256997454,
      "grad_norm": 0.3178882181706591,
      "kl": 0.04876708984375,
      "learning_rate": 2e-06,
      "loss": 0.002,
      "reward": 0.8125000447034836,
      "reward_std": 0.05903974920511246,
      "rewards/accuracy_reward": 0.8125000447034836,
      "step": 679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.04689025878906,
      "epoch": 3.4681933842239188,
      "grad_norm": 0.19129672393618688,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.816964328289032,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.816964328289032,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.73438262939453,
      "epoch": 3.4732824427480917,
      "grad_norm": 0.29768289719047186,
      "kl": 0.0411376953125,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.7031250521540642,
      "reward_std": 0.05665153544396162,
      "rewards/accuracy_reward": 0.7031250521540642,
      "step": 681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.2768096923828,
      "epoch": 3.4783715012722647,
      "grad_norm": 0.194194929002816,
      "kl": 0.041748046875,
      "learning_rate": 2e-06,
      "loss": 0.011,
      "reward": 0.7410714626312256,
      "reward_std": 0.04118260461837053,
      "rewards/accuracy_reward": 0.7410714626312256,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.7076110839844,
      "epoch": 3.4834605597964376,
      "grad_norm": 0.23879471863159246,
      "kl": 0.0382080078125,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.6517857387661934,
      "reward_std": 0.07312643527984619,
      "rewards/accuracy_reward": 0.6517857387661934,
      "step": 683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 364.2433166503906,
      "epoch": 3.4885496183206106,
      "grad_norm": 0.21154424421464163,
      "kl": 0.04107666015625,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.7566964477300644,
      "reward_std": 0.04334326460957527,
      "rewards/accuracy_reward": 0.7566964477300644,
      "step": 684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 369.3303756713867,
      "epoch": 3.4936386768447836,
      "grad_norm": 0.30815543283372915,
      "kl": 0.0421142578125,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.7500000521540642,
      "reward_std": 0.10890679247677326,
      "rewards/accuracy_reward": 0.7500000521540642,
      "step": 685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.60269927978516,
      "epoch": 3.4987277353689565,
      "grad_norm": 0.26352389485066563,
      "kl": 0.033660888671875,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.8928571790456772,
      "reward_std": 0.07110932655632496,
      "rewards/accuracy_reward": 0.8928571790456772,
      "step": 686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.5937728881836,
      "epoch": 3.5038167938931295,
      "grad_norm": 0.24194299290176932,
      "kl": 0.04248046875,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.7477678805589676,
      "reward_std": 0.05459804926067591,
      "rewards/accuracy_reward": 0.7477678805589676,
      "step": 687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.6785888671875,
      "epoch": 3.508905852417303,
      "grad_norm": 0.18752950685020256,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.7165178880095482,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.7165178880095482,
      "step": 688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.85269927978516,
      "epoch": 3.513994910941476,
      "grad_norm": 0.24417620390632744,
      "kl": 0.04461669921875,
      "learning_rate": 2e-06,
      "loss": 0.0083,
      "reward": 0.861607164144516,
      "reward_std": 0.06701801717281342,
      "rewards/accuracy_reward": 0.861607164144516,
      "step": 689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 372.46876525878906,
      "epoch": 3.519083969465649,
      "grad_norm": 0.029484001824464336,
      "kl": 0.040771484375,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.7500000447034836,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.7500000447034836,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.6116256713867,
      "epoch": 3.524173027989822,
      "grad_norm": 0.2414633809952364,
      "kl": 0.03277587890625,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.9129464626312256,
      "reward_std": 0.0887854415923357,
      "rewards/accuracy_reward": 0.9129464626312256,
      "step": 691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.06475830078125,
      "epoch": 3.5292620865139948,
      "grad_norm": 0.11041258581578524,
      "kl": 0.037139892578125,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.7455357685685158,
      "reward_std": 0.012198750860989094,
      "rewards/accuracy_reward": 0.7455357685685158,
      "step": 692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.4754638671875,
      "epoch": 3.534351145038168,
      "grad_norm": 0.19616251845327218,
      "kl": 0.03424072265625,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.814732164144516,
      "reward_std": 0.03495405800640583,
      "rewards/accuracy_reward": 0.814732164144516,
      "step": 693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.10716247558594,
      "epoch": 3.539440203562341,
      "grad_norm": 0.24316588568548692,
      "kl": 0.0418701171875,
      "learning_rate": 2e-06,
      "loss": 0.006,
      "reward": 0.8125000447034836,
      "reward_std": 0.06369487755000591,
      "rewards/accuracy_reward": 0.8125000447034836,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.47100830078125,
      "epoch": 3.544529262086514,
      "grad_norm": 0.1773523052343864,
      "kl": 0.034271240234375,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.8370536267757416,
      "reward_std": 0.03615527041256428,
      "rewards/accuracy_reward": 0.8370536267757416,
      "step": 695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.3460006713867,
      "epoch": 3.549618320610687,
      "grad_norm": 0.2892963083247234,
      "kl": 0.033294677734375,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.6808036118745804,
      "reward_std": 0.08977331966161728,
      "rewards/accuracy_reward": 0.6808036118745804,
      "step": 696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.8683166503906,
      "epoch": 3.55470737913486,
      "grad_norm": 0.28702790607198414,
      "kl": 0.0399169921875,
      "learning_rate": 2e-06,
      "loss": -0.0075,
      "reward": 0.8191964626312256,
      "reward_std": 0.09946212265640497,
      "rewards/accuracy_reward": 0.8191964626312256,
      "step": 697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.49778747558594,
      "epoch": 3.559796437659033,
      "grad_norm": 0.2780939572367632,
      "kl": 0.0367431640625,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.698660746216774,
      "reward_std": 0.075452646240592,
      "rewards/accuracy_reward": 0.698660746216774,
      "step": 698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.7031478881836,
      "epoch": 3.564885496183206,
      "grad_norm": 0.18821460351310842,
      "kl": 0.0345458984375,
      "learning_rate": 2e-06,
      "loss": 0.0023,
      "reward": 0.7745535969734192,
      "reward_std": 0.043343265540897846,
      "rewards/accuracy_reward": 0.7745535969734192,
      "step": 699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.50001525878906,
      "epoch": 3.569974554707379,
      "grad_norm": 0.24953673790939787,
      "kl": 0.041778564453125,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.6071428880095482,
      "reward_std": 0.05715452041476965,
      "rewards/accuracy_reward": 0.6071428880095482,
      "step": 700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.5580520629883,
      "epoch": 3.5750636132315523,
      "grad_norm": 0.24754069376089746,
      "kl": 0.0350341796875,
      "learning_rate": 2e-06,
      "loss": -0.0042,
      "reward": 0.7343750298023224,
      "reward_std": 0.06836387887597084,
      "rewards/accuracy_reward": 0.7343750298023224,
      "step": 701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.0736770629883,
      "epoch": 3.5801526717557253,
      "grad_norm": 0.16666219668110593,
      "kl": 0.037353515625,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.7410714626312256,
      "reward_std": 0.026025486178696156,
      "rewards/accuracy_reward": 0.7410714626312256,
      "step": 702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.9643020629883,
      "epoch": 3.5852417302798982,
      "grad_norm": 0.29210630646822694,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": -0.004,
      "reward": 0.8459821939468384,
      "reward_std": 0.11624883487820625,
      "rewards/accuracy_reward": 0.8459821939468384,
      "step": 703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.2232360839844,
      "epoch": 3.590330788804071,
      "grad_norm": 0.22899866289339668,
      "kl": 0.042236328125,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.7901785969734192,
      "reward_std": 0.04118260461837053,
      "rewards/accuracy_reward": 0.7901785969734192,
      "step": 704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.30135345458984,
      "epoch": 3.595419847328244,
      "grad_norm": 0.1885142236786285,
      "kl": 0.0372314453125,
      "learning_rate": 2e-06,
      "loss": 0.0056,
      "reward": 0.7678571939468384,
      "reward_std": 0.04099252913147211,
      "rewards/accuracy_reward": 0.7678571939468384,
      "step": 705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.07366943359375,
      "epoch": 3.6005089058524176,
      "grad_norm": 0.2685729098856735,
      "kl": 0.04595947265625,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.6294643133878708,
      "reward_std": 0.06828013248741627,
      "rewards/accuracy_reward": 0.6294643133878708,
      "step": 706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.5736846923828,
      "epoch": 3.6055979643765905,
      "grad_norm": 0.2559042774165835,
      "kl": 0.036285400390625,
      "learning_rate": 2e-06,
      "loss": 0.006,
      "reward": 0.745535746216774,
      "reward_std": 0.0840228870511055,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.39733123779297,
      "epoch": 3.6106870229007635,
      "grad_norm": 0.22606297406685444,
      "kl": 0.038909912109375,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.7745536118745804,
      "reward_std": 0.06576384603977203,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.70314025878906,
      "epoch": 3.6157760814249365,
      "grad_norm": 0.19824773731654147,
      "kl": 0.036285400390625,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.8392857611179352,
      "reward_std": 0.04099252913147211,
      "rewards/accuracy_reward": 0.8392857611179352,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.9129638671875,
      "epoch": 3.6208651399491094,
      "grad_norm": 0.1983818756553478,
      "kl": 0.03997802734375,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.6138393133878708,
      "reward_std": 0.03225403279066086,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 365.7187728881836,
      "epoch": 3.6259541984732824,
      "grad_norm": 0.21770916986776495,
      "kl": 0.044189453125,
      "learning_rate": 2e-06,
      "loss": 0.0111,
      "reward": 0.785714328289032,
      "reward_std": 0.04980097245424986,
      "rewards/accuracy_reward": 0.785714328289032,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.2968978881836,
      "epoch": 3.6310432569974553,
      "grad_norm": 0.2510553792550636,
      "kl": 0.034759521484375,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.7031250298023224,
      "reward_std": 0.062119850888848305,
      "rewards/accuracy_reward": 0.7031250298023224,
      "step": 712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.61385345458984,
      "epoch": 3.6361323155216283,
      "grad_norm": 0.20506565001429988,
      "kl": 0.03497314453125,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.761160746216774,
      "reward_std": 0.05118321720510721,
      "rewards/accuracy_reward": 0.761160746216774,
      "step": 713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.6406478881836,
      "epoch": 3.6412213740458013,
      "grad_norm": 0.21185422242343718,
      "kl": 0.0382080078125,
      "learning_rate": 2e-06,
      "loss": 0.0099,
      "reward": 0.8035714626312256,
      "reward_std": 0.04508384130895138,
      "rewards/accuracy_reward": 0.8035714626312256,
      "step": 714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.17635345458984,
      "epoch": 3.6463104325699747,
      "grad_norm": 0.25698651382335774,
      "kl": 0.04144287109375,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.7276785969734192,
      "reward_std": 0.06635581515729427,
      "rewards/accuracy_reward": 0.7276785969734192,
      "step": 715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.41297149658203,
      "epoch": 3.6513994910941476,
      "grad_norm": 0.20627442533846088,
      "kl": 0.0361328125,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.7522321939468384,
      "reward_std": 0.05742724426090717,
      "rewards/accuracy_reward": 0.7522321939468384,
      "step": 716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.09376525878906,
      "epoch": 3.6564885496183206,
      "grad_norm": 0.27671482024942784,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.832589328289032,
      "reward_std": 0.10420895181596279,
      "rewards/accuracy_reward": 0.832589328289032,
      "step": 717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.9933166503906,
      "epoch": 3.6615776081424936,
      "grad_norm": 0.15101557170066765,
      "kl": 0.03948974609375,
      "learning_rate": 2e-06,
      "loss": 0.0058,
      "reward": 0.7924107313156128,
      "reward_std": 0.024900485761463642,
      "rewards/accuracy_reward": 0.7924107313156128,
      "step": 718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.79689025878906,
      "epoch": 3.6666666666666665,
      "grad_norm": 0.25862193506295234,
      "kl": 0.0391845703125,
      "learning_rate": 2e-06,
      "loss": 0.0061,
      "reward": 0.8013393133878708,
      "reward_std": 0.06872111465781927,
      "rewards/accuracy_reward": 0.8013393133878708,
      "step": 719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.7522506713867,
      "epoch": 3.67175572519084,
      "grad_norm": 0.260646792784835,
      "kl": 0.03924560546875,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.7879464626312256,
      "reward_std": 0.06715513207018375,
      "rewards/accuracy_reward": 0.7879464626312256,
      "step": 720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.3125228881836,
      "epoch": 3.676844783715013,
      "grad_norm": 0.28823526002549565,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0069,
      "reward": 0.8191964775323868,
      "reward_std": 0.1204148530960083,
      "rewards/accuracy_reward": 0.8191964775323868,
      "step": 721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.2120666503906,
      "epoch": 3.681933842239186,
      "grad_norm": 0.25419214465949347,
      "kl": 0.04248046875,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.6339285969734192,
      "reward_std": 0.05539736431092024,
      "rewards/accuracy_reward": 0.6339285969734192,
      "step": 722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.3504638671875,
      "epoch": 3.687022900763359,
      "grad_norm": 0.22372868706439392,
      "kl": 0.040069580078125,
      "learning_rate": 2e-06,
      "loss": 0.0,
      "reward": 0.8392857611179352,
      "reward_std": 0.048795003443956375,
      "rewards/accuracy_reward": 0.8392857611179352,
      "step": 723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.1473388671875,
      "epoch": 3.6921119592875318,
      "grad_norm": 0.2294228001152274,
      "kl": 0.03973388671875,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.7120535969734192,
      "reward_std": 0.06374783627688885,
      "rewards/accuracy_reward": 0.7120535969734192,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.93528747558594,
      "epoch": 3.6972010178117047,
      "grad_norm": 0.2442438691995658,
      "kl": 0.03082275390625,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.6718750298023224,
      "reward_std": 0.06312472466379404,
      "rewards/accuracy_reward": 0.6718750298023224,
      "step": 725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.18751525878906,
      "epoch": 3.7022900763358777,
      "grad_norm": 0.11866603878757535,
      "kl": 0.041107177734375,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.7477678805589676,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.7477678805589676,
      "step": 726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.5067138671875,
      "epoch": 3.7073791348600507,
      "grad_norm": 0.2954740207443915,
      "kl": 0.037200927734375,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.678571455180645,
      "reward_std": 0.09267873037606478,
      "rewards/accuracy_reward": 0.678571455180645,
      "step": 727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.91072845458984,
      "epoch": 3.712468193384224,
      "grad_norm": 0.3182081737785432,
      "kl": 0.0426025390625,
      "learning_rate": 2e-06,
      "loss": 0.0019,
      "reward": 0.7812500298023224,
      "reward_std": 0.11135067138820887,
      "rewards/accuracy_reward": 0.7812500298023224,
      "step": 728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.5468978881836,
      "epoch": 3.717557251908397,
      "grad_norm": 0.24677606658962203,
      "kl": 0.04034423828125,
      "learning_rate": 2e-06,
      "loss": -0.0097,
      "reward": 0.8035714626312256,
      "reward_std": 0.08318000379949808,
      "rewards/accuracy_reward": 0.8035714626312256,
      "step": 729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.0111770629883,
      "epoch": 3.72264631043257,
      "grad_norm": 0.2823707794307348,
      "kl": 0.03692626953125,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.761160746216774,
      "reward_std": 0.11374405398964882,
      "rewards/accuracy_reward": 0.761160746216774,
      "step": 730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.6361770629883,
      "epoch": 3.727735368956743,
      "grad_norm": 0.15913933249198417,
      "kl": 0.0328369140625,
      "learning_rate": 2e-06,
      "loss": -0.0032,
      "reward": 0.7879464477300644,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.7879464477300644,
      "step": 731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.1897506713867,
      "epoch": 3.732824427480916,
      "grad_norm": 0.19045907251634567,
      "kl": 0.033905029296875,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.6986607313156128,
      "reward_std": 0.05118321720510721,
      "rewards/accuracy_reward": 0.6986607313156128,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.0491256713867,
      "epoch": 3.7379134860050893,
      "grad_norm": 0.2388549367648875,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.8861607313156128,
      "reward_std": 0.05665153358131647,
      "rewards/accuracy_reward": 0.8861607313156128,
      "step": 733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.5759048461914,
      "epoch": 3.7430025445292623,
      "grad_norm": 0.20600297490821115,
      "kl": 0.036651611328125,
      "learning_rate": 2e-06,
      "loss": -0.0003,
      "reward": 0.7656250447034836,
      "reward_std": 0.06885028630495071,
      "rewards/accuracy_reward": 0.7656250447034836,
      "step": 734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.54466247558594,
      "epoch": 3.7480916030534353,
      "grad_norm": 0.29283524951866263,
      "kl": 0.039947509765625,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.6852678805589676,
      "reward_std": 0.06777824461460114,
      "rewards/accuracy_reward": 0.6852678805589676,
      "step": 735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.0826110839844,
      "epoch": 3.753180661577608,
      "grad_norm": 0.27184449037617886,
      "kl": 0.0440673828125,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.7901786267757416,
      "reward_std": 0.06105575617402792,
      "rewards/accuracy_reward": 0.7901786267757416,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.75894927978516,
      "epoch": 3.758269720101781,
      "grad_norm": 0.30749464852086206,
      "kl": 0.037384033203125,
      "learning_rate": 2e-06,
      "loss": -0.0028,
      "reward": 0.8549107611179352,
      "reward_std": 0.10424642264842987,
      "rewards/accuracy_reward": 0.8549107611179352,
      "step": 737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.25447845458984,
      "epoch": 3.763358778625954,
      "grad_norm": 0.2028011950869312,
      "kl": 0.0340576171875,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.676339328289032,
      "reward_std": 0.05557949375361204,
      "rewards/accuracy_reward": 0.676339328289032,
      "step": 738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.3236770629883,
      "epoch": 3.768447837150127,
      "grad_norm": 0.6844046164923124,
      "kl": 0.04296875,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.7901786118745804,
      "reward_std": 0.05227183550596237,
      "rewards/accuracy_reward": 0.7901786118745804,
      "step": 739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.92858123779297,
      "epoch": 3.7735368956743,
      "grad_norm": 0.24123348419814913,
      "kl": 0.033660888671875,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.7723214775323868,
      "reward_std": 0.06262173876166344,
      "rewards/accuracy_reward": 0.7723214775323868,
      "step": 740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.2143020629883,
      "epoch": 3.778625954198473,
      "grad_norm": 0.3028445418192028,
      "kl": 0.0379638671875,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.7209821790456772,
      "reward_std": 0.09496746677905321,
      "rewards/accuracy_reward": 0.7209821790456772,
      "step": 741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.9799270629883,
      "epoch": 3.7837150127226464,
      "grad_norm": 0.2292052849231483,
      "kl": 0.03533935546875,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.854910746216774,
      "reward_std": 0.07339916005730629,
      "rewards/accuracy_reward": 0.854910746216774,
      "step": 742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 372.6451110839844,
      "epoch": 3.7888040712468194,
      "grad_norm": 0.4054198167074397,
      "kl": 0.04443359375,
      "learning_rate": 2e-06,
      "loss": -0.0056,
      "reward": 0.801339328289032,
      "reward_std": 0.15239506028592587,
      "rewards/accuracy_reward": 0.801339328289032,
      "step": 743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.02457427978516,
      "epoch": 3.7938931297709924,
      "grad_norm": 0.22291741053205125,
      "kl": 0.03350830078125,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.7700893133878708,
      "reward_std": 0.07167948130518198,
      "rewards/accuracy_reward": 0.7700893133878708,
      "step": 744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.22322845458984,
      "epoch": 3.7989821882951653,
      "grad_norm": 0.18747436838696635,
      "kl": 0.040924072265625,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.7834821715950966,
      "reward_std": 0.04526757914572954,
      "rewards/accuracy_reward": 0.7834821715950966,
      "step": 745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.70091247558594,
      "epoch": 3.8040712468193383,
      "grad_norm": 0.24896140649220513,
      "kl": 0.03607177734375,
      "learning_rate": 2e-06,
      "loss": 0.0023,
      "reward": 0.6473214626312256,
      "reward_std": 0.07752161100506783,
      "rewards/accuracy_reward": 0.6473214626312256,
      "step": 746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.6741180419922,
      "epoch": 3.8091603053435117,
      "grad_norm": 0.31022142257006696,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.6696428954601288,
      "reward_std": 0.09833602514117956,
      "rewards/accuracy_reward": 0.6696428954601288,
      "step": 747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.4174270629883,
      "epoch": 3.8142493638676847,
      "grad_norm": 0.2913253699428815,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": -0.0109,
      "reward": 0.642857164144516,
      "reward_std": 0.1200955081731081,
      "rewards/accuracy_reward": 0.642857164144516,
      "step": 748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.7902069091797,
      "epoch": 3.8193384223918576,
      "grad_norm": 0.3712597157276719,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.665178619325161,
      "reward_std": 0.16140617989003658,
      "rewards/accuracy_reward": 0.665178619325161,
      "step": 749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.03126525878906,
      "epoch": 3.8244274809160306,
      "grad_norm": 0.27093902520037944,
      "kl": 0.04632568359375,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.6919643133878708,
      "reward_std": 0.09154469147324562,
      "rewards/accuracy_reward": 0.6919643133878708,
      "step": 750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.2276916503906,
      "epoch": 3.8295165394402035,
      "grad_norm": 0.29686546192911756,
      "kl": 0.033721923828125,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.7008928805589676,
      "reward_std": 0.14212027192115784,
      "rewards/accuracy_reward": 0.7008928805589676,
      "step": 751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.30582427978516,
      "epoch": 3.8346055979643765,
      "grad_norm": 0.22085004970342131,
      "kl": 0.03033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.7120535969734192,
      "reward_std": 0.06885028723627329,
      "rewards/accuracy_reward": 0.7120535969734192,
      "step": 752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.79466247558594,
      "epoch": 3.8396946564885495,
      "grad_norm": 0.43788533520067846,
      "kl": 0.05401611328125,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.7477678954601288,
      "reward_std": 0.09613898769021034,
      "rewards/accuracy_reward": 0.7477678954601288,
      "step": 753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.87278747558594,
      "epoch": 3.8447837150127224,
      "grad_norm": 0.35542519471917566,
      "kl": 0.03582763671875,
      "learning_rate": 2e-06,
      "loss": -0.0039,
      "reward": 0.5937500223517418,
      "reward_std": 0.13493337389081717,
      "rewards/accuracy_reward": 0.5937500223517418,
      "step": 754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.89733123779297,
      "epoch": 3.849872773536896,
      "grad_norm": 0.3380151640344136,
      "kl": 0.04644775390625,
      "learning_rate": 2e-06,
      "loss": 0.0155,
      "reward": 0.7098214477300644,
      "reward_std": 0.11691068299114704,
      "rewards/accuracy_reward": 0.7098214477300644,
      "step": 755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.06697845458984,
      "epoch": 3.854961832061069,
      "grad_norm": 0.2347734539806386,
      "kl": 0.034698486328125,
      "learning_rate": 2e-06,
      "loss": 0.0039,
      "reward": 0.8258928656578064,
      "reward_std": 0.044468265026807785,
      "rewards/accuracy_reward": 0.8258928656578064,
      "step": 756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.1830520629883,
      "epoch": 3.8600508905852418,
      "grad_norm": 0.24185712017685304,
      "kl": 0.03448486328125,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.7165178954601288,
      "reward_std": 0.07425143383443356,
      "rewards/accuracy_reward": 0.7165178954601288,
      "step": 757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.4910888671875,
      "epoch": 3.8651399491094147,
      "grad_norm": 0.2951963968051688,
      "kl": 0.033416748046875,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.7500000298023224,
      "reward_std": 0.09267763141542673,
      "rewards/accuracy_reward": 0.7500000298023224,
      "step": 758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.8504638671875,
      "epoch": 3.8702290076335877,
      "grad_norm": 0.18748248021725178,
      "kl": 0.032257080078125,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.7276785969734192,
      "reward_std": 0.05162420216947794,
      "rewards/accuracy_reward": 0.7276785969734192,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.2143096923828,
      "epoch": 3.875318066157761,
      "grad_norm": 0.29791662797435103,
      "kl": 0.041748046875,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.6808036044239998,
      "reward_std": 0.06855395622551441,
      "rewards/accuracy_reward": 0.6808036044239998,
      "step": 760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 373.83707427978516,
      "epoch": 3.880407124681934,
      "grad_norm": 0.26490003174417803,
      "kl": 0.035186767578125,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.776785746216774,
      "reward_std": 0.04791303817182779,
      "rewards/accuracy_reward": 0.776785746216774,
      "step": 761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.9643020629883,
      "epoch": 3.885496183206107,
      "grad_norm": 0.30981060200734195,
      "kl": 0.0364990234375,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.8125000447034836,
      "reward_std": 0.10770558193325996,
      "rewards/accuracy_reward": 0.8125000447034836,
      "step": 762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.44197845458984,
      "epoch": 3.89058524173028,
      "grad_norm": 0.38912043595099843,
      "kl": 0.037353515625,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.6875000298023224,
      "reward_std": 0.2025979347527027,
      "rewards/accuracy_reward": 0.6875000298023224,
      "step": 763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.7433166503906,
      "epoch": 3.895674300254453,
      "grad_norm": 0.2557629441553287,
      "kl": 0.033203125,
      "learning_rate": 2e-06,
      "loss": -0.0106,
      "reward": 0.6718750298023224,
      "reward_std": 0.07469875458627939,
      "rewards/accuracy_reward": 0.6718750298023224,
      "step": 764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.25001525878906,
      "epoch": 3.900763358778626,
      "grad_norm": 0.30197159689609787,
      "kl": 0.0399169921875,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.8102678805589676,
      "reward_std": 0.11627852451056242,
      "rewards/accuracy_reward": 0.8102678805589676,
      "step": 765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.06250762939453,
      "epoch": 3.905852417302799,
      "grad_norm": 0.254742125961602,
      "kl": 0.034332275390625,
      "learning_rate": 2e-06,
      "loss": 0.0058,
      "reward": 0.709821455180645,
      "reward_std": 0.09593360032886267,
      "rewards/accuracy_reward": 0.709821455180645,
      "step": 766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.77903747558594,
      "epoch": 3.910941475826972,
      "grad_norm": 0.47368647427928606,
      "kl": 0.039154052734375,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.6183036118745804,
      "reward_std": 0.10362331382930279,
      "rewards/accuracy_reward": 0.6183036118745804,
      "step": 767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.5803756713867,
      "epoch": 3.916030534351145,
      "grad_norm": 0.3275919698798669,
      "kl": 0.047607421875,
      "learning_rate": 2e-06,
      "loss": -0.0006,
      "reward": 0.6852678880095482,
      "reward_std": 0.09751489572227001,
      "rewards/accuracy_reward": 0.6852678880095482,
      "step": 768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.2076110839844,
      "epoch": 3.921119592875318,
      "grad_norm": 0.29223166880501233,
      "kl": 0.03411865234375,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.7946428805589676,
      "reward_std": 0.06948134396225214,
      "rewards/accuracy_reward": 0.7946428805589676,
      "step": 769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.52457427978516,
      "epoch": 3.926208651399491,
      "grad_norm": 0.25396921114788445,
      "kl": 0.03924560546875,
      "learning_rate": 2e-06,
      "loss": -0.0082,
      "reward": 0.7321428954601288,
      "reward_std": 0.09469474013894796,
      "rewards/accuracy_reward": 0.7321428954601288,
      "step": 770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.8549270629883,
      "epoch": 3.931297709923664,
      "grad_norm": 0.31300292089138565,
      "kl": 0.0333251953125,
      "learning_rate": 2e-06,
      "loss": -0.0027,
      "reward": 0.705357164144516,
      "reward_std": 0.13857736438512802,
      "rewards/accuracy_reward": 0.705357164144516,
      "step": 771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.55135345458984,
      "epoch": 3.936386768447837,
      "grad_norm": 0.2058160494381228,
      "kl": 0.03912353515625,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.7075893133878708,
      "reward_std": 0.07281352579593658,
      "rewards/accuracy_reward": 0.7075893133878708,
      "step": 772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.27903747558594,
      "epoch": 3.94147582697201,
      "grad_norm": 0.3067818237722658,
      "kl": 0.042236328125,
      "learning_rate": 2e-06,
      "loss": -0.0017,
      "reward": 0.8325893431901932,
      "reward_std": 0.08645018190145493,
      "rewards/accuracy_reward": 0.8325893431901932,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.6205520629883,
      "epoch": 3.9465648854961835,
      "grad_norm": 0.16636432307267573,
      "kl": 0.0379638671875,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.808035746216774,
      "reward_std": 0.04602780845016241,
      "rewards/accuracy_reward": 0.808035746216774,
      "step": 774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.9241180419922,
      "epoch": 3.9516539440203564,
      "grad_norm": 0.33787964544248683,
      "kl": 0.03533935546875,
      "learning_rate": 2e-06,
      "loss": -0.0037,
      "reward": 0.7812500447034836,
      "reward_std": 0.1018800288438797,
      "rewards/accuracy_reward": 0.7812500447034836,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.5870819091797,
      "epoch": 3.9567430025445294,
      "grad_norm": 0.3350431149895565,
      "kl": 0.0416259765625,
      "learning_rate": 2e-06,
      "loss": -0.0054,
      "reward": 0.7187500298023224,
      "reward_std": 0.09280680492520332,
      "rewards/accuracy_reward": 0.7187500298023224,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.7388610839844,
      "epoch": 3.9618320610687023,
      "grad_norm": 0.24551802529542818,
      "kl": 0.03851318359375,
      "learning_rate": 2e-06,
      "loss": 0.0063,
      "reward": 0.6339285895228386,
      "reward_std": 0.07042531110346317,
      "rewards/accuracy_reward": 0.6339285895228386,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.67859649658203,
      "epoch": 3.9669211195928753,
      "grad_norm": 0.24809340170982908,
      "kl": 0.035614013671875,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.6651785969734192,
      "reward_std": 0.09752393886446953,
      "rewards/accuracy_reward": 0.6651785969734192,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.8526916503906,
      "epoch": 3.9720101781170483,
      "grad_norm": 0.23088066896309153,
      "kl": 0.032989501953125,
      "learning_rate": 2e-06,
      "loss": 0.007,
      "reward": 0.7433035969734192,
      "reward_std": 0.0267857164144516,
      "rewards/accuracy_reward": 0.7433035969734192,
      "step": 779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.15179443359375,
      "epoch": 3.9770992366412212,
      "grad_norm": 0.3047130584118383,
      "kl": 0.0352783203125,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.808035746216774,
      "reward_std": 0.09788870345801115,
      "rewards/accuracy_reward": 0.808035746216774,
      "step": 780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.7366256713867,
      "epoch": 3.982188295165394,
      "grad_norm": 0.17604449597052646,
      "kl": 0.04547119140625,
      "learning_rate": 2e-06,
      "loss": -0.0046,
      "reward": 0.707589328289032,
      "reward_std": 0.05684160999953747,
      "rewards/accuracy_reward": 0.707589328289032,
      "step": 781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 443.91297149658203,
      "epoch": 3.9872773536895676,
      "grad_norm": 0.2775453914832509,
      "kl": 0.036407470703125,
      "learning_rate": 2e-06,
      "loss": 0.0085,
      "reward": 0.7388393133878708,
      "reward_std": 0.09757689572870731,
      "rewards/accuracy_reward": 0.7388393133878708,
      "step": 782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 433.3080596923828,
      "epoch": 3.9923664122137406,
      "grad_norm": 0.1940226995371348,
      "kl": 0.031036376953125,
      "learning_rate": 2e-06,
      "loss": 0.0048,
      "reward": 0.8281250298023224,
      "reward_std": 0.029295665211975574,
      "rewards/accuracy_reward": 0.8281250298023224,
      "step": 783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 494.0384750366211,
      "epoch": 3.9974554707379135,
      "grad_norm": 0.7250192035802584,
      "kl": 0.04022216796875,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.7544643133878708,
      "reward_std": 0.12348742038011551,
      "rewards/accuracy_reward": 0.7544643133878708,
      "step": 784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.74779510498047,
      "epoch": 4.005089058524173,
      "grad_norm": 2.811524157992115,
      "kl": 0.28778076171875,
      "learning_rate": 2e-06,
      "loss": 0.0204,
      "reward": 0.8080357760190964,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.8080357760190964,
      "step": 785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.4241256713867,
      "epoch": 4.010178117048346,
      "grad_norm": 0.1993181181940805,
      "kl": 0.033721923828125,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.832589328289032,
      "reward_std": 0.034954057075083256,
      "rewards/accuracy_reward": 0.832589328289032,
      "step": 786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.6741180419922,
      "epoch": 4.015267175572519,
      "grad_norm": 0.2417088182399602,
      "kl": 0.03570556640625,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.745535746216774,
      "reward_std": 0.04388263076543808,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.4799270629883,
      "epoch": 4.020356234096692,
      "grad_norm": 0.2548614348529178,
      "kl": 0.0400390625,
      "learning_rate": 2e-06,
      "loss": 0.0073,
      "reward": 0.7901786118745804,
      "reward_std": 0.0794068444520235,
      "rewards/accuracy_reward": 0.7901786118745804,
      "step": 788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.59376525878906,
      "epoch": 4.025445292620865,
      "grad_norm": 0.2434582564987702,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": 0.005,
      "reward": 0.7767857611179352,
      "reward_std": 0.04225464537739754,
      "rewards/accuracy_reward": 0.7767857611179352,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.1607360839844,
      "epoch": 4.030534351145038,
      "grad_norm": 0.13928089190456783,
      "kl": 0.029876708984375,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.7098214626312256,
      "reward_std": 0.01785714365541935,
      "rewards/accuracy_reward": 0.7098214626312256,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.5044860839844,
      "epoch": 4.035623409669212,
      "grad_norm": 0.16853906355765638,
      "kl": 0.035400390625,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.7790178954601288,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.7790178954601288,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.5067138671875,
      "epoch": 4.040712468193385,
      "grad_norm": 0.2646094800706115,
      "kl": 0.0355224609375,
      "learning_rate": 2e-06,
      "loss": 0.006,
      "reward": 0.8147321790456772,
      "reward_std": 0.08098186831921339,
      "rewards/accuracy_reward": 0.8147321790456772,
      "step": 792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.49778747558594,
      "epoch": 4.0458015267175576,
      "grad_norm": 0.6341440366984267,
      "kl": 0.069244384765625,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.9218750447034836,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.9218750447034836,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.3616256713867,
      "epoch": 4.0508905852417305,
      "grad_norm": 0.2979099724041331,
      "kl": 0.03765869140625,
      "learning_rate": 2e-06,
      "loss": 0.009,
      "reward": 0.6897321790456772,
      "reward_std": 0.08544531557708979,
      "rewards/accuracy_reward": 0.6897321790456772,
      "step": 794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.60047149658203,
      "epoch": 4.0559796437659035,
      "grad_norm": 0.19214655493721194,
      "kl": 0.04290771484375,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.8058035969734192,
      "reward_std": 0.05118321813642979,
      "rewards/accuracy_reward": 0.8058035969734192,
      "step": 795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.7254638671875,
      "epoch": 4.061068702290076,
      "grad_norm": 0.3162787306031356,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": -0.0002,
      "reward": 0.8683036267757416,
      "reward_std": 0.09048059489578009,
      "rewards/accuracy_reward": 0.8683036267757416,
      "step": 796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.0848388671875,
      "epoch": 4.066157760814249,
      "grad_norm": 0.2111638145807613,
      "kl": 0.032012939453125,
      "learning_rate": 2e-06,
      "loss": 0.0096,
      "reward": 0.6852678991854191,
      "reward_std": 0.03495405800640583,
      "rewards/accuracy_reward": 0.6852678991854191,
      "step": 797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.00670623779297,
      "epoch": 4.071246819338422,
      "grad_norm": 0.1517608878738625,
      "kl": 0.031402587890625,
      "learning_rate": 2e-06,
      "loss": -0.0061,
      "reward": 0.7767857760190964,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.7767857760190964,
      "step": 798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.3393096923828,
      "epoch": 4.076335877862595,
      "grad_norm": 0.3384293100098083,
      "kl": 0.03985595703125,
      "learning_rate": 2e-06,
      "loss": 0.0072,
      "reward": 0.622767873108387,
      "reward_std": 0.119829211384058,
      "rewards/accuracy_reward": 0.622767873108387,
      "step": 799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 366.1942138671875,
      "epoch": 4.081424936386768,
      "grad_norm": 0.2651538321305464,
      "kl": 0.0443115234375,
      "learning_rate": 2e-06,
      "loss": 0.0115,
      "reward": 0.8727678805589676,
      "reward_std": 0.04715281073004007,
      "rewards/accuracy_reward": 0.8727678805589676,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.8928680419922,
      "epoch": 4.086513994910941,
      "grad_norm": 0.30472371616923044,
      "kl": 0.03515625,
      "learning_rate": 2e-06,
      "loss": 0.0055,
      "reward": 0.8147321790456772,
      "reward_std": 0.0788674782961607,
      "rewards/accuracy_reward": 0.8147321790456772,
      "step": 801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.50672149658203,
      "epoch": 4.091603053435114,
      "grad_norm": 0.3421851714183259,
      "kl": 0.0428466796875,
      "learning_rate": 2e-06,
      "loss": 0.0053,
      "reward": 0.6986607536673546,
      "reward_std": 0.10820856783539057,
      "rewards/accuracy_reward": 0.6986607536673546,
      "step": 802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.34600830078125,
      "epoch": 4.096692111959287,
      "grad_norm": 0.17098161714820015,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.7834821790456772,
      "reward_std": 0.048225948587059975,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.76341247558594,
      "epoch": 4.10178117048346,
      "grad_norm": 0.22814406325744335,
      "kl": 0.033172607421875,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.7031250298023224,
      "reward_std": 0.07369549013674259,
      "rewards/accuracy_reward": 0.7031250298023224,
      "step": 804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.1116256713867,
      "epoch": 4.106870229007634,
      "grad_norm": 0.19860233997921517,
      "kl": 0.04412841796875,
      "learning_rate": 2e-06,
      "loss": -0.0017,
      "reward": 0.607142873108387,
      "reward_std": 0.03552421182394028,
      "rewards/accuracy_reward": 0.607142873108387,
      "step": 805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.5602798461914,
      "epoch": 4.111959287531807,
      "grad_norm": 0.19275843985251473,
      "kl": 0.04022216796875,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.8437500298023224,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.8437500298023224,
      "step": 806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.0826110839844,
      "epoch": 4.11704834605598,
      "grad_norm": 0.250704841682075,
      "kl": 0.04388427734375,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.8660714775323868,
      "reward_std": 0.04388262890279293,
      "rewards/accuracy_reward": 0.8660714775323868,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.1875228881836,
      "epoch": 4.122137404580153,
      "grad_norm": 0.16979984097662357,
      "kl": 0.03594970703125,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.7433036118745804,
      "reward_std": 0.037099236622452736,
      "rewards/accuracy_reward": 0.7433036118745804,
      "step": 808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.0803756713867,
      "epoch": 4.127226463104326,
      "grad_norm": 0.2455151126214259,
      "kl": 0.03570556640625,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.8080357611179352,
      "reward_std": 0.07739244028925896,
      "rewards/accuracy_reward": 0.8080357611179352,
      "step": 809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.7835006713867,
      "epoch": 4.132315521628499,
      "grad_norm": 0.21382550164800357,
      "kl": 0.0396728515625,
      "learning_rate": 2e-06,
      "loss": 0.006,
      "reward": 0.8526786118745804,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.8526786118745804,
      "step": 810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.0201110839844,
      "epoch": 4.137404580152672,
      "grad_norm": 0.19735525152233377,
      "kl": 0.0390625,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.5937500298023224,
      "reward_std": 0.03382905758917332,
      "rewards/accuracy_reward": 0.5937500298023224,
      "step": 811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.32814025878906,
      "epoch": 4.142493638676845,
      "grad_norm": 0.13802141378939065,
      "kl": 0.0361328125,
      "learning_rate": 2e-06,
      "loss": 0.0032,
      "reward": 0.7790178954601288,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.7790178954601288,
      "step": 812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.1250228881836,
      "epoch": 4.147582697201018,
      "grad_norm": 0.2310794642211151,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.6651786118745804,
      "reward_std": 0.046027809381484985,
      "rewards/accuracy_reward": 0.6651786118745804,
      "step": 813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.4710006713867,
      "epoch": 4.152671755725191,
      "grad_norm": 0.3053357942012366,
      "kl": 0.037384033203125,
      "learning_rate": 2e-06,
      "loss": 0.0185,
      "reward": 0.7254464477300644,
      "reward_std": 0.0650099515914917,
      "rewards/accuracy_reward": 0.7254464477300644,
      "step": 814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.62500762939453,
      "epoch": 4.157760814249364,
      "grad_norm": 0.21865734509827353,
      "kl": 0.038818359375,
      "learning_rate": 2e-06,
      "loss": 0.003,
      "reward": 0.8437500447034836,
      "reward_std": 0.04225464537739754,
      "rewards/accuracy_reward": 0.8437500447034836,
      "step": 815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.86385345458984,
      "epoch": 4.162849872773537,
      "grad_norm": 0.24179231776701612,
      "kl": 0.03814697265625,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.6651785969734192,
      "reward_std": 0.061055753380060196,
      "rewards/accuracy_reward": 0.6651785969734192,
      "step": 816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.7053680419922,
      "epoch": 4.1679389312977095,
      "grad_norm": 0.19843637589348567,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.6919643133878708,
      "reward_std": 0.04199740011245012,
      "rewards/accuracy_reward": 0.6919643133878708,
      "step": 817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.93751525878906,
      "epoch": 4.1730279898218825,
      "grad_norm": 0.17704102142661893,
      "kl": 0.035675048828125,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.6004464626312256,
      "reward_std": 0.04634071886539459,
      "rewards/accuracy_reward": 0.6004464626312256,
      "step": 818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.5915298461914,
      "epoch": 4.178117048346056,
      "grad_norm": 0.09242972303542982,
      "kl": 0.0428466796875,
      "learning_rate": 2e-06,
      "loss": -0.0026,
      "reward": 0.8526786118745804,
      "reward_std": 0.012198750860989094,
      "rewards/accuracy_reward": 0.8526786118745804,
      "step": 819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.33260345458984,
      "epoch": 4.183206106870229,
      "grad_norm": 0.14372963375721587,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": -0.0027,
      "reward": 0.7566964477300644,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.7566964477300644,
      "step": 820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.9643096923828,
      "epoch": 4.188295165394402,
      "grad_norm": 0.2023098467286473,
      "kl": 0.036529541015625,
      "learning_rate": 2e-06,
      "loss": -0.0014,
      "reward": 0.8660714477300644,
      "reward_std": 0.04602780845016241,
      "rewards/accuracy_reward": 0.8660714477300644,
      "step": 821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 377.4553756713867,
      "epoch": 4.193384223918575,
      "grad_norm": 0.3206684638226072,
      "kl": 0.0447998046875,
      "learning_rate": 2e-06,
      "loss": -0.0134,
      "reward": 0.7700893133878708,
      "reward_std": 0.11417082417756319,
      "rewards/accuracy_reward": 0.7700893133878708,
      "step": 822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.52457427978516,
      "epoch": 4.198473282442748,
      "grad_norm": 0.32984764088242763,
      "kl": 0.042755126953125,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.7946428954601288,
      "reward_std": 0.06935327127575874,
      "rewards/accuracy_reward": 0.7946428954601288,
      "step": 823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 368.04466247558594,
      "epoch": 4.203562340966921,
      "grad_norm": 0.22933184868143322,
      "kl": 0.03912353515625,
      "learning_rate": 2e-06,
      "loss": 0.0132,
      "reward": 0.7544643133878708,
      "reward_std": 0.07796893268823624,
      "rewards/accuracy_reward": 0.7544643133878708,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.67189025878906,
      "epoch": 4.208651399491094,
      "grad_norm": 0.12470572414557056,
      "kl": 0.0421142578125,
      "learning_rate": 2e-06,
      "loss": 0.0007,
      "reward": 0.854910746216774,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.854910746216774,
      "step": 825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.5960006713867,
      "epoch": 4.213740458015267,
      "grad_norm": 0.17091613965943417,
      "kl": 0.035675048828125,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.8303571790456772,
      "reward_std": 0.026025486178696156,
      "rewards/accuracy_reward": 0.8303571790456772,
      "step": 826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.8973388671875,
      "epoch": 4.21882951653944,
      "grad_norm": 0.20923767153497144,
      "kl": 0.0361328125,
      "learning_rate": 2e-06,
      "loss": 0.0039,
      "reward": 0.8995535969734192,
      "reward_std": 0.03898446634411812,
      "rewards/accuracy_reward": 0.8995535969734192,
      "step": 827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.24108123779297,
      "epoch": 4.223918575063613,
      "grad_norm": 0.2808827694307612,
      "kl": 0.03924560546875,
      "learning_rate": 2e-06,
      "loss": -0.0028,
      "reward": 0.9062500298023224,
      "reward_std": 0.0960018690675497,
      "rewards/accuracy_reward": 0.9062500298023224,
      "step": 828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.57814025878906,
      "epoch": 4.229007633587786,
      "grad_norm": 0.15295648452446617,
      "kl": 0.038818359375,
      "learning_rate": 2e-06,
      "loss": 0.0068,
      "reward": 0.7633928954601288,
      "reward_std": 0.049800973385572433,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.3348388671875,
      "epoch": 4.234096692111959,
      "grad_norm": 0.16441625147372738,
      "kl": 0.03814697265625,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.854910746216774,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.854910746216774,
      "step": 830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.2053756713867,
      "epoch": 4.239185750636132,
      "grad_norm": 0.17638098777267436,
      "kl": 0.036163330078125,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.8080357611179352,
      "reward_std": 0.03552421275526285,
      "rewards/accuracy_reward": 0.8080357611179352,
      "step": 831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.2567138671875,
      "epoch": 4.244274809160306,
      "grad_norm": 0.20975447502420955,
      "kl": 0.03900146484375,
      "learning_rate": 2e-06,
      "loss": 0.0004,
      "reward": 0.7053571790456772,
      "reward_std": 0.05985454376786947,
      "rewards/accuracy_reward": 0.7053571790456772,
      "step": 832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.8750228881836,
      "epoch": 4.249363867684479,
      "grad_norm": 0.3182054604354566,
      "kl": 0.04150390625,
      "learning_rate": 2e-06,
      "loss": 0.0093,
      "reward": 0.7120535969734192,
      "reward_std": 0.09380373358726501,
      "rewards/accuracy_reward": 0.7120535969734192,
      "step": 833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.20091247558594,
      "epoch": 4.254452926208652,
      "grad_norm": 0.025211036575571052,
      "kl": 0.033203125,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.7857143133878708,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.7857143133878708,
      "step": 834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.10939025878906,
      "epoch": 4.259541984732825,
      "grad_norm": 0.2525774392439054,
      "kl": 0.043548583984375,
      "learning_rate": 2e-06,
      "loss": 0.0032,
      "reward": 0.714285746216774,
      "reward_std": 0.0814228504896164,
      "rewards/accuracy_reward": 0.714285746216774,
      "step": 835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.1272506713867,
      "epoch": 4.264631043256998,
      "grad_norm": 0.16986818196488973,
      "kl": 0.03411865234375,
      "learning_rate": 2e-06,
      "loss": -0.003,
      "reward": 0.7031250298023224,
      "reward_std": 0.03495405800640583,
      "rewards/accuracy_reward": 0.7031250298023224,
      "step": 836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.1071548461914,
      "epoch": 4.269720101781171,
      "grad_norm": 0.22412542114448175,
      "kl": 0.04412841796875,
      "learning_rate": 2e-06,
      "loss": 0.0023,
      "reward": 0.8125000298023224,
      "reward_std": 0.04118260461837053,
      "rewards/accuracy_reward": 0.8125000298023224,
      "step": 837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.81920623779297,
      "epoch": 4.2748091603053435,
      "grad_norm": 0.2180672815383536,
      "kl": 0.04071044921875,
      "learning_rate": 2e-06,
      "loss": 0.0004,
      "reward": 0.7812500298023224,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.7812500298023224,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 376.39957427978516,
      "epoch": 4.2798982188295165,
      "grad_norm": 0.20833110755586887,
      "kl": 0.044189453125,
      "learning_rate": 2e-06,
      "loss": -0.0027,
      "reward": 0.808035746216774,
      "reward_std": 0.04118260554969311,
      "rewards/accuracy_reward": 0.808035746216774,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.4866256713867,
      "epoch": 4.2849872773536894,
      "grad_norm": 0.234711234213088,
      "kl": 0.03778076171875,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.848214328289032,
      "reward_std": 0.0642497232183814,
      "rewards/accuracy_reward": 0.848214328289032,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.42189025878906,
      "epoch": 4.290076335877862,
      "grad_norm": 0.24843309609148229,
      "kl": 0.0455322265625,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.7276785969734192,
      "reward_std": 0.05728259216994047,
      "rewards/accuracy_reward": 0.7276785969734192,
      "step": 841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.5089416503906,
      "epoch": 4.295165394402035,
      "grad_norm": 0.21050577592264016,
      "kl": 0.039642333984375,
      "learning_rate": 2e-06,
      "loss": 0.0138,
      "reward": 0.7098214477300644,
      "reward_std": 0.04980097245424986,
      "rewards/accuracy_reward": 0.7098214477300644,
      "step": 842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.1049270629883,
      "epoch": 4.300254452926208,
      "grad_norm": 0.18409441357947398,
      "kl": 0.03839111328125,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.8125000447034836,
      "reward_std": 0.023325460962951183,
      "rewards/accuracy_reward": 0.8125000447034836,
      "step": 843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.07814025878906,
      "epoch": 4.305343511450381,
      "grad_norm": 0.1467437806973369,
      "kl": 0.0455322265625,
      "learning_rate": 2e-06,
      "loss": 0.0024,
      "reward": 0.7767857611179352,
      "reward_std": 0.023325460962951183,
      "rewards/accuracy_reward": 0.7767857611179352,
      "step": 844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.3147506713867,
      "epoch": 4.310432569974555,
      "grad_norm": 0.29829936191823747,
      "kl": 0.04718017578125,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.863839328289032,
      "reward_std": 0.07909663487225771,
      "rewards/accuracy_reward": 0.863839328289032,
      "step": 845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.1138610839844,
      "epoch": 4.315521628498728,
      "grad_norm": 0.23707148778083859,
      "kl": 0.04046630859375,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.7857143133878708,
      "reward_std": 0.05539736431092024,
      "rewards/accuracy_reward": 0.7857143133878708,
      "step": 846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.2835006713867,
      "epoch": 4.320610687022901,
      "grad_norm": 0.20599593766122942,
      "kl": 0.0306396484375,
      "learning_rate": 2e-06,
      "loss": -0.0037,
      "reward": 0.9308036118745804,
      "reward_std": 0.055542015470564365,
      "rewards/accuracy_reward": 0.9308036118745804,
      "step": 847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.47322845458984,
      "epoch": 4.325699745547074,
      "grad_norm": 0.02636438452379431,
      "kl": 0.03277587890625,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.7142857313156128,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.7142857313156128,
      "step": 848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.9754638671875,
      "epoch": 4.330788804071247,
      "grad_norm": 0.19087436827216261,
      "kl": 0.0419921875,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.7209821715950966,
      "reward_std": 0.03898446634411812,
      "rewards/accuracy_reward": 0.7209821715950966,
      "step": 849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 368.35939025878906,
      "epoch": 4.33587786259542,
      "grad_norm": 0.2088613027630745,
      "kl": 0.0362548828125,
      "learning_rate": 2e-06,
      "loss": -0.0035,
      "reward": 0.9196428954601288,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.9196428954601288,
      "step": 850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.8549270629883,
      "epoch": 4.340966921119593,
      "grad_norm": 0.17342645567006615,
      "kl": 0.039215087890625,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.8571428954601288,
      "reward_std": 0.04199740011245012,
      "rewards/accuracy_reward": 0.8571428954601288,
      "step": 851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 372.8236770629883,
      "epoch": 4.346055979643766,
      "grad_norm": 0.3317254897457984,
      "kl": 0.04852294921875,
      "learning_rate": 2e-06,
      "loss": -0.0061,
      "reward": 0.76339291036129,
      "reward_std": 0.07110932748764753,
      "rewards/accuracy_reward": 0.76339291036129,
      "step": 852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.29466247558594,
      "epoch": 4.351145038167939,
      "grad_norm": 0.22875196579000068,
      "kl": 0.04150390625,
      "learning_rate": 2e-06,
      "loss": 0.0018,
      "reward": 0.7031250298023224,
      "reward_std": 0.05262112617492676,
      "rewards/accuracy_reward": 0.7031250298023224,
      "step": 853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.44197845458984,
      "epoch": 4.356234096692112,
      "grad_norm": 0.20864766284694214,
      "kl": 0.037567138671875,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.7723214626312256,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.7723214626312256,
      "step": 854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.02010345458984,
      "epoch": 4.361323155216285,
      "grad_norm": 0.24338668900360855,
      "kl": 0.04296875,
      "learning_rate": 2e-06,
      "loss": 0.0046,
      "reward": 0.7991071939468384,
      "reward_std": 0.07771168742328882,
      "rewards/accuracy_reward": 0.7991071939468384,
      "step": 855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.0491180419922,
      "epoch": 4.366412213740458,
      "grad_norm": 0.17575756479187551,
      "kl": 0.04254150390625,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.7522321790456772,
      "reward_std": 0.030368803068995476,
      "rewards/accuracy_reward": 0.7522321790456772,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.1071548461914,
      "epoch": 4.371501272264631,
      "grad_norm": 0.23331238391328157,
      "kl": 0.03662109375,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.8995535969734192,
      "reward_std": 0.03898446727544069,
      "rewards/accuracy_reward": 0.8995535969734192,
      "step": 857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.4910888671875,
      "epoch": 4.376590330788804,
      "grad_norm": 0.2336125881174906,
      "kl": 0.043701171875,
      "learning_rate": 2e-06,
      "loss": -0.004,
      "reward": 0.7745536118745804,
      "reward_std": 0.07056996412575245,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.8884048461914,
      "epoch": 4.3816793893129775,
      "grad_norm": 0.25928277167365654,
      "kl": 0.038299560546875,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.7232143208384514,
      "reward_std": 0.07979486510157585,
      "rewards/accuracy_reward": 0.7232143208384514,
      "step": 859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.59376525878906,
      "epoch": 4.3867684478371505,
      "grad_norm": 0.24894523001997013,
      "kl": 0.03948974609375,
      "learning_rate": 2e-06,
      "loss": 0.0008,
      "reward": 0.8325893133878708,
      "reward_std": 0.07672229409217834,
      "rewards/accuracy_reward": 0.8325893133878708,
      "step": 860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.9955520629883,
      "epoch": 4.391857506361323,
      "grad_norm": 0.19577810123274583,
      "kl": 0.031036376953125,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.6852678954601288,
      "reward_std": 0.03495405800640583,
      "rewards/accuracy_reward": 0.6852678954601288,
      "step": 861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.6964416503906,
      "epoch": 4.396946564885496,
      "grad_norm": 0.20197536286494003,
      "kl": 0.0345458984375,
      "learning_rate": 2e-06,
      "loss": -0.0055,
      "reward": 0.7745536118745804,
      "reward_std": 0.047152808867394924,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.18082427978516,
      "epoch": 4.402035623409669,
      "grad_norm": 0.23976335187052797,
      "kl": 0.03790283203125,
      "learning_rate": 2e-06,
      "loss": -0.0078,
      "reward": 0.848214328289032,
      "reward_std": 0.06154969893395901,
      "rewards/accuracy_reward": 0.848214328289032,
      "step": 863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.3772506713867,
      "epoch": 4.407124681933842,
      "grad_norm": 0.26654222786104065,
      "kl": 0.03662109375,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.8504464775323868,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.8504464775323868,
      "step": 864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.76341247558594,
      "epoch": 4.412213740458015,
      "grad_norm": 0.1967512921259146,
      "kl": 0.041778564453125,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.8058036118745804,
      "reward_std": 0.03495405800640583,
      "rewards/accuracy_reward": 0.8058036118745804,
      "step": 865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.3593978881836,
      "epoch": 4.417302798982188,
      "grad_norm": 0.09163076895436262,
      "kl": 0.030914306640625,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.993303582072258,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.993303582072258,
      "step": 866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.2522506713867,
      "epoch": 4.422391857506361,
      "grad_norm": 0.2931703029014787,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.0146,
      "reward": 0.850446492433548,
      "reward_std": 0.08977332152426243,
      "rewards/accuracy_reward": 0.850446492433548,
      "step": 867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.7522506713867,
      "epoch": 4.427480916030534,
      "grad_norm": 0.1968515926497254,
      "kl": 0.0335693359375,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.8035714775323868,
      "reward_std": 0.039297375828027725,
      "rewards/accuracy_reward": 0.8035714775323868,
      "step": 868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.0268096923828,
      "epoch": 4.432569974554707,
      "grad_norm": 0.286430818687198,
      "kl": 0.04132080078125,
      "learning_rate": 2e-06,
      "loss": 0.0035,
      "reward": 0.636160746216774,
      "reward_std": 0.07733787596225739,
      "rewards/accuracy_reward": 0.636160746216774,
      "step": 869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.36832427978516,
      "epoch": 4.43765903307888,
      "grad_norm": 0.2341059195427911,
      "kl": 0.03662109375,
      "learning_rate": 2e-06,
      "loss": 0.0115,
      "reward": 0.783482164144516,
      "reward_std": 0.065534682944417,
      "rewards/accuracy_reward": 0.783482164144516,
      "step": 870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 396.6919860839844,
      "epoch": 4.442748091603053,
      "grad_norm": 0.11970007418476948,
      "kl": 0.04150390625,
      "learning_rate": 2e-06,
      "loss": 0.0012,
      "reward": 0.7745536118745804,
      "reward_std": 0.01709691435098648,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.73885345458984,
      "epoch": 4.447837150127226,
      "grad_norm": 0.31282478039226147,
      "kl": 0.0458984375,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.698660746216774,
      "reward_std": 0.09500494413077831,
      "rewards/accuracy_reward": 0.698660746216774,
      "step": 872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.75894927978516,
      "epoch": 4.4529262086514,
      "grad_norm": 0.24356834862212692,
      "kl": 0.03485107421875,
      "learning_rate": 2e-06,
      "loss": 0.0031,
      "reward": 0.7500000447034836,
      "reward_std": 0.04791303910315037,
      "rewards/accuracy_reward": 0.7500000447034836,
      "step": 873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.5000228881836,
      "epoch": 4.458015267175573,
      "grad_norm": 0.1360092734005078,
      "kl": 0.03814697265625,
      "learning_rate": 2e-06,
      "loss": 0.0017,
      "reward": 0.7209821790456772,
      "reward_std": 0.014396889135241508,
      "rewards/accuracy_reward": 0.7209821790456772,
      "step": 874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.7321548461914,
      "epoch": 4.463104325699746,
      "grad_norm": 0.1426407133224715,
      "kl": 0.03326416015625,
      "learning_rate": 2e-06,
      "loss": -0.0014,
      "reward": 0.8437500298023224,
      "reward_std": 0.02817066479474306,
      "rewards/accuracy_reward": 0.8437500298023224,
      "step": 875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.80358123779297,
      "epoch": 4.468193384223919,
      "grad_norm": 0.12763067916860432,
      "kl": 0.037017822265625,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.8125000149011612,
      "reward_std": 0.023325461894273758,
      "rewards/accuracy_reward": 0.8125000149011612,
      "step": 876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.27903747558594,
      "epoch": 4.473282442748092,
      "grad_norm": 0.11032439569795521,
      "kl": 0.03985595703125,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.7120536118745804,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.7120536118745804,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.1093978881836,
      "epoch": 4.478371501272265,
      "grad_norm": 0.2320263882725396,
      "kl": 0.0384521484375,
      "learning_rate": 2e-06,
      "loss": -0.0034,
      "reward": 0.7901785969734192,
      "reward_std": 0.09605608507990837,
      "rewards/accuracy_reward": 0.7901785969734192,
      "step": 878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.83038330078125,
      "epoch": 4.483460559796438,
      "grad_norm": 0.21192156496262965,
      "kl": 0.04827880859375,
      "learning_rate": 2e-06,
      "loss": 0.0045,
      "reward": 0.7633928954601288,
      "reward_std": 0.036299921572208405,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.55135345458984,
      "epoch": 4.488549618320611,
      "grad_norm": 0.25738803265288984,
      "kl": 0.031890869140625,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.8906250447034836,
      "reward_std": 0.08501227665692568,
      "rewards/accuracy_reward": 0.8906250447034836,
      "step": 880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.48663330078125,
      "epoch": 4.493638676844784,
      "grad_norm": 0.27092509914408425,
      "kl": 0.035369873046875,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.8571428805589676,
      "reward_std": 0.054453397169709206,
      "rewards/accuracy_reward": 0.8571428805589676,
      "step": 881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.95091247558594,
      "epoch": 4.4987277353689565,
      "grad_norm": 0.24018205422589223,
      "kl": 0.037353515625,
      "learning_rate": 2e-06,
      "loss": 0.0064,
      "reward": 0.8258928805589676,
      "reward_std": 0.07589362747967243,
      "rewards/accuracy_reward": 0.8258928805589676,
      "step": 882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.5156478881836,
      "epoch": 4.5038167938931295,
      "grad_norm": 0.23727106426883504,
      "kl": 0.03857421875,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.7232143059372902,
      "reward_std": 0.05149612668901682,
      "rewards/accuracy_reward": 0.7232143059372902,
      "step": 883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.11608123779297,
      "epoch": 4.5089058524173025,
      "grad_norm": 0.13483502286872068,
      "kl": 0.037078857421875,
      "learning_rate": 2e-06,
      "loss": -0.001,
      "reward": 0.7879464626312256,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.7879464626312256,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.12054443359375,
      "epoch": 4.513994910941475,
      "grad_norm": 0.20034625013776441,
      "kl": 0.038543701171875,
      "learning_rate": 2e-06,
      "loss": -0.0,
      "reward": 0.7566964477300644,
      "reward_std": 0.060552770271897316,
      "rewards/accuracy_reward": 0.7566964477300644,
      "step": 885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.7857360839844,
      "epoch": 4.519083969465649,
      "grad_norm": 0.19729643597777208,
      "kl": 0.04119873046875,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.6272321864962578,
      "reward_std": 0.03772235009819269,
      "rewards/accuracy_reward": 0.6272321864962578,
      "step": 886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.87501525878906,
      "epoch": 4.524173027989822,
      "grad_norm": 0.18801744726938205,
      "kl": 0.03948974609375,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.8616071939468384,
      "reward_std": 0.056020475924015045,
      "rewards/accuracy_reward": 0.8616071939468384,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.9955520629883,
      "epoch": 4.529262086513995,
      "grad_norm": 0.19562726001505468,
      "kl": 0.03466796875,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.8839286267757416,
      "reward_std": 0.05445339623838663,
      "rewards/accuracy_reward": 0.8839286267757416,
      "step": 888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.70760345458984,
      "epoch": 4.534351145038168,
      "grad_norm": 0.17042521269209113,
      "kl": 0.03570556640625,
      "learning_rate": 2e-06,
      "loss": 0.0041,
      "reward": 0.8258928954601288,
      "reward_std": 0.03149380348622799,
      "rewards/accuracy_reward": 0.8258928954601288,
      "step": 889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.9643096923828,
      "epoch": 4.539440203562341,
      "grad_norm": 0.22965700334950423,
      "kl": 0.04071044921875,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.6473214477300644,
      "reward_std": 0.06532286200672388,
      "rewards/accuracy_reward": 0.6473214477300644,
      "step": 890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.0245666503906,
      "epoch": 4.544529262086514,
      "grad_norm": 0.25719299513895816,
      "kl": 0.03466796875,
      "learning_rate": 2e-06,
      "loss": -0.0024,
      "reward": 0.761160746216774,
      "reward_std": 0.11224650964140892,
      "rewards/accuracy_reward": 0.761160746216774,
      "step": 891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.10269927978516,
      "epoch": 4.549618320610687,
      "grad_norm": 0.11139812071369241,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.8504464775323868,
      "reward_std": 0.014396889135241508,
      "rewards/accuracy_reward": 0.8504464775323868,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.9486770629883,
      "epoch": 4.55470737913486,
      "grad_norm": 0.23697728587123176,
      "kl": 0.036773681640625,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.9062500447034836,
      "reward_std": 0.053571431897580624,
      "rewards/accuracy_reward": 0.9062500447034836,
      "step": 893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.9085006713867,
      "epoch": 4.559796437659033,
      "grad_norm": 0.07626423571726532,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": -0.0007,
      "reward": 0.8906250298023224,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.8906250298023224,
      "step": 894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.55135345458984,
      "epoch": 4.564885496183206,
      "grad_norm": 0.19714519917345338,
      "kl": 0.032379150390625,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.8058036118745804,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.8058036118745804,
      "step": 895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.3192138671875,
      "epoch": 4.569974554707379,
      "grad_norm": 0.288336479461512,
      "kl": 0.03863525390625,
      "learning_rate": 2e-06,
      "loss": -0.0023,
      "reward": 0.7098214626312256,
      "reward_std": 0.05822656024247408,
      "rewards/accuracy_reward": 0.7098214626312256,
      "step": 896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.8660888671875,
      "epoch": 4.575063613231552,
      "grad_norm": 0.3599363849448018,
      "kl": 0.04449462890625,
      "learning_rate": 2e-06,
      "loss": -0.0061,
      "reward": 0.7745535969734192,
      "reward_std": 0.07652468606829643,
      "rewards/accuracy_reward": 0.7745535969734192,
      "step": 897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.87278747558594,
      "epoch": 4.580152671755725,
      "grad_norm": 0.14609726333330406,
      "kl": 0.032073974609375,
      "learning_rate": 2e-06,
      "loss": 0.0019,
      "reward": 0.8348214626312256,
      "reward_std": 0.026025486178696156,
      "rewards/accuracy_reward": 0.8348214626312256,
      "step": 898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.6205520629883,
      "epoch": 4.585241730279899,
      "grad_norm": 0.12916363110190998,
      "kl": 0.037841796875,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.7321428954601288,
      "reward_std": 0.018442777916789055,
      "rewards/accuracy_reward": 0.7321428954601288,
      "step": 899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.19420623779297,
      "epoch": 4.590330788804071,
      "grad_norm": 0.28617106879822374,
      "kl": 0.0439453125,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.8638393133878708,
      "reward_std": 0.07984782662242651,
      "rewards/accuracy_reward": 0.8638393133878708,
      "step": 900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.13394927978516,
      "epoch": 4.595419847328245,
      "grad_norm": 0.10636725500810751,
      "kl": 0.03497314453125,
      "learning_rate": 2e-06,
      "loss": -0.0024,
      "reward": 0.8705357313156128,
      "reward_std": 0.03064152877777815,
      "rewards/accuracy_reward": 0.8705357313156128,
      "step": 901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 383.54913330078125,
      "epoch": 4.600508905852418,
      "grad_norm": 0.24339070909937904,
      "kl": 0.0423583984375,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.863839328289032,
      "reward_std": 0.06494904682040215,
      "rewards/accuracy_reward": 0.863839328289032,
      "step": 902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.40403747558594,
      "epoch": 4.6055979643765905,
      "grad_norm": 0.20227152112236466,
      "kl": 0.033447265625,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.8727678805589676,
      "reward_std": 0.030496877618134022,
      "rewards/accuracy_reward": 0.8727678805589676,
      "step": 903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.9486770629883,
      "epoch": 4.6106870229007635,
      "grad_norm": 0.16361222563326544,
      "kl": 0.03900146484375,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.9218750298023224,
      "reward_std": 0.026785715483129025,
      "rewards/accuracy_reward": 0.9218750298023224,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.8951110839844,
      "epoch": 4.6157760814249365,
      "grad_norm": 0.2266848944786614,
      "kl": 0.047119140625,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.6808036118745804,
      "reward_std": 0.03898446634411812,
      "rewards/accuracy_reward": 0.6808036118745804,
      "step": 905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.9018096923828,
      "epoch": 4.620865139949109,
      "grad_norm": 0.2179588960514838,
      "kl": 0.0423583984375,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.8348214626312256,
      "reward_std": 0.027226698584854603,
      "rewards/accuracy_reward": 0.8348214626312256,
      "step": 906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.7745666503906,
      "epoch": 4.625954198473282,
      "grad_norm": 0.34057522082116987,
      "kl": 0.042022705078125,
      "learning_rate": 2e-06,
      "loss": -0.0029,
      "reward": 0.7633928954601288,
      "reward_std": 0.049350948072969913,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.3973388671875,
      "epoch": 4.631043256997455,
      "grad_norm": 0.18572980686580934,
      "kl": 0.04290771484375,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.848214328289032,
      "reward_std": 0.03552421182394028,
      "rewards/accuracy_reward": 0.848214328289032,
      "step": 908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.79689025878906,
      "epoch": 4.636132315521628,
      "grad_norm": 0.19450772268488836,
      "kl": 0.04345703125,
      "learning_rate": 2e-06,
      "loss": -0.0031,
      "reward": 0.707589328289032,
      "reward_std": 0.037099236622452736,
      "rewards/accuracy_reward": 0.707589328289032,
      "step": 909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.9955520629883,
      "epoch": 4.641221374045801,
      "grad_norm": 0.2464243407356205,
      "kl": 0.032684326171875,
      "learning_rate": 2e-06,
      "loss": 0.0113,
      "reward": 0.8705357313156128,
      "reward_std": 0.056081379763782024,
      "rewards/accuracy_reward": 0.8705357313156128,
      "step": 910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.4933166503906,
      "epoch": 4.646310432569974,
      "grad_norm": 0.19269356120770217,
      "kl": 0.03448486328125,
      "learning_rate": 2e-06,
      "loss": 0.0031,
      "reward": 0.7455357611179352,
      "reward_std": 0.05256816744804382,
      "rewards/accuracy_reward": 0.7455357611179352,
      "step": 911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.3482360839844,
      "epoch": 4.651399491094148,
      "grad_norm": 0.1859106335445496,
      "kl": 0.037506103515625,
      "learning_rate": 2e-06,
      "loss": -0.0026,
      "reward": 0.7901786118745804,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.7901786118745804,
      "step": 912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.24332427978516,
      "epoch": 4.65648854961832,
      "grad_norm": 0.2648912200352862,
      "kl": 0.038330078125,
      "learning_rate": 2e-06,
      "loss": 0.0038,
      "reward": 0.745535746216774,
      "reward_std": 0.0806080549955368,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 428.8236846923828,
      "epoch": 4.661577608142494,
      "grad_norm": 0.1829319708419935,
      "kl": 0.04278564453125,
      "learning_rate": 2e-06,
      "loss": 0.0013,
      "reward": 0.7812500447034836,
      "reward_std": 0.047465717419981956,
      "rewards/accuracy_reward": 0.7812500447034836,
      "step": 914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.9464416503906,
      "epoch": 4.666666666666667,
      "grad_norm": 0.2926283781432313,
      "kl": 0.04461669921875,
      "learning_rate": 2e-06,
      "loss": -0.0008,
      "reward": 0.7366071790456772,
      "reward_std": 0.09345444105565548,
      "rewards/accuracy_reward": 0.7366071790456772,
      "step": 915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.5714416503906,
      "epoch": 4.67175572519084,
      "grad_norm": 0.22387212250904767,
      "kl": 0.04425048828125,
      "learning_rate": 2e-06,
      "loss": -0.0061,
      "reward": 0.6138393133878708,
      "reward_std": 0.057595500722527504,
      "rewards/accuracy_reward": 0.6138393133878708,
      "step": 916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.3326110839844,
      "epoch": 4.676844783715013,
      "grad_norm": 0.26370251833787844,
      "kl": 0.04241943359375,
      "learning_rate": 2e-06,
      "loss": -0.0024,
      "reward": 0.821428582072258,
      "reward_std": 0.07764968648552895,
      "rewards/accuracy_reward": 0.821428582072258,
      "step": 917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.5446472167969,
      "epoch": 4.681933842239186,
      "grad_norm": 0.25080278769285724,
      "kl": 0.03936767578125,
      "learning_rate": 2e-06,
      "loss": -0.0012,
      "reward": 0.7321428954601288,
      "reward_std": 0.07716327719390392,
      "rewards/accuracy_reward": 0.7321428954601288,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.2611846923828,
      "epoch": 4.687022900763359,
      "grad_norm": 0.20208596526498823,
      "kl": 0.03656005859375,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.8616071939468384,
      "reward_std": 0.05162420030683279,
      "rewards/accuracy_reward": 0.8616071939468384,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 434.10716247558594,
      "epoch": 4.692111959287532,
      "grad_norm": 0.30049536325985265,
      "kl": 0.042266845703125,
      "learning_rate": 2e-06,
      "loss": 0.0022,
      "reward": 0.7834821790456772,
      "reward_std": 0.06885028537362814,
      "rewards/accuracy_reward": 0.7834821790456772,
      "step": 920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.13841247558594,
      "epoch": 4.697201017811705,
      "grad_norm": 0.2683491526428907,
      "kl": 0.0406494140625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.7879464775323868,
      "reward_std": 0.0731173912063241,
      "rewards/accuracy_reward": 0.7879464775323868,
      "step": 921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.92859649658203,
      "epoch": 4.702290076335878,
      "grad_norm": 0.22634896088942552,
      "kl": 0.0382080078125,
      "learning_rate": 2e-06,
      "loss": 0.0002,
      "reward": 0.8549107760190964,
      "reward_std": 0.06764907389879227,
      "rewards/accuracy_reward": 0.8549107760190964,
      "step": 922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.20091247558594,
      "epoch": 4.707379134860051,
      "grad_norm": 0.16277773296573514,
      "kl": 0.037017822265625,
      "learning_rate": 2e-06,
      "loss": -0.003,
      "reward": 0.7723214775323868,
      "reward_std": 0.047465719282627106,
      "rewards/accuracy_reward": 0.7723214775323868,
      "step": 923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 394.81251525878906,
      "epoch": 4.712468193384224,
      "grad_norm": 0.12720180950778365,
      "kl": 0.034912109375,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.6763393133878708,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.6763393133878708,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 412.18082427978516,
      "epoch": 4.717557251908397,
      "grad_norm": 0.11652032046572212,
      "kl": 0.04071044921875,
      "learning_rate": 2e-06,
      "loss": -0.0002,
      "reward": 0.7879464477300644,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.7879464477300644,
      "step": 925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.50447845458984,
      "epoch": 4.7226463104325695,
      "grad_norm": 0.21457645604463352,
      "kl": 0.035308837890625,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.95089291036129,
      "reward_std": 0.03552421182394028,
      "rewards/accuracy_reward": 0.95089291036129,
      "step": 926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 418.89957427978516,
      "epoch": 4.727735368956743,
      "grad_norm": 0.15558747985672983,
      "kl": 0.03607177734375,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "reward": 0.7388393133878708,
      "reward_std": 0.024900486692786217,
      "rewards/accuracy_reward": 0.7388393133878708,
      "step": 927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 387.4933166503906,
      "epoch": 4.732824427480916,
      "grad_norm": 0.1597714283291237,
      "kl": 0.0367431640625,
      "learning_rate": 2e-06,
      "loss": 0.0055,
      "reward": 0.776785746216774,
      "reward_std": 0.03552421182394028,
      "rewards/accuracy_reward": 0.776785746216774,
      "step": 928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.53795623779297,
      "epoch": 4.737913486005089,
      "grad_norm": 0.07468168970450847,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.0043,
      "reward": 0.8191964775323868,
      "reward_std": 0.008928571827709675,
      "rewards/accuracy_reward": 0.8191964775323868,
      "step": 929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 385.9375228881836,
      "epoch": 4.743002544529262,
      "grad_norm": 0.2934059460481899,
      "kl": 0.0416259765625,
      "learning_rate": 2e-06,
      "loss": 0.0021,
      "reward": 0.658482164144516,
      "reward_std": 0.06432593520730734,
      "rewards/accuracy_reward": 0.658482164144516,
      "step": 930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 392.9442138671875,
      "epoch": 4.748091603053435,
      "grad_norm": 0.22240959310546268,
      "kl": 0.0391845703125,
      "learning_rate": 2e-06,
      "loss": -0.0026,
      "reward": 0.9196428656578064,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.9196428656578064,
      "step": 931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.7120666503906,
      "epoch": 4.753180661577608,
      "grad_norm": 0.2358872098669444,
      "kl": 0.037109375,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.8013393208384514,
      "reward_std": 0.04715280793607235,
      "rewards/accuracy_reward": 0.8013393208384514,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 380.70760345458984,
      "epoch": 4.758269720101781,
      "grad_norm": 0.1334731560244657,
      "kl": 0.03448486328125,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.7633928954601288,
      "reward_std": 0.032695015892386436,
      "rewards/accuracy_reward": 0.7633928954601288,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.9977798461914,
      "epoch": 4.763358778625954,
      "grad_norm": 0.31297726198591813,
      "kl": 0.04241943359375,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.8102678954601288,
      "reward_std": 0.09022335056215525,
      "rewards/accuracy_reward": 0.8102678954601288,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.9174270629883,
      "epoch": 4.768447837150127,
      "grad_norm": 0.026221145898374452,
      "kl": 0.0355224609375,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.8214286267757416,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.8214286267757416,
      "step": 935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.58260345458984,
      "epoch": 4.7735368956743,
      "grad_norm": 0.168073179995655,
      "kl": 0.03997802734375,
      "learning_rate": 2e-06,
      "loss": -0.0015,
      "reward": 0.8169643133878708,
      "reward_std": 0.01785714365541935,
      "rewards/accuracy_reward": 0.8169643133878708,
      "step": 936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.5223388671875,
      "epoch": 4.778625954198473,
      "grad_norm": 0.2884104922115341,
      "kl": 0.037750244140625,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.7522321790456772,
      "reward_std": 0.10550854075700045,
      "rewards/accuracy_reward": 0.7522321790456772,
      "step": 937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.6272430419922,
      "epoch": 4.783715012722646,
      "grad_norm": 0.2598500667086591,
      "kl": 0.0487060546875,
      "learning_rate": 2e-06,
      "loss": -0.0013,
      "reward": 0.714285746216774,
      "reward_std": 0.05985454376786947,
      "rewards/accuracy_reward": 0.714285746216774,
      "step": 938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.4955520629883,
      "epoch": 4.788804071246819,
      "grad_norm": 0.27851473283558814,
      "kl": 0.03985595703125,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.8883928954601288,
      "reward_std": 0.08506523165851831,
      "rewards/accuracy_reward": 0.8883928954601288,
      "step": 939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.6160888671875,
      "epoch": 4.793893129770993,
      "grad_norm": 0.21474023419633984,
      "kl": 0.034393310546875,
      "learning_rate": 2e-06,
      "loss": 0.0086,
      "reward": 0.8772321790456772,
      "reward_std": 0.055579494684934616,
      "rewards/accuracy_reward": 0.8772321790456772,
      "step": 940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.59376525878906,
      "epoch": 4.798982188295166,
      "grad_norm": 0.028555917958620735,
      "kl": 0.03741455078125,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.7857143133878708,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.7857143133878708,
      "step": 941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.28126525878906,
      "epoch": 4.804071246819339,
      "grad_norm": 0.1607631720795229,
      "kl": 0.04486083984375,
      "learning_rate": 2e-06,
      "loss": 0.0039,
      "reward": 0.8370536267757416,
      "reward_std": 0.018298126757144928,
      "rewards/accuracy_reward": 0.8370536267757416,
      "step": 942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.9643096923828,
      "epoch": 4.809160305343512,
      "grad_norm": 0.2168750544382541,
      "kl": 0.04241943359375,
      "learning_rate": 2e-06,
      "loss": 0.0026,
      "reward": 0.7901785969734192,
      "reward_std": 0.03822423703968525,
      "rewards/accuracy_reward": 0.7901785969734192,
      "step": 943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.9151916503906,
      "epoch": 4.814249363867685,
      "grad_norm": 0.1716971817309448,
      "kl": 0.03436279296875,
      "learning_rate": 2e-06,
      "loss": 0.0059,
      "reward": 0.6875000298023224,
      "reward_std": 0.0357142873108387,
      "rewards/accuracy_reward": 0.6875000298023224,
      "step": 944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.6250305175781,
      "epoch": 4.819338422391858,
      "grad_norm": 0.1343055994024979,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.8861607760190964,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.8861607760190964,
      "step": 945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.5245819091797,
      "epoch": 4.824427480916031,
      "grad_norm": 0.2720922671554191,
      "kl": 0.03759765625,
      "learning_rate": 2e-06,
      "loss": 0.0051,
      "reward": 0.7187500298023224,
      "reward_std": 0.0533813564106822,
      "rewards/accuracy_reward": 0.7187500298023224,
      "step": 946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.68751525878906,
      "epoch": 4.8295165394402035,
      "grad_norm": 0.22572044314359166,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.004,
      "reward": 0.745535746216774,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.9085006713867,
      "epoch": 4.8346055979643765,
      "grad_norm": 0.026522899248282012,
      "kl": 0.03717041015625,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.8214286118745804,
      "reward_std": 0.0,
      "rewards/accuracy_reward": 0.8214286118745804,
      "step": 948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.3125228881836,
      "epoch": 4.8396946564885495,
      "grad_norm": 0.29020971281239255,
      "kl": 0.039581298828125,
      "learning_rate": 2e-06,
      "loss": 0.0063,
      "reward": 0.6941964626312256,
      "reward_std": 0.07047826983034611,
      "rewards/accuracy_reward": 0.6941964626312256,
      "step": 949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.8973388671875,
      "epoch": 4.844783715012722,
      "grad_norm": 0.1472612735975997,
      "kl": 0.0362548828125,
      "learning_rate": 2e-06,
      "loss": -0.0001,
      "reward": 0.8526785969734192,
      "reward_std": 0.01785714365541935,
      "rewards/accuracy_reward": 0.8526785969734192,
      "step": 950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.2388687133789,
      "epoch": 4.849872773536895,
      "grad_norm": 0.2380951682550517,
      "kl": 0.03533935546875,
      "learning_rate": 2e-06,
      "loss": -0.0037,
      "reward": 0.7678571790456772,
      "reward_std": 0.07091925386339426,
      "rewards/accuracy_reward": 0.7678571790456772,
      "step": 951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.0848388671875,
      "epoch": 4.854961832061068,
      "grad_norm": 0.2303177194221303,
      "kl": 0.039794921875,
      "learning_rate": 2e-06,
      "loss": -0.002,
      "reward": 0.7745536118745804,
      "reward_std": 0.0458906926214695,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.20538330078125,
      "epoch": 4.860050890585242,
      "grad_norm": 0.1625724698535417,
      "kl": 0.03607177734375,
      "learning_rate": 2e-06,
      "loss": 0.0015,
      "reward": 0.6651785969734192,
      "reward_std": 0.026025486178696156,
      "rewards/accuracy_reward": 0.6651785969734192,
      "step": 953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.0379638671875,
      "epoch": 4.865139949109414,
      "grad_norm": 0.1632224309248044,
      "kl": 0.03607177734375,
      "learning_rate": 2e-06,
      "loss": 0.0033,
      "reward": 0.870535746216774,
      "reward_std": 0.027226698584854603,
      "rewards/accuracy_reward": 0.870535746216774,
      "step": 954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.2098388671875,
      "epoch": 4.870229007633588,
      "grad_norm": 0.24119040391822338,
      "kl": 0.036865234375,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.6540178805589676,
      "reward_std": 0.041494415141642094,
      "rewards/accuracy_reward": 0.6540178805589676,
      "step": 955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.7031478881836,
      "epoch": 4.875318066157761,
      "grad_norm": 0.16184515316598308,
      "kl": 0.03790283203125,
      "learning_rate": 2e-06,
      "loss": 0.0024,
      "reward": 0.8705357313156128,
      "reward_std": 0.04118260554969311,
      "rewards/accuracy_reward": 0.8705357313156128,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.06251525878906,
      "epoch": 4.880407124681934,
      "grad_norm": 0.29608545960046034,
      "kl": 0.0374755859375,
      "learning_rate": 2e-06,
      "loss": 0.0003,
      "reward": 0.8348214775323868,
      "reward_std": 0.07559729740023613,
      "rewards/accuracy_reward": 0.8348214775323868,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.70760345458984,
      "epoch": 4.885496183206107,
      "grad_norm": 0.3114491331748652,
      "kl": 0.036529541015625,
      "learning_rate": 2e-06,
      "loss": -0.0006,
      "reward": 0.7477678805589676,
      "reward_std": 0.09519501589238644,
      "rewards/accuracy_reward": 0.7477678805589676,
      "step": 958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.5893020629883,
      "epoch": 4.89058524173028,
      "grad_norm": 0.21215812775294332,
      "kl": 0.038909912109375,
      "learning_rate": 2e-06,
      "loss": 0.0112,
      "reward": 0.7321428880095482,
      "reward_std": 0.05055216047912836,
      "rewards/accuracy_reward": 0.7321428880095482,
      "step": 959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.93082427978516,
      "epoch": 4.895674300254453,
      "grad_norm": 0.13748623377345526,
      "kl": 0.037139892578125,
      "learning_rate": 2e-06,
      "loss": 0.008,
      "reward": 0.8526785969734192,
      "reward_std": 0.01785714365541935,
      "rewards/accuracy_reward": 0.8526785969734192,
      "step": 960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 414.26788330078125,
      "epoch": 4.900763358778626,
      "grad_norm": 0.20103315255542173,
      "kl": 0.034149169921875,
      "learning_rate": 2e-06,
      "loss": 0.0001,
      "reward": 0.7343750298023224,
      "reward_std": 0.03709923755377531,
      "rewards/accuracy_reward": 0.7343750298023224,
      "step": 961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.8504638671875,
      "epoch": 4.905852417302799,
      "grad_norm": 0.2960793778985056,
      "kl": 0.037841796875,
      "learning_rate": 2e-06,
      "loss": -0.0088,
      "reward": 0.9196428805589676,
      "reward_std": 0.08776525780558586,
      "rewards/accuracy_reward": 0.9196428805589676,
      "step": 962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.05135345458984,
      "epoch": 4.910941475826972,
      "grad_norm": 0.5996619896909581,
      "kl": 0.03887939453125,
      "learning_rate": 2e-06,
      "loss": -0.0062,
      "reward": 0.7388392984867096,
      "reward_std": 0.06764907389879227,
      "rewards/accuracy_reward": 0.7388392984867096,
      "step": 963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 398.14510345458984,
      "epoch": 4.916030534351145,
      "grad_norm": 0.15519140037016188,
      "kl": 0.031097412109375,
      "learning_rate": 2e-06,
      "loss": -0.0032,
      "reward": 0.8616071939468384,
      "reward_std": 0.03382905758917332,
      "rewards/accuracy_reward": 0.8616071939468384,
      "step": 964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.8817138671875,
      "epoch": 4.921119592875318,
      "grad_norm": 0.14356234993332598,
      "kl": 0.0345458984375,
      "learning_rate": 2e-06,
      "loss": 0.0019,
      "reward": 0.8593750447034836,
      "reward_std": 0.02112732268869877,
      "rewards/accuracy_reward": 0.8593750447034836,
      "step": 965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.2678756713867,
      "epoch": 4.926208651399491,
      "grad_norm": 0.20752419401112923,
      "kl": 0.039215087890625,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.8258928954601288,
      "reward_std": 0.050552159547805786,
      "rewards/accuracy_reward": 0.8258928954601288,
      "step": 966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.64957427978516,
      "epoch": 4.931297709923664,
      "grad_norm": 0.17012701589914264,
      "kl": 0.03399658203125,
      "learning_rate": 2e-06,
      "loss": 0.0065,
      "reward": 0.7098214626312256,
      "reward_std": 0.030055894516408443,
      "rewards/accuracy_reward": 0.7098214626312256,
      "step": 967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.7901916503906,
      "epoch": 4.9363867684478375,
      "grad_norm": 0.2558617450354285,
      "kl": 0.03790283203125,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.7968750447034836,
      "reward_std": 0.06885028630495071,
      "rewards/accuracy_reward": 0.7968750447034836,
      "step": 968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 403.5268020629883,
      "epoch": 4.9414758269720105,
      "grad_norm": 0.11205730650145966,
      "kl": 0.03814697265625,
      "learning_rate": 2e-06,
      "loss": 0.0009,
      "reward": 0.8973214328289032,
      "reward_std": 0.012198750860989094,
      "rewards/accuracy_reward": 0.8973214328289032,
      "step": 969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.9085006713867,
      "epoch": 4.9465648854961835,
      "grad_norm": 0.15661667644129318,
      "kl": 0.03424072265625,
      "learning_rate": 2e-06,
      "loss": -0.004,
      "reward": 0.9017857313156128,
      "reward_std": 0.026025486178696156,
      "rewards/accuracy_reward": 0.9017857313156128,
      "step": 970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.1384048461914,
      "epoch": 4.951653944020356,
      "grad_norm": 0.23728140080143653,
      "kl": 0.043212890625,
      "learning_rate": 2e-06,
      "loss": 0.0032,
      "reward": 0.7745536118745804,
      "reward_std": 0.060614774003624916,
      "rewards/accuracy_reward": 0.7745536118745804,
      "step": 971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.42635345458984,
      "epoch": 4.956743002544529,
      "grad_norm": 0.27996267425507504,
      "kl": 0.037109375,
      "learning_rate": 2e-06,
      "loss": 0.0014,
      "reward": 0.6830357313156128,
      "reward_std": 0.08018255140632391,
      "rewards/accuracy_reward": 0.6830357313156128,
      "step": 972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.0580596923828,
      "epoch": 4.961832061068702,
      "grad_norm": 0.24214198480984425,
      "kl": 0.038848876953125,
      "learning_rate": 2e-06,
      "loss": -0.0011,
      "reward": 0.745535746216774,
      "reward_std": 0.0629409858956933,
      "rewards/accuracy_reward": 0.745535746216774,
      "step": 973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.69644927978516,
      "epoch": 4.966921119592875,
      "grad_norm": 0.27715338649050936,
      "kl": 0.0360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0062,
      "reward": 0.7857143133878708,
      "reward_std": 0.10380434338003397,
      "rewards/accuracy_reward": 0.7857143133878708,
      "step": 974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.4107360839844,
      "epoch": 4.972010178117048,
      "grad_norm": 0.19920602212269978,
      "kl": 0.037261962890625,
      "learning_rate": 2e-06,
      "loss": -0.0004,
      "reward": 0.8303571790456772,
      "reward_std": 0.05728259403258562,
      "rewards/accuracy_reward": 0.8303571790456772,
      "step": 975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 391.82144927978516,
      "epoch": 4.977099236641221,
      "grad_norm": 0.256930814820825,
      "kl": 0.03521728515625,
      "learning_rate": 2e-06,
      "loss": 0.0044,
      "reward": 0.848214328289032,
      "reward_std": 0.04791303817182779,
      "rewards/accuracy_reward": 0.848214328289032,
      "step": 976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.7165298461914,
      "epoch": 4.982188295165394,
      "grad_norm": 0.2157480024136974,
      "kl": 0.034698486328125,
      "learning_rate": 2e-06,
      "loss": -0.0067,
      "reward": 0.8928571790456772,
      "reward_std": 0.054453397169709206,
      "rewards/accuracy_reward": 0.8928571790456772,
      "step": 977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.3236770629883,
      "epoch": 4.987277353689567,
      "grad_norm": 0.23626104407159002,
      "kl": 0.03369140625,
      "learning_rate": 2e-06,
      "loss": 0.0093,
      "reward": 0.8214285969734192,
      "reward_std": 0.09703753143548965,
      "rewards/accuracy_reward": 0.8214285969734192,
      "step": 978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.3303756713867,
      "epoch": 4.99236641221374,
      "grad_norm": 0.21612155133967967,
      "kl": 0.03619384765625,
      "learning_rate": 2e-06,
      "loss": 0.0027,
      "reward": 0.7410714477300644,
      "reward_std": 0.03942544851452112,
      "rewards/accuracy_reward": 0.7410714477300644,
      "step": 979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.9423294067383,
      "epoch": 4.997455470737913,
      "grad_norm": 0.19596386095038013,
      "kl": 0.0380859375,
      "learning_rate": 2e-06,
      "loss": 0.0032,
      "reward": 0.8883928656578064,
      "reward_std": 0.038224236108362675,
      "rewards/accuracy_reward": 0.8883928656578064,
      "step": 980
    },
    {
      "epoch": 4.997455470737913,
      "step": 980,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 1.0788,
      "train_samples_per_second": 25495.021,
      "train_steps_per_second": 908.385
    }
  ],
  "logging_steps": 1,
  "max_steps": 980,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 5,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}