{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2570694087403599,
  "eval_steps": 500,
  "global_step": 300,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 3156.8612060546875,
      "epoch": 0.000856898029134533,
      "grad_norm": 0.08138668537139893,
      "kl": 0.0,
      "learning_rate": 3.3333333333333334e-08,
      "loss": 0.0383,
      "reward": -2.583333373069763,
      "reward_std": 0.09128709882497787,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4166666716337204,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3294.638916015625,
      "epoch": 0.001713796058269066,
      "grad_norm": 0.3450618088245392,
      "kl": 0.0,
      "learning_rate": 6.666666666666667e-08,
      "loss": 0.1933,
      "reward": -2.6666667461395264,
      "reward_std": 0.2221490517258644,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3111.7222900390625,
      "epoch": 0.002570694087403599,
      "grad_norm": 0.4438176453113556,
      "kl": 4.208087921142578e-05,
      "learning_rate": 1e-07,
      "loss": 0.1213,
      "reward": -2.435185194015503,
      "reward_std": 0.46697859466075897,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.4722222238779068,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3262.8612060546875,
      "epoch": 0.003427592116538132,
      "grad_norm": 0.1571539044380188,
      "kl": 3.17692756652832e-05,
      "learning_rate": 1.3333333333333334e-07,
      "loss": 0.1405,
      "reward": -2.6944445371627808,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555671453476,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3076.166748046875,
      "epoch": 0.004284490145672665,
      "grad_norm": 0.0744488388299942,
      "kl": 4.2378902435302734e-05,
      "learning_rate": 1.6666666666666665e-07,
      "loss": 0.0441,
      "reward": -2.4722222089767456,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777910232544,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3411.5277099609375,
      "epoch": 0.005141388174807198,
      "grad_norm": 0.1373002976179123,
      "kl": 2.950429916381836e-05,
      "learning_rate": 2e-07,
      "loss": 0.0469,
      "reward": -2.583333373069763,
      "reward_std": 0.33668186515569687,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.361111119389534,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3255.5556640625,
      "epoch": 0.005998286203941731,
      "grad_norm": 0.1415766477584839,
      "kl": 3.135204315185547e-05,
      "learning_rate": 2.3333333333333333e-07,
      "loss": 0.0729,
      "reward": -2.5,
      "reward_std": 0.2453947737812996,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000149011612,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2561.388916015625,
      "epoch": 0.006855184233076264,
      "grad_norm": 0.3355005085468292,
      "kl": 3.647804260253906e-05,
      "learning_rate": 2.6666666666666667e-07,
      "loss": 0.0923,
      "reward": -2.170351266860962,
      "reward_std": 0.6446175873279572,
      "rewards/confidence_reward": -2.809240221977234,
      "rewards/format_reward": 0.6388888955116272,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3178.75,
      "epoch": 0.007712082262210797,
      "grad_norm": 0.2505446970462799,
      "kl": 2.8371810913085938e-05,
      "learning_rate": 3e-07,
      "loss": 0.2008,
      "reward": -2.4722222089767456,
      "reward_std": 0.409944087266922,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4722222238779068,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2501.1944580078125,
      "epoch": 0.00856898029134533,
      "grad_norm": 0.2691773772239685,
      "kl": 2.0295381546020508e-05,
      "learning_rate": 3.333333333333333e-07,
      "loss": 0.1208,
      "reward": -2.4166667461395264,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5833333283662796,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3531.8333740234375,
      "epoch": 0.009425878320479864,
      "grad_norm": 0.1814609318971634,
      "kl": 3.612041473388672e-05,
      "learning_rate": 3.666666666666666e-07,
      "loss": 0.0473,
      "reward": -2.555555582046509,
      "reward_std": 0.3082153648138046,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444552063942,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3414.6666259765625,
      "epoch": 0.010282776349614395,
      "grad_norm": 0.14940866827964783,
      "kl": 4.470348358154297e-05,
      "learning_rate": 4e-07,
      "loss": 0.1065,
      "reward": -2.6388888359069824,
      "reward_std": 0.25061557441949844,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3611111268401146,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2698.2222900390625,
      "epoch": 0.011139674378748929,
      "grad_norm": 0.193411186337471,
      "kl": 3.445148468017578e-05,
      "learning_rate": 4.3333333333333335e-07,
      "loss": 0.0232,
      "reward": -2.2592592239379883,
      "reward_std": 0.37295398116111755,
      "rewards/confidence_reward": -2.814814805984497,
      "rewards/format_reward": 0.5555555671453476,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.9166870117188,
      "epoch": 0.011996572407883462,
      "grad_norm": 0.1921149641275406,
      "kl": 2.7477741241455078e-05,
      "learning_rate": 4.6666666666666666e-07,
      "loss": 0.0873,
      "reward": -1.929610550403595,
      "reward_std": 0.4556204080581665,
      "rewards/confidence_reward": -2.7907216548919678,
      "rewards/format_reward": 0.8611111044883728,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2907.9444580078125,
      "epoch": 0.012853470437017995,
      "grad_norm": 0.1692344844341278,
      "kl": 2.485513687133789e-05,
      "learning_rate": 5e-07,
      "loss": 0.0894,
      "reward": -2.305555582046509,
      "reward_std": 0.22736986726522446,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6944444477558136,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2834.5833740234375,
      "epoch": 0.013710368466152529,
      "grad_norm": 0.20961543917655945,
      "kl": 2.6166439056396484e-05,
      "learning_rate": 5.333333333333333e-07,
      "loss": 0.1212,
      "reward": -2.3611111640930176,
      "reward_std": 0.4495189040899277,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.5833333283662796,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1908.2777709960938,
      "epoch": 0.01456726649528706,
      "grad_norm": 0.13441088795661926,
      "kl": 1.5820842236280441e-06,
      "learning_rate": 5.666666666666666e-07,
      "loss": 0.0448,
      "reward": -2.25,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.75,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3585.5001220703125,
      "epoch": 0.015424164524421594,
      "grad_norm": 0.23104918003082275,
      "kl": 3.343820571899414e-05,
      "learning_rate": 6e-07,
      "loss": 0.1208,
      "reward": -2.6666667461395264,
      "reward_std": 0.3314610570669174,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2827.1943969726562,
      "epoch": 0.016281062553556127,
      "grad_norm": 0.24534867703914642,
      "kl": 2.0720064640045166e-05,
      "learning_rate": 6.333333333333332e-07,
      "loss": 0.0672,
      "reward": -2.555555582046509,
      "reward_std": 0.245394766330719,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444477558136,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4031.5556640625,
      "epoch": 0.01713796058269066,
      "grad_norm": 0.14566344022750854,
      "kl": 3.6835670471191406e-05,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.0251,
      "reward": -2.8333332538604736,
      "reward_std": 0.30821534991264343,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1666666716337204,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2811.638916015625,
      "epoch": 0.017994858611825194,
      "grad_norm": 0.2733408510684967,
      "kl": 1.6957521438598633e-05,
      "learning_rate": 7e-07,
      "loss": 0.1159,
      "reward": -2.3611111640930176,
      "reward_std": 0.48556874692440033,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.5277777910232544,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3465.8056640625,
      "epoch": 0.018851756640959727,
      "grad_norm": 0.197701096534729,
      "kl": 2.8759241104125977e-05,
      "learning_rate": 7.333333333333332e-07,
      "loss": 0.0588,
      "reward": -2.694444417953491,
      "reward_std": 0.2901904359459877,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.25000000186264515,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3026.4722900390625,
      "epoch": 0.01970865467009426,
      "grad_norm": 0.3089195787906647,
      "kl": 1.558661460876465e-05,
      "learning_rate": 7.666666666666667e-07,
      "loss": 0.2173,
      "reward": -2.555555582046509,
      "reward_std": 0.2221490517258644,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444552063942,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3507.5555419921875,
      "epoch": 0.02056555269922879,
      "grad_norm": 0.40053591132164,
      "kl": 5.6743621826171875e-05,
      "learning_rate": 8e-07,
      "loss": 0.1994,
      "reward": -2.6944445371627808,
      "reward_std": 0.3995024412870407,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555671453476,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3597.361083984375,
      "epoch": 0.021422450728363324,
      "grad_norm": 0.3277747929096222,
      "kl": 2.2545456886291504e-05,
      "learning_rate": 8.333333333333333e-07,
      "loss": 0.1546,
      "reward": -2.583333373069763,
      "reward_std": 0.49601033329963684,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4166666716337204,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2460.694580078125,
      "epoch": 0.022279348757497857,
      "grad_norm": 0.24351875483989716,
      "kl": 3.9458274841308594e-05,
      "learning_rate": 8.666666666666667e-07,
      "loss": 0.1574,
      "reward": -2.305555582046509,
      "reward_std": 0.22736985981464386,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6944444477558136,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3784.3056640625,
      "epoch": 0.02313624678663239,
      "grad_norm": 0.14974988996982574,
      "kl": 0.0001430511474609375,
      "learning_rate": 9e-07,
      "loss": 0.0993,
      "reward": -2.7777777910232544,
      "reward_std": 0.2453947588801384,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.22222222574055195,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2524.4444580078125,
      "epoch": 0.023993144815766924,
      "grad_norm": 0.2757478356361389,
      "kl": 0.00017023086547851562,
      "learning_rate": 9.333333333333333e-07,
      "loss": 0.0924,
      "reward": -2.305555582046509,
      "reward_std": 0.3995024487376213,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.5833333432674408,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3612.0556640625,
      "epoch": 0.024850042844901457,
      "grad_norm": 0.31738027930259705,
      "kl": 0.00018072128295898438,
      "learning_rate": 9.666666666666666e-07,
      "loss": 0.1543,
      "reward": -2.6666667461395264,
      "reward_std": 0.4803479313850403,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333358168602,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3717.0555419921875,
      "epoch": 0.02570694087403599,
      "grad_norm": 0.223888099193573,
      "kl": 0.00014829635620117188,
      "learning_rate": 1e-06,
      "loss": 0.1178,
      "reward": -2.583333373069763,
      "reward_std": 0.49601036310195923,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4166666716337204,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2863.02783203125,
      "epoch": 0.026563838903170524,
      "grad_norm": 0.15052570402622223,
      "kl": 0.0007700920104980469,
      "learning_rate": 9.99969538601693e-07,
      "loss": 0.1475,
      "reward": -2.472222328186035,
      "reward_std": 0.25061558187007904,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777761220932,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2220.388916015625,
      "epoch": 0.027420736932305057,
      "grad_norm": 0.1104821264743805,
      "kl": 0.0006427764892578125,
      "learning_rate": 9.998781585307575e-07,
      "loss": 0.0251,
      "reward": -2.1944445371627808,
      "reward_std": 0.2901904359459877,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.75,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2779.8055419921875,
      "epoch": 0.028277634961439587,
      "grad_norm": 0.2474696934223175,
      "kl": 0.00033664703369140625,
      "learning_rate": 9.997258721585931e-07,
      "loss": 0.0417,
      "reward": -2.2870370149612427,
      "reward_std": 0.4910118281841278,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.5277777910232544,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3405.7777099609375,
      "epoch": 0.02913453299057412,
      "grad_norm": 0.11548831313848495,
      "kl": 0.0003676414489746094,
      "learning_rate": 9.99512700102336e-07,
      "loss": 0.0451,
      "reward": -2.7222222089767456,
      "reward_std": 0.15932847559452057,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2777777835726738,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3638.1666259765625,
      "epoch": 0.029991431019708654,
      "grad_norm": 0.22043903172016144,
      "kl": 0.0002741813659667969,
      "learning_rate": 9.992386712220707e-07,
      "loss": 0.1144,
      "reward": -2.6666667461395264,
      "reward_std": 0.31865697354078293,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3693.27783203125,
      "epoch": 0.030848329048843187,
      "grad_norm": 0.008012472651898861,
      "kl": 0.0003859996795654297,
      "learning_rate": 9.989038226169207e-07,
      "loss": 0.0,
      "reward": -2.833333373069763,
      "reward_std": 0.0,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1666666716337204,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3796.9444580078125,
      "epoch": 0.031705227077977724,
      "grad_norm": 0.18162299692630768,
      "kl": 0.001140594482421875,
      "learning_rate": 9.985081996200277e-07,
      "loss": 0.056,
      "reward": -2.75,
      "reward_std": 0.22736985236406326,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.25000000186264515,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3224.5001220703125,
      "epoch": 0.032562125107112254,
      "grad_norm": 0.33491846919059753,
      "kl": 0.00153350830078125,
      "learning_rate": 9.98051855792412e-07,
      "loss": 0.222,
      "reward": -2.4722222089767456,
      "reward_std": 0.4983728677034378,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777910232544,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3594.0,
      "epoch": 0.033419023136246784,
      "grad_norm": 0.17942766845226288,
      "kl": 0.0010013580322265625,
      "learning_rate": 9.975348529157229e-07,
      "loss": 0.0885,
      "reward": -2.6111111640930176,
      "reward_std": 0.36798951029777527,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3333333432674408,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3537.7222900390625,
      "epoch": 0.03427592116538132,
      "grad_norm": 0.20143228769302368,
      "kl": 0.001697540283203125,
      "learning_rate": 9.969572609838744e-07,
      "loss": 0.1538,
      "reward": -2.555555582046509,
      "reward_std": 0.3814775347709656,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444626569748,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3175.02783203125,
      "epoch": 0.03513281919451585,
      "grad_norm": 0.16529610753059387,
      "kl": 0.000911712646484375,
      "learning_rate": 9.963191581935677e-07,
      "loss": 0.1,
      "reward": -2.555555582046509,
      "reward_std": 0.15932848304510117,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444477558136,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3616.7777099609375,
      "epoch": 0.03598971722365039,
      "grad_norm": 0.08064991980791092,
      "kl": 0.001148223876953125,
      "learning_rate": 9.956206309337066e-07,
      "loss": 0.0023,
      "reward": -2.7777777910232544,
      "reward_std": 0.08606629818677902,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2222222276031971,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3938.3055419921875,
      "epoch": 0.03684661525278492,
      "grad_norm": 0.12845046818256378,
      "kl": 0.00202178955078125,
      "learning_rate": 9.948617737737001e-07,
      "loss": 0.0375,
      "reward": -2.805555582046509,
      "reward_std": 0.1773533970117569,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1944444477558136,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3115.9722900390625,
      "epoch": 0.037703513281919454,
      "grad_norm": 0.1982233226299286,
      "kl": 0.0035247802734375,
      "learning_rate": 9.940426894506606e-07,
      "loss": 0.1154,
      "reward": -2.3240740299224854,
      "reward_std": 0.49022431671619415,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5833333432674408,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3329.9444580078125,
      "epoch": 0.038560411311053984,
      "grad_norm": 0.2199009209871292,
      "kl": 0.00379180908203125,
      "learning_rate": 9.931634888554935e-07,
      "loss": 0.0548,
      "reward": -2.6388888359069824,
      "reward_std": 0.36345263570547104,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.30555556900799274,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3532.666748046875,
      "epoch": 0.03941730934018852,
      "grad_norm": 0.1083880066871643,
      "kl": 0.0015277862548828125,
      "learning_rate": 9.922242910178859e-07,
      "loss": 0.0935,
      "reward": -2.638888955116272,
      "reward_std": 0.29313045740127563,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3055555671453476,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2992.1944580078125,
      "epoch": 0.04027420736932305,
      "grad_norm": 1.405205488204956,
      "kl": 0.0645751953125,
      "learning_rate": 9.912252230901906e-07,
      "loss": 0.0511,
      "reward": -2.5,
      "reward_std": 0.3582318127155304,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4444444477558136,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3138.666748046875,
      "epoch": 0.04113110539845758,
      "grad_norm": 0.19926148653030396,
      "kl": 0.02202606201171875,
      "learning_rate": 9.901664203302124e-07,
      "loss": 0.0487,
      "reward": -2.5,
      "reward_std": 0.13608276098966599,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2550.6944580078125,
      "epoch": 0.04198800342759212,
      "grad_norm": 0.2365693897008896,
      "kl": 0.0066070556640625,
      "learning_rate": 9.890480260828965e-07,
      "loss": 0.1002,
      "reward": -2.138888955116272,
      "reward_std": 0.49953536689281464,
      "rewards/confidence_reward": -2.8888888359069824,
      "rewards/format_reward": 0.75,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2925.8612060546875,
      "epoch": 0.04284490145672665,
      "grad_norm": 0.2880806028842926,
      "kl": 0.0074005126953125,
      "learning_rate": 9.878701917609207e-07,
      "loss": 0.2009,
      "reward": -2.4166667461395264,
      "reward_std": 0.49203380942344666,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.4166666716337204,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3871.27783203125,
      "epoch": 0.043701799485861184,
      "grad_norm": 0.1224374920129776,
      "kl": 0.00347900390625,
      "learning_rate": 9.866330768241983e-07,
      "loss": 0.067,
      "reward": -2.75,
      "reward_std": 0.2634196802973747,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2500000037252903,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3002.5833740234375,
      "epoch": 0.044558697514995714,
      "grad_norm": 0.3084653913974762,
      "kl": 0.010650634765625,
      "learning_rate": 9.853368487582886e-07,
      "loss": 0.1609,
      "reward": -2.472222328186035,
      "reward_std": 0.22736985981464386,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777761220932,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3275.4444580078125,
      "epoch": 0.04541559554413025,
      "grad_norm": 0.1611953228712082,
      "kl": 0.00553131103515625,
      "learning_rate": 9.839816830517225e-07,
      "loss": 0.0874,
      "reward": -2.5277777910232544,
      "reward_std": 0.34948598593473434,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4722222238779068,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3493.4444580078125,
      "epoch": 0.04627249357326478,
      "grad_norm": 0.05148787423968315,
      "kl": 0.00240325927734375,
      "learning_rate": 9.825677631722435e-07,
      "loss": 0.0171,
      "reward": -2.4166667461395264,
      "reward_std": 0.09128709882497787,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5833333283662796,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2541.5,
      "epoch": 0.04712939160239932,
      "grad_norm": 0.2660021483898163,
      "kl": 0.0017642974853515625,
      "learning_rate": 9.8109528054197e-07,
      "loss": 0.0367,
      "reward": -2.2685184478759766,
      "reward_std": 0.5399755388498306,
      "rewards/confidence_reward": -2.7962963581085205,
      "rewards/format_reward": 0.5277777761220932,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2758.02783203125,
      "epoch": 0.04798628963153385,
      "grad_norm": 0.1080583855509758,
      "kl": 0.002521514892578125,
      "learning_rate": 9.795644345114794e-07,
      "loss": 0.0841,
      "reward": -2.277777910232544,
      "reward_std": 0.2221490517258644,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.7222222089767456,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3181.0001220703125,
      "epoch": 0.04884318766066838,
      "grad_norm": 0.3695676922798157,
      "kl": 0.0043849945068359375,
      "learning_rate": 9.779754323328192e-07,
      "loss": 0.2031,
      "reward": -2.4166667461395264,
      "reward_std": 0.34020692110061646,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5833333283662796,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3258.4722900390625,
      "epoch": 0.049700085689802914,
      "grad_norm": 0.45151278376579285,
      "kl": 0.002864837646484375,
      "learning_rate": 9.76328489131448e-07,
      "loss": 0.2348,
      "reward": -2.5833332538604736,
      "reward_std": 0.3995024561882019,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.416666679084301,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3137.638916015625,
      "epoch": 0.050556983718937444,
      "grad_norm": 0.24236924946308136,
      "kl": 0.007476806640625,
      "learning_rate": 9.746238278771125e-07,
      "loss": 0.0592,
      "reward": -2.3888888359069824,
      "reward_std": 0.245394766330719,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6111111044883728,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2517.861083984375,
      "epoch": 0.05141388174807198,
      "grad_norm": 0.17099355161190033,
      "kl": 0.00344085693359375,
      "learning_rate": 9.728616793536587e-07,
      "loss": 0.0848,
      "reward": -2.1944445371627808,
      "reward_std": 0.36345260590314865,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.75,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3005.75,
      "epoch": 0.05227077977720651,
      "grad_norm": 0.17847158014774323,
      "kl": 0.0050048828125,
      "learning_rate": 9.71042282127789e-07,
      "loss": 0.0104,
      "reward": -2.379629611968994,
      "reward_std": 0.2948459982872009,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5277777761220932,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2963.4722900390625,
      "epoch": 0.05312767780634105,
      "grad_norm": 0.22850683331489563,
      "kl": 0.00327301025390625,
      "learning_rate": 9.69165882516764e-07,
      "loss": -0.0198,
      "reward": -2.6111111640930176,
      "reward_std": 0.1360827535390854,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3333333432674408,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2869.3056640625,
      "epoch": 0.05398457583547558,
      "grad_norm": 0.1911962777376175,
      "kl": 0.001628875732421875,
      "learning_rate": 9.672327345550543e-07,
      "loss": 0.0823,
      "reward": -2.1574073433876038,
      "reward_std": 0.4669785648584366,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.75,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3161.9444580078125,
      "epoch": 0.054841473864610114,
      "grad_norm": 0.349700391292572,
      "kl": 0.00498199462890625,
      "learning_rate": 9.65243099959949e-07,
      "loss": 0.2121,
      "reward": -2.5,
      "reward_std": 0.4047232121229172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000149011612,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2009.166748046875,
      "epoch": 0.055698371893744644,
      "grad_norm": 0.03013899177312851,
      "kl": 0.00531768798828125,
      "learning_rate": 9.631972480961233e-07,
      "loss": 0.0002,
      "reward": -2.3333332538604736,
      "reward_std": 0.0,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6666666865348816,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3121.138916015625,
      "epoch": 0.056555269922879174,
      "grad_norm": 0.1924014538526535,
      "kl": 0.003353118896484375,
      "learning_rate": 9.610954559391704e-07,
      "loss": 0.0912,
      "reward": -2.333333373069763,
      "reward_std": 0.5140352547168732,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.6111111044883728,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2877.4722900390625,
      "epoch": 0.05741216795201371,
      "grad_norm": 0.34053537249565125,
      "kl": 0.005859375,
      "learning_rate": 9.589380080381038e-07,
      "loss": -0.0137,
      "reward": -2.3425925970077515,
      "reward_std": 0.5270616561174393,
      "rewards/confidence_reward": -2.7037038803100586,
      "rewards/format_reward": 0.361111119389534,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3536.3333740234375,
      "epoch": 0.05826906598114824,
      "grad_norm": 0.24978837370872498,
      "kl": 0.00376129150390625,
      "learning_rate": 9.567251964768342e-07,
      "loss": 0.1593,
      "reward": -2.555555582046509,
      "reward_std": 0.4675438404083252,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444477558136,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3617.8612060546875,
      "epoch": 0.05912596401028278,
      "grad_norm": 0.23766648769378662,
      "kl": 0.00360870361328125,
      "learning_rate": 9.54457320834625e-07,
      "loss": 0.1549,
      "reward": -2.722222328186035,
      "reward_std": 0.3814775347709656,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2777777910232544,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3170.4444580078125,
      "epoch": 0.05998286203941731,
      "grad_norm": 0.11585590243339539,
      "kl": 0.0055084228515625,
      "learning_rate": 9.521346881455354e-07,
      "loss": 0.0739,
      "reward": -2.5,
      "reward_std": 0.2221490666270256,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4444444626569748,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3495.6666259765625,
      "epoch": 0.060839760068551844,
      "grad_norm": 0.25010788440704346,
      "kl": 0.0036468505859375,
      "learning_rate": 9.497576128568518e-07,
      "loss": 0.0395,
      "reward": -2.7041274309158325,
      "reward_std": 0.316488653421402,
      "rewards/confidence_reward": -2.8985718488693237,
      "rewards/format_reward": 0.1944444477558136,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2589.1111450195312,
      "epoch": 0.061696658097686374,
      "grad_norm": 0.08638693392276764,
      "kl": 0.0056915283203125,
      "learning_rate": 9.473264167865171e-07,
      "loss": 0.0357,
      "reward": -2.4444445371627808,
      "reward_std": 0.13608276844024658,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.5000000149011612,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3842.9444580078125,
      "epoch": 0.06255355612682091,
      "grad_norm": 0.2187531441450119,
      "kl": 0.003448486328125,
      "learning_rate": 9.448414290795618e-07,
      "loss": 0.1007,
      "reward": -2.833333373069763,
      "reward_std": 0.30821535736322403,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.16666667349636555,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3221.8055419921875,
      "epoch": 0.06341045415595545,
      "grad_norm": 0.21624591946601868,
      "kl": 0.00295257568359375,
      "learning_rate": 9.42302986163543e-07,
      "loss": 0.073,
      "reward": -2.4444445371627808,
      "reward_std": 0.3082153648138046,
      "rewards/confidence_reward": -2.7777777910232544,
      "rewards/format_reward": 0.3333333432674408,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3737.4722900390625,
      "epoch": 0.06426735218508997,
      "grad_norm": 0.18189558386802673,
      "kl": 0.007110595703125,
      "learning_rate": 9.397114317029974e-07,
      "loss": 0.0809,
      "reward": -2.6944445371627808,
      "reward_std": 0.3134361654520035,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3442.9722900390625,
      "epoch": 0.06512425021422451,
      "grad_norm": 0.11082806438207626,
      "kl": 0.005889892578125,
      "learning_rate": 9.370671165529144e-07,
      "loss": 0.0364,
      "reward": -2.5277777910232544,
      "reward_std": 0.0680413767695427,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.472222238779068,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3204.166748046875,
      "epoch": 0.06598114824335904,
      "grad_norm": 0.1811019480228424,
      "kl": 0.007965087890625,
      "learning_rate": 9.343703987112365e-07,
      "loss": 0.1172,
      "reward": -2.5277777910232544,
      "reward_std": 0.37919675558805466,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4166666865348816,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3627.47216796875,
      "epoch": 0.06683804627249357,
      "grad_norm": 0.15231575071811676,
      "kl": 0.00232696533203125,
      "learning_rate": 9.316216432703916e-07,
      "loss": 0.0252,
      "reward": -2.694444417953491,
      "reward_std": 0.24017397314310074,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555559694767,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2848.416748046875,
      "epoch": 0.0676949443016281,
      "grad_norm": 0.2135927677154541,
      "kl": 0.0075836181640625,
      "learning_rate": 9.288212223678658e-07,
      "loss": 0.0625,
      "reward": -2.407407283782959,
      "reward_std": 0.44895367324352264,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3405.77783203125,
      "epoch": 0.06855184233076264,
      "grad_norm": 0.12736141681671143,
      "kl": 0.00734710693359375,
      "learning_rate": 9.259695151358214e-07,
      "loss": 0.0282,
      "reward": -2.638888955116272,
      "reward_std": 0.20412413775920868,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.361111119389534,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3254.0555419921875,
      "epoch": 0.06940874035989718,
      "grad_norm": 0.20807288587093353,
      "kl": 0.008697509765625,
      "learning_rate": 9.230669076497687e-07,
      "loss": 0.1148,
      "reward": -2.638888955116272,
      "reward_std": 0.20412413775920868,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3055555522441864,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3111.166748046875,
      "epoch": 0.0702656383890317,
      "grad_norm": 0.2130519151687622,
      "kl": 0.006744384765625,
      "learning_rate": 9.20113792876298e-07,
      "loss": 0.1215,
      "reward": -2.5,
      "reward_std": 0.18257419764995575,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000149011612,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2739.3333740234375,
      "epoch": 0.07112253641816624,
      "grad_norm": 0.2380674034357071,
      "kl": 0.007965087890625,
      "learning_rate": 9.171105706198774e-07,
      "loss": 0.0922,
      "reward": -2.2222222685813904,
      "reward_std": 0.31865695118904114,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.6111111342906952,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2850.4166259765625,
      "epoch": 0.07197943444730077,
      "grad_norm": 0.19383125007152557,
      "kl": 0.00946044921875,
      "learning_rate": 9.140576474687263e-07,
      "loss": 0.0087,
      "reward": -1.9166666865348816,
      "reward_std": 0.6418167352676392,
      "rewards/confidence_reward": -2.4444446563720703,
      "rewards/format_reward": 0.5277777910232544,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2711.52783203125,
      "epoch": 0.0728363324764353,
      "grad_norm": 0.23535026609897614,
      "kl": 0.004180908203125,
      "learning_rate": 9.109554367397697e-07,
      "loss": 0.1209,
      "reward": -2.333333373069763,
      "reward_std": 0.3412187397480011,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.6111111044883728,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3125.9722900390625,
      "epoch": 0.07369323050556983,
      "grad_norm": 0.07005209475755692,
      "kl": 0.010040283203125,
      "learning_rate": 9.078043584226815e-07,
      "loss": 0.0131,
      "reward": -2.583333373069763,
      "reward_std": 0.09128709882497787,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4166666567325592,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3320.0555419921875,
      "epoch": 0.07455012853470437,
      "grad_norm": 0.1450621485710144,
      "kl": 0.010986328125,
      "learning_rate": 9.046048391230247e-07,
      "loss": 0.0719,
      "reward": -2.5277777910232544,
      "reward_std": 0.20412413030862808,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4722222238779068,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3126.52783203125,
      "epoch": 0.07540702656383891,
      "grad_norm": 0.14595212042331696,
      "kl": 0.004852294921875,
      "learning_rate": 9.013573120044966e-07,
      "loss": 0.0592,
      "reward": -2.3518518209457397,
      "reward_std": 0.3861331045627594,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5555555671453476,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3651.5833740234375,
      "epoch": 0.07626392459297343,
      "grad_norm": 0.21023660898208618,
      "kl": 0.006317138671875,
      "learning_rate": 8.980622167302837e-07,
      "loss": 0.0916,
      "reward": -2.5740740299224854,
      "reward_std": 0.5071772634983063,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.3333333358168602,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3397.9722900390625,
      "epoch": 0.07712082262210797,
      "grad_norm": 0.188828244805336,
      "kl": 0.0087890625,
      "learning_rate": 8.9471999940354e-07,
      "loss": 0.0082,
      "reward": -2.2870370149612427,
      "reward_std": 0.40421056002378464,
      "rewards/confidence_reward": -2.7037036418914795,
      "rewards/format_reward": 0.4166666716337204,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2552.52783203125,
      "epoch": 0.0779777206512425,
      "grad_norm": 0.40647614002227783,
      "kl": 0.01544189453125,
      "learning_rate": 8.91331112506991e-07,
      "loss": 0.1232,
      "reward": -1.8888888955116272,
      "reward_std": 0.5773965641856194,
      "rewards/confidence_reward": -2.444444477558136,
      "rewards/format_reward": 0.5555555522441864,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2823.6944580078125,
      "epoch": 0.07883461868037704,
      "grad_norm": 0.12339317798614502,
      "kl": 0.0080413818359375,
      "learning_rate": 8.878960148416747e-07,
      "loss": 0.0168,
      "reward": -2.3611111640930176,
      "reward_std": 0.22736985236406326,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.638888880610466,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3384.52783203125,
      "epoch": 0.07969151670951156,
      "grad_norm": 0.29699084162712097,
      "kl": 0.00677490234375,
      "learning_rate": 8.844151714648274e-07,
      "loss": 0.1672,
      "reward": -2.5,
      "reward_std": 0.3814775347709656,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000149011612,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3287.25,
      "epoch": 0.0805484147386461,
      "grad_norm": 0.39855116605758667,
      "kl": 0.0213623046875,
      "learning_rate": 8.808890536269229e-07,
      "loss": 0.1707,
      "reward": -2.416666626930237,
      "reward_std": 0.3995024412870407,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5833333432674408,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2682.75,
      "epoch": 0.08140531276778064,
      "grad_norm": 0.391999751329422,
      "kl": 0.015533447265625,
      "learning_rate": 8.773181387078719e-07,
      "loss": 0.1891,
      "reward": -2.3611111640930176,
      "reward_std": 0.49601034820079803,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.5833333432674408,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3296.9722900390625,
      "epoch": 0.08226221079691516,
      "grad_norm": 0.22848033905029297,
      "kl": 0.0120391845703125,
      "learning_rate": 8.737029101523929e-07,
      "loss": 0.1268,
      "reward": -2.3240740299224854,
      "reward_std": 0.5216506272554398,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5833333432674408,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2389.4166870117188,
      "epoch": 0.0831191088260497,
      "grad_norm": 0.2833496034145355,
      "kl": 0.0255126953125,
      "learning_rate": 8.700438574045617e-07,
      "loss": 0.239,
      "reward": -2.0092591643333435,
      "reward_std": 0.788471519947052,
      "rewards/confidence_reward": -2.759259343147278,
      "rewards/format_reward": 0.75,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2865.9722900390625,
      "epoch": 0.08397600685518423,
      "grad_norm": 0.18639419972896576,
      "kl": 0.014617919921875,
      "learning_rate": 8.663414758415478e-07,
      "loss": 0.0585,
      "reward": -2.333333373069763,
      "reward_std": 0.3314610570669174,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6666666567325592,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2753.9444580078125,
      "epoch": 0.08483290488431877,
      "grad_norm": 0.26271870732307434,
      "kl": 0.021240234375,
      "learning_rate": 8.625962667065487e-07,
      "loss": 0.1419,
      "reward": -2.25,
      "reward_std": 0.41698990762233734,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.6944444477558136,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3196.52783203125,
      "epoch": 0.0856898029134533,
      "grad_norm": 0.2569756805896759,
      "kl": 0.027587890625,
      "learning_rate": 8.588087370409302e-07,
      "loss": 0.0869,
      "reward": -2.2129629254341125,
      "reward_std": 0.642838716506958,
      "rewards/confidence_reward": -2.574074149131775,
      "rewards/format_reward": 0.361111119389534,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3108.25,
      "epoch": 0.08654670094258783,
      "grad_norm": 0.221235990524292,
      "kl": 0.0069427490234375,
      "learning_rate": 8.549793996155795e-07,
      "loss": 0.1286,
      "reward": -2.3518518209457397,
      "reward_std": 0.5122157335281372,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5555555820465088,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3036.388916015625,
      "epoch": 0.08740359897172237,
      "grad_norm": 0.19860585033893585,
      "kl": 0.01312255859375,
      "learning_rate": 8.511087728614862e-07,
      "loss": 0.1027,
      "reward": -2.472222328186035,
      "reward_std": 0.4092601239681244,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4722222238779068,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3937.0833740234375,
      "epoch": 0.08826049700085689,
      "grad_norm": 0.18265724182128906,
      "kl": 0.0120849609375,
      "learning_rate": 8.471973807995534e-07,
      "loss": 0.0519,
      "reward": -2.805555582046509,
      "reward_std": 0.3134361505508423,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.19444444961845875,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2821.6944580078125,
      "epoch": 0.08911739502999143,
      "grad_norm": 0.1616680771112442,
      "kl": 0.013641357421875,
      "learning_rate": 8.432457529696548e-07,
      "loss": 0.0466,
      "reward": -2.240740656852722,
      "reward_std": 0.409378819167614,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.6666666567325592,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3072.27783203125,
      "epoch": 0.08997429305912596,
      "grad_norm": 0.27748724818229675,
      "kl": 0.008331298828125,
      "learning_rate": 8.392544243589427e-07,
      "loss": 0.1194,
      "reward": -2.305555582046509,
      "reward_std": 0.6821095794439316,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.5277777910232544,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3736.3333740234375,
      "epoch": 0.0908311910882605,
      "grad_norm": 0.19383865594863892,
      "kl": 0.014678955078125,
      "learning_rate": 8.352239353294194e-07,
      "loss": 0.0574,
      "reward": -2.5,
      "reward_std": 0.37219424545764923,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4444444477558136,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3856.583251953125,
      "epoch": 0.09168808911739502,
      "grad_norm": 0.17508208751678467,
      "kl": 0.0128173828125,
      "learning_rate": 8.31154831544782e-07,
      "loss": 0.0391,
      "reward": -2.6944445371627808,
      "reward_std": 0.24017397314310074,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.5555419921875,
      "epoch": 0.09254498714652956,
      "grad_norm": 0.15831252932548523,
      "kl": 0.017791748046875,
      "learning_rate": 8.270476638965461e-07,
      "loss": 0.0035,
      "reward": -2.518518567085266,
      "reward_std": 0.47219936549663544,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.3888889029622078,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3814.5555419921875,
      "epoch": 0.0934018851756641,
      "grad_norm": 0.2039646953344345,
      "kl": 0.012939453125,
      "learning_rate": 8.229029884294662e-07,
      "loss": 0.0669,
      "reward": -2.666666626930237,
      "reward_std": 0.3547067865729332,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333358168602,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3702.0,
      "epoch": 0.09425878320479864,
      "grad_norm": 0.19251351058483124,
      "kl": 0.00970458984375,
      "learning_rate": 8.187213662662538e-07,
      "loss": 0.0819,
      "reward": -2.5833332538604736,
      "reward_std": 0.3995024412870407,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.416666679084301,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3664.5833740234375,
      "epoch": 0.09511568123393316,
      "grad_norm": 0.10979479551315308,
      "kl": 0.013031005859375,
      "learning_rate": 8.145033635316128e-07,
      "loss": 0.0533,
      "reward": -2.75,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.25,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3430.5556640625,
      "epoch": 0.0959725792630677,
      "grad_norm": 0.21925541758537292,
      "kl": 0.012115478515625,
      "learning_rate": 8.102495512755938e-07,
      "loss": 0.0942,
      "reward": -2.5833332538604736,
      "reward_std": 0.2738612964749336,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.416666679084301,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3218.166748046875,
      "epoch": 0.09682947729220223,
      "grad_norm": 0.2610246241092682,
      "kl": 0.014556884765625,
      "learning_rate": 8.059605053962833e-07,
      "loss": 0.1001,
      "reward": -2.435185194015503,
      "reward_std": 0.39413511008024216,
      "rewards/confidence_reward": -2.8518519401550293,
      "rewards/format_reward": 0.4166666865348816,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3630.8612060546875,
      "epoch": 0.09768637532133675,
      "grad_norm": 0.14595678448677063,
      "kl": 0.011627197265625,
      "learning_rate": 8.01636806561836e-07,
      "loss": 0.0598,
      "reward": -2.555555582046509,
      "reward_std": 0.1360827535390854,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444477558136,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3384.3055419921875,
      "epoch": 0.09854327335047129,
      "grad_norm": 0.1648566722869873,
      "kl": 0.02392578125,
      "learning_rate": 7.972790401318627e-07,
      "loss": 0.103,
      "reward": -2.6388888359069824,
      "reward_std": 0.2901904359459877,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.30555556900799274,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3751.388916015625,
      "epoch": 0.09940017137960583,
      "grad_norm": 0.21332553029060364,
      "kl": 0.02044677734375,
      "learning_rate": 7.928877960781808e-07,
      "loss": 0.1117,
      "reward": -2.75,
      "reward_std": 0.3134361356496811,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2500000037252903,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3072.2501220703125,
      "epoch": 0.10025706940874037,
      "grad_norm": 0.1085459515452385,
      "kl": 0.02020263671875,
      "learning_rate": 7.884636689049422e-07,
      "loss": 0.0629,
      "reward": -2.2685184478759766,
      "reward_std": 0.2494850754737854,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.6388888955116272,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3123.9722900390625,
      "epoch": 0.10111396743787489,
      "grad_norm": 0.16286125779151917,
      "kl": 0.0162353515625,
      "learning_rate": 7.840072575681468e-07,
      "loss": 0.0787,
      "reward": -2.4452918767929077,
      "reward_std": 0.4294184446334839,
      "rewards/confidence_reward": -2.889736294746399,
      "rewards/format_reward": 0.4444444626569748,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3960.3333740234375,
      "epoch": 0.10197086546700942,
      "grad_norm": 0.15160124003887177,
      "kl": 0.01416015625,
      "learning_rate": 7.795191653945538e-07,
      "loss": 0.0512,
      "reward": -2.805555582046509,
      "reward_std": 0.32624027132987976,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1944444477558136,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3328.9722900390625,
      "epoch": 0.10282776349614396,
      "grad_norm": 0.1655929535627365,
      "kl": 0.01312255859375,
      "learning_rate": 7.75e-07,
      "loss": 0.0925,
      "reward": -2.4722222089767456,
      "reward_std": 0.3134361580014229,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777910232544,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3596.25,
      "epoch": 0.1036846615252785,
      "grad_norm": 0.08225465565919876,
      "kl": 0.011199951171875,
      "learning_rate": 7.704503732071391e-07,
      "loss": 0.0074,
      "reward": -2.6111111640930176,
      "reward_std": 0.08606629818677902,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3888888955116272,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3386.77783203125,
      "epoch": 0.10454155955441302,
      "grad_norm": 0.14785455167293549,
      "kl": 0.02099609375,
      "learning_rate": 7.658709009626109e-07,
      "loss": 0.0373,
      "reward": -2.6944445371627808,
      "reward_std": 0.24017397314310074,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555671453476,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3158.27783203125,
      "epoch": 0.10539845758354756,
      "grad_norm": 0.22167643904685974,
      "kl": 0.02313232421875,
      "learning_rate": 7.612622032536507e-07,
      "loss": 0.0777,
      "reward": -2.444444417953491,
      "reward_std": 0.3582318276166916,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.5000000149011612,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2636.861083984375,
      "epoch": 0.1062553556126821,
      "grad_norm": 0.2819192707538605,
      "kl": 0.01617431640625,
      "learning_rate": 7.566249040241553e-07,
      "loss": -0.0007,
      "reward": -1.9629629850387573,
      "reward_std": 0.656702309846878,
      "rewards/confidence_reward": -2.574074149131775,
      "rewards/format_reward": 0.6111111044883728,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3987.638916015625,
      "epoch": 0.10711225364181662,
      "grad_norm": 0.15799356997013092,
      "kl": 0.0233154296875,
      "learning_rate": 7.51959631090208e-07,
      "loss": 0.0367,
      "reward": -2.833333373069763,
      "reward_std": 0.245394766330719,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1666666716337204,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3636.0001220703125,
      "epoch": 0.10796915167095116,
      "grad_norm": 0.20148231089115143,
      "kl": 0.012664794921875,
      "learning_rate": 7.472670160550848e-07,
      "loss": 0.1046,
      "reward": -2.555555582046509,
      "reward_std": 0.3814775347709656,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444477558136,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3470.388916015625,
      "epoch": 0.10882604970008569,
      "grad_norm": 0.29233935475349426,
      "kl": 0.018310546875,
      "learning_rate": 7.425476942237444e-07,
      "loss": 0.0973,
      "reward": -2.5407216548919678,
      "reward_std": 0.5416866987943649,
      "rewards/confidence_reward": -2.8462772369384766,
      "rewards/format_reward": 0.305555559694767,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3344.111083984375,
      "epoch": 0.10968294772922023,
      "grad_norm": 0.12020324170589447,
      "kl": 0.02020263671875,
      "learning_rate": 7.37802304516818e-07,
      "loss": 0.025,
      "reward": -2.583333373069763,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4166666716337204,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3881.1944580078125,
      "epoch": 0.11053984575835475,
      "grad_norm": 0.17918746173381805,
      "kl": 0.01629638671875,
      "learning_rate": 7.330314893841101e-07,
      "loss": 0.0508,
      "reward": -2.6944445371627808,
      "reward_std": 0.3995024263858795,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2966.388916015625,
      "epoch": 0.11139674378748929,
      "grad_norm": 0.2491733282804489,
      "kl": 0.02984619140625,
      "learning_rate": 7.282358947176205e-07,
      "loss": 0.075,
      "reward": -2.3425925970077515,
      "reward_std": 0.514257550239563,
      "rewards/confidence_reward": -2.814814805984497,
      "rewards/format_reward": 0.472222238779068,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3056.888916015625,
      "epoch": 0.11225364181662383,
      "grad_norm": 0.1091020330786705,
      "kl": 0.018524169921875,
      "learning_rate": 7.234161697641017e-07,
      "loss": 0.0298,
      "reward": -2.5008474588394165,
      "reward_std": 0.35615625977516174,
      "rewards/confidence_reward": -2.889736294746399,
      "rewards/format_reward": 0.3888888955116272,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3280.1944580078125,
      "epoch": 0.11311053984575835,
      "grad_norm": 0.22998450696468353,
      "kl": 0.016815185546875,
      "learning_rate": 7.185729670371604e-07,
      "loss": 0.0906,
      "reward": -2.555555582046509,
      "reward_std": 0.3082153648138046,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3888888955116272,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3573.75,
      "epoch": 0.11396743787489289,
      "grad_norm": 0.17413660883903503,
      "kl": 0.018310546875,
      "learning_rate": 7.137069422289181e-07,
      "loss": 0.089,
      "reward": -2.638888955116272,
      "reward_std": 0.3134361431002617,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.361111119389534,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3063.0555419921875,
      "epoch": 0.11482433590402742,
      "grad_norm": 0.214571014046669,
      "kl": 0.03424072265625,
      "learning_rate": 7.08818754121241e-07,
      "loss": 0.039,
      "reward": -2.2314815521240234,
      "reward_std": 0.4910118132829666,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.5277777910232544,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3406.6112060546875,
      "epoch": 0.11568123393316196,
      "grad_norm": 0.20651200413703918,
      "kl": 0.026092529296875,
      "learning_rate": 7.039090644965509e-07,
      "loss": 0.0338,
      "reward": -2.296296238899231,
      "reward_std": 0.5582656562328339,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.6111111044883728,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3469.6944580078125,
      "epoch": 0.11653813196229648,
      "grad_norm": 0.12644833326339722,
      "kl": 0.01922607421875,
      "learning_rate": 6.989785380482312e-07,
      "loss": 0.0342,
      "reward": -2.6944445371627808,
      "reward_std": 0.1541076824069023,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3475.4166259765625,
      "epoch": 0.11739502999143102,
      "grad_norm": 0.2094379961490631,
      "kl": 0.02838134765625,
      "learning_rate": 6.940278422906372e-07,
      "loss": 0.1098,
      "reward": -2.5,
      "reward_std": 0.5836373865604401,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.3888888955116272,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3328.0,
      "epoch": 0.11825192802056556,
      "grad_norm": 0.26488515734672546,
      "kl": 0.019317626953125,
      "learning_rate": 6.890576474687263e-07,
      "loss": 0.0928,
      "reward": -2.260106682777405,
      "reward_std": 0.5154568552970886,
      "rewards/confidence_reward": -2.704551100730896,
      "rewards/format_reward": 0.4444444626569748,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2500.5833740234375,
      "epoch": 0.11910882604970009,
      "grad_norm": 0.22615168988704681,
      "kl": 0.0201416015625,
      "learning_rate": 6.840686264673168e-07,
      "loss": 0.0361,
      "reward": -2.000847280025482,
      "reward_std": 0.6050563156604767,
      "rewards/confidence_reward": -2.5564029216766357,
      "rewards/format_reward": 0.5555555671453476,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3513.1112060546875,
      "epoch": 0.11996572407883462,
      "grad_norm": 0.3215450644493103,
      "kl": 0.02825927734375,
      "learning_rate": 6.790614547199906e-07,
      "loss": 0.1226,
      "reward": -2.555555582046509,
      "reward_std": 0.41752736270427704,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.4444444552063942,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3207.638916015625,
      "epoch": 0.12082262210796915,
      "grad_norm": 0.2798071503639221,
      "kl": 0.02069091796875,
      "learning_rate": 6.740368101176495e-07,
      "loss": 0.0961,
      "reward": -2.2870370149612427,
      "reward_std": 0.6775233149528503,
      "rewards/confidence_reward": -2.6481481790542603,
      "rewards/format_reward": 0.3611111119389534,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3339.1666259765625,
      "epoch": 0.12167952013710369,
      "grad_norm": 0.26966410875320435,
      "kl": 0.02276611328125,
      "learning_rate": 6.68995372916741e-07,
      "loss": 0.0981,
      "reward": -2.3518519401550293,
      "reward_std": 0.44895367324352264,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.5555555522441864,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3242.3055419921875,
      "epoch": 0.12253641816623821,
      "grad_norm": 0.2671264111995697,
      "kl": 0.02325439453125,
      "learning_rate": 6.639378256471608e-07,
      "loss": 0.0918,
      "reward": -2.3611111640930176,
      "reward_std": 0.4583979994058609,
      "rewards/confidence_reward": -2.722222328186035,
      "rewards/format_reward": 0.361111119389534,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2081.5,
      "epoch": 0.12339331619537275,
      "grad_norm": 0.25654128193855286,
      "kl": 0.02740478515625,
      "learning_rate": 6.588648530198504e-07,
      "loss": 0.0977,
      "reward": -1.8148147463798523,
      "reward_std": 0.7522204518318176,
      "rewards/confidence_reward": -2.64814829826355,
      "rewards/format_reward": 0.8333333432674408,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3736.9166259765625,
      "epoch": 0.12425021422450729,
      "grad_norm": 0.12350251525640488,
      "kl": 0.02294921875,
      "learning_rate": 6.537771418340981e-07,
      "loss": 0.0331,
      "reward": -2.7222222089767456,
      "reward_std": 0.2221490517258644,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.27777777798473835,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2392.7222900390625,
      "epoch": 0.12510711225364182,
      "grad_norm": 0.5596352815628052,
      "kl": 0.03955078125,
      "learning_rate": 6.486753808845564e-07,
      "loss": 0.2211,
      "reward": -1.8425925374031067,
      "reward_std": 0.754194438457489,
      "rewards/confidence_reward": -2.6481481790542603,
      "rewards/format_reward": 0.8055555820465088,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2694.5001220703125,
      "epoch": 0.12596401028277635,
      "grad_norm": 0.0685691088438034,
      "kl": 0.02557373046875,
      "learning_rate": 6.435602608679916e-07,
      "loss": 0.0014,
      "reward": -2.305555582046509,
      "reward_std": 0.0680413767695427,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6944444477558136,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2063.6111450195312,
      "epoch": 0.1268209083119109,
      "grad_norm": 0.34025242924690247,
      "kl": 0.03564453125,
      "learning_rate": 6.384324742897735e-07,
      "loss": 0.1248,
      "reward": -2.0092591047286987,
      "reward_std": 0.7437996864318848,
      "rewards/confidence_reward": -2.759259343147278,
      "rewards/format_reward": 0.75,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3310.27783203125,
      "epoch": 0.12767780634104542,
      "grad_norm": 0.17515520751476288,
      "kl": 0.02813720703125,
      "learning_rate": 6.332927153701215e-07,
      "loss": 0.105,
      "reward": -2.3518518805503845,
      "reward_std": 0.44590386003255844,
      "rewards/confidence_reward": -2.8518519401550293,
      "rewards/format_reward": 0.5000000074505806,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3576.0833740234375,
      "epoch": 0.12853470437017994,
      "grad_norm": 0.540116012096405,
      "kl": 0.0565185546875,
      "learning_rate": 6.281416799501187e-07,
      "loss": 0.0537,
      "reward": -2.638888955116272,
      "reward_std": 0.3134361505508423,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3611111119389534,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2819.3056640625,
      "epoch": 0.1293916023993145,
      "grad_norm": 0.3199455440044403,
      "kl": 0.032958984375,
      "learning_rate": 6.229800653975054e-07,
      "loss": 0.1061,
      "reward": -2.2314813137054443,
      "reward_std": 0.6937831938266754,
      "rewards/confidence_reward": -2.814814805984497,
      "rewards/format_reward": 0.5833333432674408,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2036.6388854980469,
      "epoch": 0.13024850042844902,
      "grad_norm": 0.25735169649124146,
      "kl": 0.02972412109375,
      "learning_rate": 6.178085705122674e-07,
      "loss": 0.023,
      "reward": -1.5749215185642242,
      "reward_std": 1.027258962392807,
      "rewards/confidence_reward": -2.3526992201805115,
      "rewards/format_reward": 0.7777777910232544,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3299.5833740234375,
      "epoch": 0.13110539845758354,
      "grad_norm": 0.14086535573005676,
      "kl": 0.0323486328125,
      "learning_rate": 6.126278954320294e-07,
      "loss": 0.0307,
      "reward": -2.42592591047287,
      "reward_std": 0.4229704663157463,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.38888888247311115,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3406.5001220703125,
      "epoch": 0.1319622964867181,
      "grad_norm": 0.258766233921051,
      "kl": 0.03106689453125,
      "learning_rate": 6.074387415372676e-07,
      "loss": 0.1326,
      "reward": -2.5000001192092896,
      "reward_std": 0.30821535736322403,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000074505806,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3476.388916015625,
      "epoch": 0.1328191945158526,
      "grad_norm": 0.20013348758220673,
      "kl": 0.02679443359375,
      "learning_rate": 6.022418113563535e-07,
      "loss": 0.0258,
      "reward": -2.472645878791809,
      "reward_std": 0.39784130454063416,
      "rewards/confidence_reward": -2.6948680877685547,
      "rewards/format_reward": 0.22222222574055195,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3603.3612060546875,
      "epoch": 0.13367609254498714,
      "grad_norm": 0.31663545966148376,
      "kl": 0.027587890625,
      "learning_rate": 5.97037808470444e-07,
      "loss": 0.1284,
      "reward": -2.4444445371627808,
      "reward_std": 0.7150397002696991,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.4444444626569748,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2918.666748046875,
      "epoch": 0.13453299057412169,
      "grad_norm": 0.2475612312555313,
      "kl": 0.031494140625,
      "learning_rate": 5.918274374182266e-07,
      "loss": 0.0837,
      "reward": -2.083333432674408,
      "reward_std": 0.5944807976484299,
      "rewards/confidence_reward": -2.666666626930237,
      "rewards/format_reward": 0.5833333432674408,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2515.8612060546875,
      "epoch": 0.1353898886032562,
      "grad_norm": 0.5117289423942566,
      "kl": 0.0574951171875,
      "learning_rate": 5.866114036005362e-07,
      "loss": 0.2035,
      "reward": -1.8888888955116272,
      "reward_std": 0.8281392157077789,
      "rewards/confidence_reward": -2.611111044883728,
      "rewards/format_reward": 0.7222222089767456,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3296.861083984375,
      "epoch": 0.13624678663239073,
      "grad_norm": 0.28699225187301636,
      "kl": 0.02996826171875,
      "learning_rate": 5.813904131848564e-07,
      "loss": 0.1256,
      "reward": -2.4629629850387573,
      "reward_std": 0.4850034862756729,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.4444444626569748,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3864.2777099609375,
      "epoch": 0.13710368466152528,
      "grad_norm": 0.21943415701389313,
      "kl": 0.0389404296875,
      "learning_rate": 5.761651730097142e-07,
      "loss": 0.0667,
      "reward": -2.6944445371627808,
      "reward_std": 0.3995024561882019,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3015.02783203125,
      "epoch": 0.1379605826906598,
      "grad_norm": 0.22274136543273926,
      "kl": 0.03118896484375,
      "learning_rate": 5.709363904889861e-07,
      "loss": 0.0465,
      "reward": -2.3518518209457397,
      "reward_std": 0.47685495018959045,
      "rewards/confidence_reward": -2.7962963581085205,
      "rewards/format_reward": 0.4444444477558136,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3587.5,
      "epoch": 0.13881748071979436,
      "grad_norm": 0.1837625652551651,
      "kl": 0.042724609375,
      "learning_rate": 5.657047735161255e-07,
      "loss": 0.0174,
      "reward": -2.5000001192092896,
      "reward_std": 0.42796894907951355,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.3333333432674408,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3590.4444580078125,
      "epoch": 0.13967437874892888,
      "grad_norm": 0.41009289026260376,
      "kl": 0.0537109375,
      "learning_rate": 5.604710303683253e-07,
      "loss": 0.1195,
      "reward": -2.4259259700775146,
      "reward_std": 0.5152682885527611,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.3333333432674408,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3338.5,
      "epoch": 0.1405312767780634,
      "grad_norm": 0.22997497022151947,
      "kl": 0.034423828125,
      "learning_rate": 5.552358696106288e-07,
      "loss": 0.1118,
      "reward": -2.4907407760620117,
      "reward_std": 0.5448963046073914,
      "rewards/confidence_reward": -2.7962963581085205,
      "rewards/format_reward": 0.305555559694767,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3163.6944580078125,
      "epoch": 0.14138817480719795,
      "grad_norm": 0.29789188504219055,
      "kl": 0.0311279296875,
      "learning_rate": 5.5e-07,
      "loss": 0.0972,
      "reward": -2.2870370149612427,
      "reward_std": 0.5160937160253525,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.4722222238779068,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2695.3333740234375,
      "epoch": 0.14224507283633248,
      "grad_norm": 0.25649595260620117,
      "kl": 0.02960205078125,
      "learning_rate": 5.447641303893714e-07,
      "loss": 0.0524,
      "reward": -1.7499999403953552,
      "reward_std": 0.6183541268110275,
      "rewards/confidence_reward": -2.3333334922790527,
      "rewards/format_reward": 0.5833333432674408,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2688.6944580078125,
      "epoch": 0.143101970865467,
      "grad_norm": 0.23350609838962555,
      "kl": 0.037353515625,
      "learning_rate": 5.395289696316747e-07,
      "loss": 0.0783,
      "reward": -2.1582546830177307,
      "reward_std": 0.7079941481351852,
      "rewards/confidence_reward": -2.7415881156921387,
      "rewards/format_reward": 0.5833333358168602,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3119.22216796875,
      "epoch": 0.14395886889460155,
      "grad_norm": 0.3477858006954193,
      "kl": 0.0482177734375,
      "learning_rate": 5.342952264838747e-07,
      "loss": 0.0932,
      "reward": -2.129629611968994,
      "reward_std": 0.7057251632213593,
      "rewards/confidence_reward": -2.7407407760620117,
      "rewards/format_reward": 0.6111111044883728,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2954.166748046875,
      "epoch": 0.14481576692373607,
      "grad_norm": 0.16720406711101532,
      "kl": 0.03704833984375,
      "learning_rate": 5.29063609511014e-07,
      "loss": 0.0403,
      "reward": -2.1481481194496155,
      "reward_std": 0.20082136243581772,
      "rewards/confidence_reward": -2.5925925970077515,
      "rewards/format_reward": 0.4444444626569748,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3575.3333740234375,
      "epoch": 0.1456726649528706,
      "grad_norm": 0.3258434534072876,
      "kl": 0.0556640625,
      "learning_rate": 5.238348269902859e-07,
      "loss": 0.1092,
      "reward": -2.305555522441864,
      "reward_std": 0.49953543394804,
      "rewards/confidence_reward": -2.7222222089767456,
      "rewards/format_reward": 0.416666679084301,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3646.138916015625,
      "epoch": 0.14652956298200515,
      "grad_norm": 0.22214293479919434,
      "kl": 0.0496826171875,
      "learning_rate": 5.186095868151436e-07,
      "loss": 0.0377,
      "reward": -2.6111111640930176,
      "reward_std": 0.222149059176445,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3888888955116272,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2912.25,
      "epoch": 0.14738646101113967,
      "grad_norm": 0.24840864539146423,
      "kl": 0.0555419921875,
      "learning_rate": 5.133885963994639e-07,
      "loss": 0.0503,
      "reward": -1.9814814329147339,
      "reward_std": 0.8633435666561127,
      "rewards/confidence_reward": -2.481481671333313,
      "rewards/format_reward": 0.5,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3765.916748046875,
      "epoch": 0.14824335904027422,
      "grad_norm": 0.1615857183933258,
      "kl": 0.05029296875,
      "learning_rate": 5.081725625817735e-07,
      "loss": 0.0285,
      "reward": -2.6666667461395264,
      "reward_std": 0.17213259637355804,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4005.888916015625,
      "epoch": 0.14910025706940874,
      "grad_norm": 0.124207504093647,
      "kl": 0.042724609375,
      "learning_rate": 5.02962191529556e-07,
      "loss": 0.0302,
      "reward": -2.8611111640930176,
      "reward_std": 0.15410767495632172,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1388888955116272,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3589.7222900390625,
      "epoch": 0.14995715509854327,
      "grad_norm": 0.38185247778892517,
      "kl": 0.0694580078125,
      "learning_rate": 4.977581886436462e-07,
      "loss": 0.0932,
      "reward": -2.5740740299224854,
      "reward_std": 0.472199410200119,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.33333333022892475,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3721.1666259765625,
      "epoch": 0.15081405312767782,
      "grad_norm": 0.35303542017936707,
      "kl": 0.0589599609375,
      "learning_rate": 4.925612584627324e-07,
      "loss": 0.0818,
      "reward": -2.722222328186035,
      "reward_std": 0.3082153648138046,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2777777910232544,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3804.9444580078125,
      "epoch": 0.15167095115681234,
      "grad_norm": 0.37888699769973755,
      "kl": 0.0810546875,
      "learning_rate": 4.873721045679706e-07,
      "loss": 0.0564,
      "reward": -2.7777777910232544,
      "reward_std": 0.2221490517258644,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.22222222574055195,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2334.8056640625,
      "epoch": 0.15252784918594686,
      "grad_norm": 0.49078041315078735,
      "kl": 0.076171875,
      "learning_rate": 4.821914294877326e-07,
      "loss": 0.0592,
      "reward": -1.5222030878067017,
      "reward_std": 0.7962049543857574,
      "rewards/confidence_reward": -2.2722031474113464,
      "rewards/format_reward": 0.75,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3370.944580078125,
      "epoch": 0.1533847472150814,
      "grad_norm": 0.2731315791606903,
      "kl": 0.07763671875,
      "learning_rate": 4.770199346024947e-07,
      "loss": 0.0635,
      "reward": -2.2685184478759766,
      "reward_std": 0.6701640486717224,
      "rewards/confidence_reward": -2.6296297311782837,
      "rewards/format_reward": 0.361111119389534,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3099.888916015625,
      "epoch": 0.15424164524421594,
      "grad_norm": 0.24881142377853394,
      "kl": 0.0810546875,
      "learning_rate": 4.7185832004988133e-07,
      "loss": 0.0609,
      "reward": -2.3072502613067627,
      "reward_std": 0.5294433236122131,
      "rewards/confidence_reward": -2.6683614253997803,
      "rewards/format_reward": 0.36111112125217915,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3101.72216796875,
      "epoch": 0.15509854327335046,
      "grad_norm": 1.0561292171478271,
      "kl": 0.08544921875,
      "learning_rate": 4.667072846298785e-07,
      "loss": 0.2226,
      "reward": -2.259259283542633,
      "reward_std": 0.7795112058520317,
      "rewards/confidence_reward": -2.703703761100769,
      "rewards/format_reward": 0.4444444514811039,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3213.9444580078125,
      "epoch": 0.155955441302485,
      "grad_norm": 1.2878997325897217,
      "kl": 0.116455078125,
      "learning_rate": 4.6156752571022637e-07,
      "loss": 0.1861,
      "reward": -2.1574074029922485,
      "reward_std": 0.556253120303154,
      "rewards/confidence_reward": -2.6296297311782837,
      "rewards/format_reward": 0.472222238779068,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3509.138916015625,
      "epoch": 0.15681233933161953,
      "grad_norm": 0.22494396567344666,
      "kl": 0.082763671875,
      "learning_rate": 4.5643973913200837e-07,
      "loss": 0.0309,
      "reward": -2.453703761100769,
      "reward_std": 0.44099532812833786,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.361111119389534,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2710.97216796875,
      "epoch": 0.15766923736075408,
      "grad_norm": 0.5510160326957703,
      "kl": 0.1025390625,
      "learning_rate": 4.513246191154434e-07,
      "loss": 0.0585,
      "reward": -1.4814814329147339,
      "reward_std": 1.0165367126464844,
      "rewards/confidence_reward": -2.2037038803100586,
      "rewards/format_reward": 0.7222222089767456,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2879.8333740234375,
      "epoch": 0.1585261353898886,
      "grad_norm": 0.36165350675582886,
      "kl": 0.10498046875,
      "learning_rate": 4.4622285816590186e-07,
      "loss": 0.0904,
      "reward": -2.0740740299224854,
      "reward_std": 0.648275762796402,
      "rewards/confidence_reward": -2.7407408952713013,
      "rewards/format_reward": 0.6666666865348816,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3437.3612060546875,
      "epoch": 0.15938303341902313,
      "grad_norm": 0.8590340614318848,
      "kl": 0.116455078125,
      "learning_rate": 4.4113514698014953e-07,
      "loss": 0.1109,
      "reward": -2.472222328186035,
      "reward_std": 0.5155959874391556,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.4166666641831398,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2466.5,
      "epoch": 0.16023993144815768,
      "grad_norm": 1.367401123046875,
      "kl": 0.115234375,
      "learning_rate": 4.360621743528392e-07,
      "loss": 0.1976,
      "reward": -2.037884473800659,
      "reward_std": 0.6841873377561569,
      "rewards/confidence_reward": -2.6489956378936768,
      "rewards/format_reward": 0.6111111044883728,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3534.27783203125,
      "epoch": 0.1610968294772922,
      "grad_norm": 0.3216284215450287,
      "kl": 0.12939453125,
      "learning_rate": 4.3100462708325914e-07,
      "loss": 0.082,
      "reward": -2.4722222089767456,
      "reward_std": 0.3762567266821861,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.36111112125217915,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3046.138916015625,
      "epoch": 0.16195372750642673,
      "grad_norm": 0.9523129463195801,
      "kl": 0.142578125,
      "learning_rate": 4.2596318988235037e-07,
      "loss": 0.1065,
      "reward": -2.120370388031006,
      "reward_std": 0.5169950574636459,
      "rewards/confidence_reward": -2.5370370149612427,
      "rewards/format_reward": 0.4166666716337204,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3720.416748046875,
      "epoch": 0.16281062553556128,
      "grad_norm": 3.160400629043579,
      "kl": 0.734375,
      "learning_rate": 4.209385452800095e-07,
      "loss": 0.103,
      "reward": -2.6944445371627808,
      "reward_std": 0.32624026387929916,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555671453476,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2635.75,
      "epoch": 0.1636675235646958,
      "grad_norm": 0.7508611679077148,
      "kl": 0.119384765625,
      "learning_rate": 4.1593137353268303e-07,
      "loss": 0.1062,
      "reward": -1.9452918767929077,
      "reward_std": 0.717644564807415,
      "rewards/confidence_reward": -2.7786251306533813,
      "rewards/format_reward": 0.8333333432674408,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3603.5277099609375,
      "epoch": 0.16452442159383032,
      "grad_norm": 0.4415644109249115,
      "kl": 0.17431640625,
      "learning_rate": 4.1094235253127374e-07,
      "loss": 0.0745,
      "reward": -2.5000001192092896,
      "reward_std": 0.34894853830337524,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.444444440305233,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3744.3612060546875,
      "epoch": 0.16538131962296487,
      "grad_norm": 0.7602421045303345,
      "kl": 0.2197265625,
      "learning_rate": 4.059721577093628e-07,
      "loss": 0.1049,
      "reward": -2.60185170173645,
      "reward_std": 0.5628926083445549,
      "rewards/confidence_reward": -2.8518519401550293,
      "rewards/format_reward": 0.2500000074505806,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3876.5833740234375,
      "epoch": 0.1662382176520994,
      "grad_norm": 0.7295367121696472,
      "kl": 0.2158203125,
      "learning_rate": 4.0102146195176887e-07,
      "loss": 0.0704,
      "reward": -2.6111111640930176,
      "reward_std": 0.5082770437002182,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.3333333432674408,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2431.638916015625,
      "epoch": 0.16709511568123395,
      "grad_norm": 0.7587151527404785,
      "kl": 0.18994140625,
      "learning_rate": 3.9609093550344907e-07,
      "loss": 0.0699,
      "reward": -1.851851761341095,
      "reward_std": 0.8921651542186737,
      "rewards/confidence_reward": -2.574074149131775,
      "rewards/format_reward": 0.722222238779068,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3446.888916015625,
      "epoch": 0.16795201371036847,
      "grad_norm": 1.1345640420913696,
      "kl": 0.22265625,
      "learning_rate": 3.911812458787591e-07,
      "loss": 0.0911,
      "reward": -2.407407283782959,
      "reward_std": 0.4854222238063812,
      "rewards/confidence_reward": -2.8518519401550293,
      "rewards/format_reward": 0.4444444626569748,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3630.3612060546875,
      "epoch": 0.168808911739503,
      "grad_norm": 1.328270673751831,
      "kl": 0.26318359375,
      "learning_rate": 3.86293057771082e-07,
      "loss": 0.158,
      "reward": -2.6018518209457397,
      "reward_std": 0.5880227982997894,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.305555559694767,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3384.02783203125,
      "epoch": 0.16966580976863754,
      "grad_norm": 0.4739760160446167,
      "kl": 0.2294921875,
      "learning_rate": 3.8142703296283953e-07,
      "loss": 0.0634,
      "reward": -2.5,
      "reward_std": 0.222149059176445,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5000000149011612,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3382.8055419921875,
      "epoch": 0.17052270779777207,
      "grad_norm": 1.6106473207473755,
      "kl": 0.306640625,
      "learning_rate": 3.7658383023589833e-07,
      "loss": 0.1537,
      "reward": -2.3333333134651184,
      "reward_std": 0.6125056594610214,
      "rewards/confidence_reward": -2.7222222089767456,
      "rewards/format_reward": 0.3888888880610466,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3306.388916015625,
      "epoch": 0.1713796058269066,
      "grad_norm": 1.5265542268753052,
      "kl": 0.2734375,
      "learning_rate": 3.7176410528237945e-07,
      "loss": 0.1782,
      "reward": -2.1851852536201477,
      "reward_std": 0.5692443251609802,
      "rewards/confidence_reward": -2.629629611968994,
      "rewards/format_reward": 0.4444444552063942,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3631.8612060546875,
      "epoch": 0.17223650385604114,
      "grad_norm": 0.6827831864356995,
      "kl": 0.2685546875,
      "learning_rate": 3.6696851061588994e-07,
      "loss": 0.0805,
      "reward": -2.4814815521240234,
      "reward_std": 0.5552610009908676,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.3333333432674408,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2938.4166259765625,
      "epoch": 0.17309340188517566,
      "grad_norm": 0.9321132898330688,
      "kl": 0.30078125,
      "learning_rate": 3.62197695483182e-07,
      "loss": 0.0521,
      "reward": -1.944444477558136,
      "reward_std": 0.6303547322750092,
      "rewards/confidence_reward": -2.3888888359069824,
      "rewards/format_reward": 0.4444444626569748,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3787.6112060546875,
      "epoch": 0.17395029991431019,
      "grad_norm": 0.8994116187095642,
      "kl": 0.31640625,
      "learning_rate": 3.5745230577625573e-07,
      "loss": 0.0792,
      "reward": -2.75,
      "reward_std": 0.2901904284954071,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2500000037252903,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2426.9444580078125,
      "epoch": 0.17480719794344474,
      "grad_norm": 1.54979407787323,
      "kl": 0.265625,
      "learning_rate": 3.5273298394491515e-07,
      "loss": 0.0357,
      "reward": -1.6300533711910248,
      "reward_std": 0.7251835763454437,
      "rewards/confidence_reward": -2.3800532817840576,
      "rewards/format_reward": 0.75,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3264.6112060546875,
      "epoch": 0.17566409597257926,
      "grad_norm": 2.0934314727783203,
      "kl": 0.22802734375,
      "learning_rate": 3.4804036890979205e-07,
      "loss": 0.1002,
      "reward": -2.2971436977386475,
      "reward_std": 0.725425198674202,
      "rewards/confidence_reward": -2.7415882349014282,
      "rewards/format_reward": 0.4444444477558136,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3399.5001220703125,
      "epoch": 0.17652099400171378,
      "grad_norm": 0.9074891209602356,
      "kl": 0.2763671875,
      "learning_rate": 3.433750959758446e-07,
      "loss": 0.0878,
      "reward": -2.287036955356598,
      "reward_std": 0.3180917277932167,
      "rewards/confidence_reward": -2.5370370149612427,
      "rewards/format_reward": 0.2500000074505806,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 4017.02783203125,
      "epoch": 0.17737789203084833,
      "grad_norm": 0.922761857509613,
      "kl": 0.2080078125,
      "learning_rate": 3.387377967463493e-07,
      "loss": 0.0294,
      "reward": -2.638888955116272,
      "reward_std": 0.3995024412870407,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.361111119389534,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3856.416748046875,
      "epoch": 0.17823479005998286,
      "grad_norm": 1.1995811462402344,
      "kl": 0.3154296875,
      "learning_rate": 3.3412909903738936e-07,
      "loss": 0.078,
      "reward": -2.7777777910232544,
      "reward_std": 0.30821534991264343,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2222222313284874,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2836.25,
      "epoch": 0.1790916880891174,
      "grad_norm": 1.958234429359436,
      "kl": 0.138671875,
      "learning_rate": 3.295496267928609e-07,
      "loss": 0.078,
      "reward": -1.870370328426361,
      "reward_std": 0.5447776764631271,
      "rewards/confidence_reward": -2.4814815521240234,
      "rewards/format_reward": 0.6111111342906952,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3240.2777099609375,
      "epoch": 0.17994858611825193,
      "grad_norm": 6.24537992477417,
      "kl": 0.2060546875,
      "learning_rate": 3.250000000000001e-07,
      "loss": 0.1677,
      "reward": -2.046296238899231,
      "reward_std": 0.6235695481300354,
      "rewards/confidence_reward": -2.6296297311782837,
      "rewards/format_reward": 0.5833333432674408,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3614.166748046875,
      "epoch": 0.18080548414738645,
      "grad_norm": 0.7317115068435669,
      "kl": 0.1962890625,
      "learning_rate": 3.204808346054461e-07,
      "loss": 0.0303,
      "reward": -2.694444417953491,
      "reward_std": 0.22736986726522446,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.2500000074505806,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3610.52783203125,
      "epoch": 0.181662382176521,
      "grad_norm": 1.1794308423995972,
      "kl": 0.22119140625,
      "learning_rate": 3.159927424318531e-07,
      "loss": 0.0538,
      "reward": -2.666666626930237,
      "reward_std": 0.2453947737812996,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333358168602,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3340.8333740234375,
      "epoch": 0.18251928020565553,
      "grad_norm": 1.3902448415756226,
      "kl": 0.216796875,
      "learning_rate": 3.115363310950578e-07,
      "loss": 0.0582,
      "reward": -2.5277777910232544,
      "reward_std": 0.35992757976055145,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.4166666865348816,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3454.6666259765625,
      "epoch": 0.18337617823479005,
      "grad_norm": 0.36289045214653015,
      "kl": 0.1962890625,
      "learning_rate": 3.0711220392181934e-07,
      "loss": 0.0291,
      "reward": -2.3055556416511536,
      "reward_std": 0.4583980068564415,
      "rewards/confidence_reward": -2.7222222089767456,
      "rewards/format_reward": 0.4166666641831398,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3966.5555419921875,
      "epoch": 0.1842330762639246,
      "grad_norm": 1.2311605215072632,
      "kl": 0.2685546875,
      "learning_rate": 3.027209598681373e-07,
      "loss": 0.0363,
      "reward": -2.638888955116272,
      "reward_std": 0.4123065620660782,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.361111119389534,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3175.3333740234375,
      "epoch": 0.18508997429305912,
      "grad_norm": 4.940278053283691,
      "kl": 0.24658203125,
      "learning_rate": 2.9836319343816397e-07,
      "loss": 0.1492,
      "reward": -2.037036955356598,
      "reward_std": 0.7587747424840927,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.722222238779068,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3212.5277099609375,
      "epoch": 0.18594687232219365,
      "grad_norm": 7.882691383361816,
      "kl": 0.310546875,
      "learning_rate": 2.9403949460371677e-07,
      "loss": 0.2234,
      "reward": -2.2685185074806213,
      "reward_std": 0.5562530681490898,
      "rewards/confidence_reward": -2.6296297311782837,
      "rewards/format_reward": 0.3611111082136631,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3286.27783203125,
      "epoch": 0.1868037703513282,
      "grad_norm": 2.209719657897949,
      "kl": 0.3505859375,
      "learning_rate": 2.897504487244061e-07,
      "loss": 0.0156,
      "reward": -2.370370388031006,
      "reward_std": 0.5552610009908676,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.444444440305233,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3131.8333740234375,
      "epoch": 0.18766066838046272,
      "grad_norm": 6.650064945220947,
      "kl": 0.3505859375,
      "learning_rate": 2.854966364683872e-07,
      "loss": 0.2357,
      "reward": -1.9518136978149414,
      "reward_std": 0.8104376196861267,
      "rewards/confidence_reward": -2.4518136978149414,
      "rewards/format_reward": 0.5000000149011612,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3573.47216796875,
      "epoch": 0.18851756640959727,
      "grad_norm": 1.0854192972183228,
      "kl": 0.427734375,
      "learning_rate": 2.812786337337463e-07,
      "loss": 0.1052,
      "reward": -2.5185184478759766,
      "reward_std": 0.5389864593744278,
      "rewards/confidence_reward": -2.8518518209457397,
      "rewards/format_reward": 0.3333333432674408,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3518.25,
      "epoch": 0.1893744644387318,
      "grad_norm": 2.382499933242798,
      "kl": 0.46484375,
      "learning_rate": 2.770970115705341e-07,
      "loss": 0.0109,
      "reward": -2.740740656852722,
      "reward_std": 0.22680462896823883,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.1666666716337204,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3254.4166259765625,
      "epoch": 0.19023136246786632,
      "grad_norm": 1.3864353895187378,
      "kl": 0.4208984375,
      "learning_rate": 2.729523361034538e-07,
      "loss": 0.0677,
      "reward": -2.388888955116272,
      "reward_std": 0.3582318052649498,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.5555555522441864,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3346.9444580078125,
      "epoch": 0.19108826049700087,
      "grad_norm": 1.5457087755203247,
      "kl": 0.4716796875,
      "learning_rate": 2.68845168455218e-07,
      "loss": 0.0491,
      "reward": -2.388888955116272,
      "reward_std": 0.41752736270427704,
      "rewards/confidence_reward": -2.888888955116272,
      "rewards/format_reward": 0.5,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2871.02783203125,
      "epoch": 0.1919451585261354,
      "grad_norm": 2.262847661972046,
      "kl": 0.3876953125,
      "learning_rate": 2.6477606467058035e-07,
      "loss": 0.1352,
      "reward": -1.7870370149612427,
      "reward_std": 0.7662279903888702,
      "rewards/confidence_reward": -2.3703705072402954,
      "rewards/format_reward": 0.5833333432674408,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3113.6944580078125,
      "epoch": 0.1928020565552699,
      "grad_norm": 1.4508579969406128,
      "kl": 0.3828125,
      "learning_rate": 2.6074557564105724e-07,
      "loss": 0.1048,
      "reward": -1.7592592239379883,
      "reward_std": 0.5638266205787659,
      "rewards/confidence_reward": -2.3703704476356506,
      "rewards/format_reward": 0.6111111044883728,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3247.4444580078125,
      "epoch": 0.19365895458440446,
      "grad_norm": 0.9147429466247559,
      "kl": 0.513671875,
      "learning_rate": 2.567542470303452e-07,
      "loss": 0.1006,
      "reward": -2.1296297311782837,
      "reward_std": 0.44895367324352264,
      "rewards/confidence_reward": -2.574074149131775,
      "rewards/format_reward": 0.4444444477558136,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2793.9722290039062,
      "epoch": 0.194515852613539,
      "grad_norm": 1.462504506111145,
      "kl": 0.3876953125,
      "learning_rate": 2.528026192004466e-07,
      "loss": 0.0483,
      "reward": -1.8333333134651184,
      "reward_std": 0.6497750803828239,
      "rewards/confidence_reward": -2.3888890147209167,
      "rewards/format_reward": 0.5555555522441864,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3342.916748046875,
      "epoch": 0.1953727506426735,
      "grad_norm": 1.5023483037948608,
      "kl": 0.494140625,
      "learning_rate": 2.488912271385139e-07,
      "loss": 0.0784,
      "reward": -2.4907407760620117,
      "reward_std": 0.4541744738817215,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.4166666641831398,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3135.138916015625,
      "epoch": 0.19622964867180806,
      "grad_norm": 2.827260971069336,
      "kl": 0.53125,
      "learning_rate": 2.450206003844205e-07,
      "loss": 0.1925,
      "reward": -2.130053400993347,
      "reward_std": 0.9212456345558167,
      "rewards/confidence_reward": -2.713386654853821,
      "rewards/format_reward": 0.5833333432674408,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2381.2500610351562,
      "epoch": 0.19708654670094258,
      "grad_norm": 4.513103008270264,
      "kl": 0.28125,
      "learning_rate": 2.411912629590699e-07,
      "loss": 0.1358,
      "reward": -1.7693658471107483,
      "reward_std": 1.0916875302791595,
      "rewards/confidence_reward": -2.4638103246688843,
      "rewards/format_reward": 0.6944444477558136,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3553.3055419921875,
      "epoch": 0.19794344473007713,
      "grad_norm": 2.0678796768188477,
      "kl": 0.4443359375,
      "learning_rate": 2.374037332934512e-07,
      "loss": 0.0392,
      "reward": -2.5,
      "reward_std": 0.27888670563697815,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.3333333432674408,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3584.0833740234375,
      "epoch": 0.19880034275921166,
      "grad_norm": 2.108079433441162,
      "kl": 0.408203125,
      "learning_rate": 2.336585241584522e-07,
      "loss": 0.0707,
      "reward": -2.6944445371627808,
      "reward_std": 0.06804138422012329,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3055555671453476,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3742.2777099609375,
      "epoch": 0.19965724078834618,
      "grad_norm": 2.253603219985962,
      "kl": 0.4912109375,
      "learning_rate": 2.299561425954383e-07,
      "loss": 0.1806,
      "reward": -2.6388888359069824,
      "reward_std": 0.47208072245121,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.305555559694767,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2601.3056640625,
      "epoch": 0.20051413881748073,
      "grad_norm": 4.904290676116943,
      "kl": 0.3974609375,
      "learning_rate": 2.2629708984760706e-07,
      "loss": 0.2103,
      "reward": -1.3475482165813446,
      "reward_std": 1.2650677561759949,
      "rewards/confidence_reward": -2.0975481271743774,
      "rewards/format_reward": 0.75,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2671.6944580078125,
      "epoch": 0.20137103684661525,
      "grad_norm": 1.6857246160507202,
      "kl": 0.3486328125,
      "learning_rate": 2.2268186129212807e-07,
      "loss": 0.0329,
      "reward": -1.7870370149612427,
      "reward_std": 0.40415799617767334,
      "rewards/confidence_reward": -2.425925850868225,
      "rewards/format_reward": 0.638888880610466,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3342.5277099609375,
      "epoch": 0.20222793487574978,
      "grad_norm": 1.9274342060089111,
      "kl": 0.4453125,
      "learning_rate": 2.1911094637307714e-07,
      "loss": 0.108,
      "reward": -2.3796294927597046,
      "reward_std": 0.48020143806934357,
      "rewards/confidence_reward": -2.8518518209457397,
      "rewards/format_reward": 0.4722222238779068,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2953.388916015625,
      "epoch": 0.20308483290488433,
      "grad_norm": 2.970611810684204,
      "kl": 0.4501953125,
      "learning_rate": 2.1558482853517253e-07,
      "loss": 0.1648,
      "reward": -1.9360325932502747,
      "reward_std": 0.8466525003314018,
      "rewards/confidence_reward": -2.4082547426223755,
      "rewards/format_reward": 0.4722222164273262,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3137.4166259765625,
      "epoch": 0.20394173093401885,
      "grad_norm": 0.7161883115768433,
      "kl": 0.5009765625,
      "learning_rate": 2.1210398515832536e-07,
      "loss": 0.0883,
      "reward": -2.138888895511627,
      "reward_std": 0.7037928402423859,
      "rewards/confidence_reward": -2.6111111640930176,
      "rewards/format_reward": 0.4722222164273262,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3184.7222900390625,
      "epoch": 0.20479862896315337,
      "grad_norm": 2.5145044326782227,
      "kl": 0.638671875,
      "learning_rate": 2.08668887493009e-07,
      "loss": 0.1968,
      "reward": -2.231481432914734,
      "reward_std": 1.0531733930110931,
      "rewards/confidence_reward": -2.703703761100769,
      "rewards/format_reward": 0.472222238779068,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3364.638916015625,
      "epoch": 0.20565552699228792,
      "grad_norm": 1.4906601905822754,
      "kl": 0.533203125,
      "learning_rate": 2.0528000059645995e-07,
      "loss": 0.1209,
      "reward": -2.3888888359069824,
      "reward_std": 0.3082153648138046,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.6111111342906952,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3299.583251953125,
      "epoch": 0.20651242502142245,
      "grad_norm": 2.7216079235076904,
      "kl": 0.529296875,
      "learning_rate": 2.0193778326971628e-07,
      "loss": 0.089,
      "reward": -2.4722222089767456,
      "reward_std": 0.24017397314310074,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.5277777910232544,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3405.8055419921875,
      "epoch": 0.207369323050557,
      "grad_norm": 2.5555975437164307,
      "kl": 0.71875,
      "learning_rate": 1.986426879955034e-07,
      "loss": 0.1951,
      "reward": -2.0925925374031067,
      "reward_std": 0.9167544841766357,
      "rewards/confidence_reward": -2.5370370149612427,
      "rewards/format_reward": 0.4444444477558136,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3524.3055419921875,
      "epoch": 0.20822622107969152,
      "grad_norm": 1.0115513801574707,
      "kl": 0.794921875,
      "learning_rate": 1.9539516087697517e-07,
      "loss": 0.1157,
      "reward": -2.666666626930237,
      "reward_std": 0.30821534991264343,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333358168602,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3453.833251953125,
      "epoch": 0.20908311910882604,
      "grad_norm": 2.69391131401062,
      "kl": 0.6103515625,
      "learning_rate": 1.9219564157731844e-07,
      "loss": 0.1843,
      "reward": -2.5277777910232544,
      "reward_std": 0.5088144540786743,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.472222238779068,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3176.9444580078125,
      "epoch": 0.2099400171379606,
      "grad_norm": 3.2699921131134033,
      "kl": 0.609375,
      "learning_rate": 1.8904456326023027e-07,
      "loss": 0.1116,
      "reward": -1.9722222685813904,
      "reward_std": 0.5316601991653442,
      "rewards/confidence_reward": -2.5,
      "rewards/format_reward": 0.5277777910232544,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2784.6944580078125,
      "epoch": 0.21079691516709512,
      "grad_norm": 1.8682541847229004,
      "kl": 0.56640625,
      "learning_rate": 1.8594235253127372e-07,
      "loss": 0.1448,
      "reward": -1.6203704178333282,
      "reward_std": 0.6126016080379486,
      "rewards/confidence_reward": -2.2037037014961243,
      "rewards/format_reward": 0.5833333283662796,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3728.638916015625,
      "epoch": 0.21165381319622964,
      "grad_norm": 3.5428807735443115,
      "kl": 0.78515625,
      "learning_rate": 1.8288942938012267e-07,
      "loss": 0.0706,
      "reward": -2.490740656852722,
      "reward_std": 0.46697860956192017,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.4166666716337204,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3008.138916015625,
      "epoch": 0.2125107112253642,
      "grad_norm": 1.3764863014221191,
      "kl": 0.57421875,
      "learning_rate": 1.7988620712370195e-07,
      "loss": 0.0807,
      "reward": -1.9478338956832886,
      "reward_std": 0.7096292227506638,
      "rewards/confidence_reward": -2.4478338956832886,
      "rewards/format_reward": 0.5,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3118.1944580078125,
      "epoch": 0.2133676092544987,
      "grad_norm": 3.4032788276672363,
      "kl": 0.5830078125,
      "learning_rate": 1.7693309235023127e-07,
      "loss": 0.1638,
      "reward": -2.0277777314186096,
      "reward_std": 0.8689809292554855,
      "rewards/confidence_reward": -2.666666626930237,
      "rewards/format_reward": 0.6388888955116272,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3191.4166259765625,
      "epoch": 0.21422450728363324,
      "grad_norm": 1.8875763416290283,
      "kl": 0.689453125,
      "learning_rate": 1.7403048486417868e-07,
      "loss": 0.1055,
      "reward": -1.9545510709285736,
      "reward_std": 0.48649683594703674,
      "rewards/confidence_reward": -2.26010662317276,
      "rewards/format_reward": 0.3055555522441864,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2924.638916015625,
      "epoch": 0.2150814053127678,
      "grad_norm": 2.471829414367676,
      "kl": 0.5673828125,
      "learning_rate": 1.711787776321341e-07,
      "loss": 0.1465,
      "reward": -2.101851761341095,
      "reward_std": 0.6704803705215454,
      "rewards/confidence_reward": -2.7407407760620117,
      "rewards/format_reward": 0.638888880610466,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3481.8056640625,
      "epoch": 0.2159383033419023,
      "grad_norm": 1.7675886154174805,
      "kl": 0.8046875,
      "learning_rate": 1.6837835672960831e-07,
      "loss": 0.1342,
      "reward": -2.314814805984497,
      "reward_std": 0.5083017498254776,
      "rewards/confidence_reward": -2.7037036418914795,
      "rewards/format_reward": 0.3888888955116272,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3527.4166259765625,
      "epoch": 0.21679520137103683,
      "grad_norm": 2.999549627304077,
      "kl": 0.759765625,
      "learning_rate": 1.6562960128876353e-07,
      "loss": 0.0954,
      "reward": -2.5185184478759766,
      "reward_std": 0.3628873750567436,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.3888888955116272,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3826.916748046875,
      "epoch": 0.21765209940017138,
      "grad_norm": 4.181079864501953,
      "kl": 0.927734375,
      "learning_rate": 1.6293288344708566e-07,
      "loss": 0.0935,
      "reward": -2.6944445371627808,
      "reward_std": 0.3134361654520035,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3448.47216796875,
      "epoch": 0.2185089974293059,
      "grad_norm": 1.8785117864608765,
      "kl": 0.7265625,
      "learning_rate": 1.6028856829700258e-07,
      "loss": 0.0921,
      "reward": -2.4175140261650085,
      "reward_std": 0.5672558695077896,
      "rewards/confidence_reward": -2.83418071269989,
      "rewards/format_reward": 0.4166666604578495,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3920.5833740234375,
      "epoch": 0.21936589545844046,
      "grad_norm": 7.422764778137207,
      "kl": 0.84375,
      "learning_rate": 1.5769701383645698e-07,
      "loss": 0.034,
      "reward": -2.7777777910232544,
      "reward_std": 0.1360827535390854,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2222222276031971,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3276.9444580078125,
      "epoch": 0.22022279348757498,
      "grad_norm": 6.746924877166748,
      "kl": 0.640625,
      "learning_rate": 1.551585709204381e-07,
      "loss": 0.1587,
      "reward": -1.9537036418914795,
      "reward_std": 0.6170237511396408,
      "rewards/confidence_reward": -2.425925850868225,
      "rewards/format_reward": 0.4722222238779068,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2808.27783203125,
      "epoch": 0.2210796915167095,
      "grad_norm": 9.45202350616455,
      "kl": 0.42578125,
      "learning_rate": 1.5267358321348285e-07,
      "loss": 0.224,
      "reward": -1.5564029514789581,
      "reward_std": 0.6021379753947258,
      "rewards/confidence_reward": -2.0564029812812805,
      "rewards/format_reward": 0.5,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.3611145019531,
      "epoch": 0.22193658954584405,
      "grad_norm": 1.4079711437225342,
      "kl": 0.208984375,
      "learning_rate": 1.5024238714314825e-07,
      "loss": 0.0526,
      "reward": -0.9999999701976776,
      "reward_std": 1.1379348039627075,
      "rewards/confidence_reward": -2.0000001192092896,
      "rewards/format_reward": 1.0,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3447.47216796875,
      "epoch": 0.22279348757497858,
      "grad_norm": 2.1422104835510254,
      "kl": 0.6162109375,
      "learning_rate": 1.4786531185446452e-07,
      "loss": 0.1618,
      "reward": -2.4018328189849854,
      "reward_std": 0.5933989137411118,
      "rewards/confidence_reward": -2.7907216548919678,
      "rewards/format_reward": 0.3888889029622078,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3852.666748046875,
      "epoch": 0.2236503856041131,
      "grad_norm": 2.077338457107544,
      "kl": 0.70703125,
      "learning_rate": 1.4554267916537495e-07,
      "loss": 0.1074,
      "reward": -2.805555582046509,
      "reward_std": 0.3134361580014229,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1944444514811039,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3562.7222900390625,
      "epoch": 0.22450728363324765,
      "grad_norm": 0.7478838562965393,
      "kl": 0.5234375,
      "learning_rate": 1.432748035231658e-07,
      "loss": 0.0809,
      "reward": -2.546296238899231,
      "reward_std": 0.4309287667274475,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.3611111082136631,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3409.388916015625,
      "epoch": 0.22536418166238217,
      "grad_norm": 1.5875946283340454,
      "kl": 0.560546875,
      "learning_rate": 1.4106199196189608e-07,
      "loss": 0.0874,
      "reward": -2.5,
      "reward_std": 0.44037309288978577,
      "rewards/confidence_reward": -2.722222328186035,
      "rewards/format_reward": 0.2222222276031971,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3616.8055419921875,
      "epoch": 0.2262210796915167,
      "grad_norm": 4.355445384979248,
      "kl": 0.47265625,
      "learning_rate": 1.3890454406082956e-07,
      "loss": 0.1239,
      "reward": -2.6111111640930176,
      "reward_std": 0.3814775198698044,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3888888955116272,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3070.0833740234375,
      "epoch": 0.22707797772065125,
      "grad_norm": 1.2011481523513794,
      "kl": 0.4541015625,
      "learning_rate": 1.3680275190387675e-07,
      "loss": 0.1067,
      "reward": -2.2592591047286987,
      "reward_std": 0.46683208644390106,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.5000000149011612,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3636.3333740234375,
      "epoch": 0.22793487574978577,
      "grad_norm": 2.6138198375701904,
      "kl": 0.5703125,
      "learning_rate": 1.3475690004005097e-07,
      "loss": 0.1152,
      "reward": -2.6111111640930176,
      "reward_std": 0.29541122913360596,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3888888955116272,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2078.4444580078125,
      "epoch": 0.22879177377892032,
      "grad_norm": 1.8686110973358154,
      "kl": 0.294921875,
      "learning_rate": 1.3276726544494571e-07,
      "loss": 0.0517,
      "reward": -1.2870370149612427,
      "reward_std": 1.2355431914329529,
      "rewards/confidence_reward": -2.203703761100769,
      "rewards/format_reward": 0.9166666567325592,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3596.3055419921875,
      "epoch": 0.22964867180805484,
      "grad_norm": 5.496092319488525,
      "kl": 0.66796875,
      "learning_rate": 1.308341174832359e-07,
      "loss": 0.023,
      "reward": -2.75,
      "reward_std": 0.2901904433965683,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.2500000074505806,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2841.416748046875,
      "epoch": 0.23050556983718937,
      "grad_norm": 8.94375991821289,
      "kl": 0.4580078125,
      "learning_rate": 1.2895771787221088e-07,
      "loss": 0.2275,
      "reward": -1.8611111044883728,
      "reward_std": 1.1322061419487,
      "rewards/confidence_reward": -2.3888890743255615,
      "rewards/format_reward": 0.5277777910232544,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2849.9722900390625,
      "epoch": 0.23136246786632392,
      "grad_norm": 3.3087215423583984,
      "kl": 0.439453125,
      "learning_rate": 1.2713832064634125e-07,
      "loss": 0.1111,
      "reward": -1.6481481790542603,
      "reward_std": 0.5850364714860916,
      "rewards/confidence_reward": -2.203703761100769,
      "rewards/format_reward": 0.5555555671453476,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3841.3055419921875,
      "epoch": 0.23221936589545844,
      "grad_norm": 4.227456569671631,
      "kl": 0.708984375,
      "learning_rate": 1.2537617212288742e-07,
      "loss": 0.0629,
      "reward": -2.6666667461395264,
      "reward_std": 0.2453947588801384,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2746.083251953125,
      "epoch": 0.23307626392459296,
      "grad_norm": 10.011512756347656,
      "kl": 0.48046875,
      "learning_rate": 1.2367151086855187e-07,
      "loss": 0.2351,
      "reward": -2.0833332538604736,
      "reward_std": 0.8133133947849274,
      "rewards/confidence_reward": -2.6666667461395264,
      "rewards/format_reward": 0.5833333432674408,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3753.7501220703125,
      "epoch": 0.23393316195372751,
      "grad_norm": 2.311757802963257,
      "kl": 0.7109375,
      "learning_rate": 1.220245676671809e-07,
      "loss": 0.0975,
      "reward": -2.6111111640930176,
      "reward_std": 0.41752736270427704,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3888889029622078,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3697.3333740234375,
      "epoch": 0.23479005998286204,
      "grad_norm": 1.5095380544662476,
      "kl": 0.76953125,
      "learning_rate": 1.2043556548852063e-07,
      "loss": 0.1038,
      "reward": -2.666666626930237,
      "reward_std": 0.245394766330719,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.3333333432674408,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3706.2222900390625,
      "epoch": 0.23564695801199656,
      "grad_norm": 1.542810320854187,
      "kl": 0.72265625,
      "learning_rate": 1.1890471945802999e-07,
      "loss": 0.1013,
      "reward": -2.694444417953491,
      "reward_std": 0.20412413030862808,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.30555556900799274,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2976.6944580078125,
      "epoch": 0.2365038560411311,
      "grad_norm": 5.164428234100342,
      "kl": 0.4794921875,
      "learning_rate": 1.1743223682775649e-07,
      "loss": 0.1522,
      "reward": -1.759259283542633,
      "reward_std": 0.9753341376781464,
      "rewards/confidence_reward": -2.425925850868225,
      "rewards/format_reward": 0.6666666567325592,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3707.5555419921875,
      "epoch": 0.23736075407026563,
      "grad_norm": 4.900428295135498,
      "kl": 0.76171875,
      "learning_rate": 1.160183169482775e-07,
      "loss": 0.0093,
      "reward": -2.6666667461395264,
      "reward_std": 0.2221490666270256,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.2777777835726738,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3226.5,
      "epoch": 0.23821765209940018,
      "grad_norm": 2.8093297481536865,
      "kl": 0.66796875,
      "learning_rate": 1.1466315124171128e-07,
      "loss": 0.0633,
      "reward": -2.314814805984497,
      "reward_std": 0.5322824418544769,
      "rewards/confidence_reward": -2.8148149251937866,
      "rewards/format_reward": 0.5,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2714.388916015625,
      "epoch": 0.2390745501285347,
      "grad_norm": 12.48595142364502,
      "kl": 0.61328125,
      "learning_rate": 1.1336692317580158e-07,
      "loss": 0.2795,
      "reward": -1.5740740895271301,
      "reward_std": 0.9820237159729004,
      "rewards/confidence_reward": -2.2407408952713013,
      "rewards/format_reward": 0.6666666716337204,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2800.166748046875,
      "epoch": 0.23993144815766923,
      "grad_norm": 1.2126076221466064,
      "kl": 0.53125,
      "learning_rate": 1.1212980823907929e-07,
      "loss": 0.0903,
      "reward": -1.9259259104728699,
      "reward_std": 0.8028348684310913,
      "rewards/confidence_reward": -2.425926089286804,
      "rewards/format_reward": 0.5000000149011612,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3716.638916015625,
      "epoch": 0.24078834618680378,
      "grad_norm": 2.4237496852874756,
      "kl": 0.677734375,
      "learning_rate": 1.1095197391710362e-07,
      "loss": 0.0919,
      "reward": -2.7222222089767456,
      "reward_std": 0.3419027030467987,
      "rewards/confidence_reward": -2.944444417953491,
      "rewards/format_reward": 0.22222222574055195,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3308.6944580078125,
      "epoch": 0.2416452442159383,
      "grad_norm": 1.1492081880569458,
      "kl": 0.5546875,
      "learning_rate": 1.0983357966978745e-07,
      "loss": 0.0932,
      "reward": -2.2592592239379883,
      "reward_std": 0.5841350704431534,
      "rewards/confidence_reward": -2.759259343147278,
      "rewards/format_reward": 0.5000000149011612,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3500.416748046875,
      "epoch": 0.24250214224507283,
      "grad_norm": 2.292973756790161,
      "kl": 0.67578125,
      "learning_rate": 1.0877477690980931e-07,
      "loss": 0.1152,
      "reward": -2.4629629850387573,
      "reward_std": 0.44895368814468384,
      "rewards/confidence_reward": -2.9074074029922485,
      "rewards/format_reward": 0.4444444552063942,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3665.1666259765625,
      "epoch": 0.24335904027420738,
      "grad_norm": 3.3193001747131348,
      "kl": 0.6337890625,
      "learning_rate": 1.0777570898211405e-07,
      "loss": 0.0586,
      "reward": -2.5370370149612427,
      "reward_std": 0.41200246661901474,
      "rewards/confidence_reward": -2.7592592239379883,
      "rewards/format_reward": 0.22222222574055195,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3264.75,
      "epoch": 0.2442159383033419,
      "grad_norm": 2.722202777862549,
      "kl": 0.541015625,
      "learning_rate": 1.068365111445064e-07,
      "loss": 0.1034,
      "reward": -2.185185134410858,
      "reward_std": 0.6440667659044266,
      "rewards/confidence_reward": -2.7962963581085205,
      "rewards/format_reward": 0.6111111044883728,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3782.638916015625,
      "epoch": 0.24507283633247642,
      "grad_norm": 1.6381499767303467,
      "kl": 0.720703125,
      "learning_rate": 1.0595731054933934e-07,
      "loss": 0.0854,
      "reward": -2.75,
      "reward_std": 0.09128709882497787,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.25,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3756.0,
      "epoch": 0.24592973436161097,
      "grad_norm": 1.275122880935669,
      "kl": 0.6171875,
      "learning_rate": 1.0513822622629978e-07,
      "loss": 0.0672,
      "reward": -2.694444417953491,
      "reward_std": 0.1773533970117569,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.305555559694767,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3949.111083984375,
      "epoch": 0.2467866323907455,
      "grad_norm": 2.43125057220459,
      "kl": 0.619140625,
      "learning_rate": 1.0437936906629334e-07,
      "loss": 0.0689,
      "reward": -2.8888888359069824,
      "reward_std": 0.15932847559452057,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.11111111380159855,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2842.9722900390625,
      "epoch": 0.24764353041988005,
      "grad_norm": 1.0015897750854492,
      "kl": 0.3935546875,
      "learning_rate": 1.0368084180643224e-07,
      "loss": 0.0667,
      "reward": -1.8897362351417542,
      "reward_std": 0.7023327946662903,
      "rewards/confidence_reward": -2.500847339630127,
      "rewards/format_reward": 0.6111111342906952,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3495.5833740234375,
      "epoch": 0.24850042844901457,
      "grad_norm": 3.5078701972961426,
      "kl": 0.6806640625,
      "learning_rate": 1.0304273901612565e-07,
      "loss": 0.0511,
      "reward": -2.3055556416511536,
      "reward_std": 0.5096173286437988,
      "rewards/confidence_reward": -2.6111111640930176,
      "rewards/format_reward": 0.305555559694767,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3017.638916015625,
      "epoch": 0.2493573264781491,
      "grad_norm": 5.197386264801025,
      "kl": 0.466796875,
      "learning_rate": 1.0246514708427701e-07,
      "loss": 0.1364,
      "reward": -1.9907406568527222,
      "reward_std": 0.7819060683250427,
      "rewards/confidence_reward": -2.518518567085266,
      "rewards/format_reward": 0.5277777910232544,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3224.52783203125,
      "epoch": 0.25021422450728364,
      "grad_norm": 3.3562777042388916,
      "kl": 0.552734375,
      "learning_rate": 1.0194814420758804e-07,
      "loss": 0.1602,
      "reward": -2.0740740299224854,
      "reward_std": 0.3628873825073242,
      "rewards/confidence_reward": -2.5740740299224854,
      "rewards/format_reward": 0.5,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3234.861083984375,
      "epoch": 0.25107112253641817,
      "grad_norm": 3.984017848968506,
      "kl": 0.572265625,
      "learning_rate": 1.0149180037997228e-07,
      "loss": 0.1834,
      "reward": -2.114795684814453,
      "reward_std": 0.7361937463283539,
      "rewards/confidence_reward": -2.5870180130004883,
      "rewards/format_reward": 0.472222238779068,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3694.6666259765625,
      "epoch": 0.2519280205655527,
      "grad_norm": 1.1605571508407593,
      "kl": 0.556640625,
      "learning_rate": 1.0109617738307911e-07,
      "loss": 0.1052,
      "reward": -2.6574074029922485,
      "reward_std": 0.6070278584957123,
      "rewards/confidence_reward": -2.8518518209457397,
      "rewards/format_reward": 0.1944444514811039,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3409.02783203125,
      "epoch": 0.2527849185946872,
      "grad_norm": 2.5758039951324463,
      "kl": 0.54296875,
      "learning_rate": 1.0076132877792932e-07,
      "loss": 0.1731,
      "reward": -2.5277777910232544,
      "reward_std": 0.3936592862010002,
      "rewards/confidence_reward": -2.833333373069763,
      "rewards/format_reward": 0.3055555559694767,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3830.166748046875,
      "epoch": 0.2536418166238218,
      "grad_norm": 2.108941078186035,
      "kl": 0.630859375,
      "learning_rate": 1.0048729989766394e-07,
      "loss": 0.0874,
      "reward": -2.805555582046509,
      "reward_std": 0.20412414520978928,
      "rewards/confidence_reward": -3.0,
      "rewards/format_reward": 0.1944444514811039,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3393.3333740234375,
      "epoch": 0.2544987146529563,
      "grad_norm": 3.105771064758301,
      "kl": 0.484375,
      "learning_rate": 1.002741278414069e-07,
      "loss": 0.1061,
      "reward": -2.240740716457367,
      "reward_std": 0.654969185590744,
      "rewards/confidence_reward": -2.796296238899231,
      "rewards/format_reward": 0.5555555671453476,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2507.611114501953,
      "epoch": 0.25535561268209084,
      "grad_norm": 9.724431991577148,
      "kl": 0.693359375,
      "learning_rate": 1.0012184146924223e-07,
      "loss": 0.0067,
      "reward": -2.000847339630127,
      "reward_std": 0.5755332708358765,
      "rewards/confidence_reward": -2.500847339630127,
      "rewards/format_reward": 0.5,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 3396.9444580078125,
      "epoch": 0.25621251071122536,
      "grad_norm": 5.069945335388184,
      "kl": 0.4638671875,
      "learning_rate": 1.0003046139830701e-07,
      "loss": 0.1936,
      "reward": -2.2314815521240234,
      "reward_std": 0.8970945179462433,
      "rewards/confidence_reward": -2.759259343147278,
      "rewards/format_reward": 0.5277777910232544,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 2547.6945190429688,
      "epoch": 0.2570694087403599,
      "grad_norm": 6.607841491699219,
      "kl": 0.3818359375,
      "learning_rate": 1e-07,
      "loss": 0.1813,
      "reward": -1.5749214887619019,
      "reward_std": 0.6023647785186768,
      "rewards/confidence_reward": -2.1860325932502747,
      "rewards/format_reward": 0.6111111044883728,
      "step": 300
    },
    {
      "epoch": 0.2570694087403599,
      "step": 300,
      "total_flos": 0.0,
      "train_loss": 0.09362538470280318,
      "train_runtime": 37056.8122,
      "train_samples_per_second": 0.291,
      "train_steps_per_second": 0.008
    }
  ],
  "logging_steps": 1,
  "max_steps": 300,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}