{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 1630,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.001226993865030675,
      "grad_norm": 151.20719434086217,
      "learning_rate": 6.000000000000001e-08,
      "logits/chosen": -0.32421875,
      "logits/rejected": -0.515625,
      "logps/chosen": -474.0,
      "logps/rejected": -316.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.00245398773006135,
      "grad_norm": 162.171350331327,
      "learning_rate": 1.2000000000000002e-07,
      "logits/chosen": -0.22265625,
      "logits/rejected": -0.439453125,
      "logps/chosen": -436.0,
      "logps/rejected": -292.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.0036809815950920245,
      "grad_norm": 121.91181472288706,
      "learning_rate": 1.8e-07,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2578125,
      "logps/chosen": -368.0,
      "logps/rejected": -272.0,
      "loss": 0.7012,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.0006103515625,
      "rewards/margins": -0.00506591796875,
      "rewards/rejected": 0.00445556640625,
      "step": 3
    },
    {
      "epoch": 0.0049079754601227,
      "grad_norm": 138.61232520616198,
      "learning_rate": 2.4000000000000003e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.326171875,
      "logps/chosen": -422.0,
      "logps/rejected": -242.0,
      "loss": 0.7003,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": 0.0057373046875,
      "rewards/margins": -0.00762939453125,
      "rewards/rejected": 0.01336669921875,
      "step": 4
    },
    {
      "epoch": 0.006134969325153374,
      "grad_norm": 318.4843338984337,
      "learning_rate": 3.0000000000000004e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.32421875,
      "logps/chosen": -380.0,
      "logps/rejected": -270.0,
      "loss": 0.7002,
      "rewards/accuracies": 0.296875,
      "rewards/chosen": -0.0037994384765625,
      "rewards/margins": -0.01123046875,
      "rewards/rejected": 0.007415771484375,
      "step": 5
    },
    {
      "epoch": 0.007361963190184049,
      "grad_norm": 356.6573592678301,
      "learning_rate": 3.6e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": -0.345703125,
      "logps/chosen": -412.0,
      "logps/rejected": -306.0,
      "loss": 0.692,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": 0.0208740234375,
      "rewards/margins": 0.01153564453125,
      "rewards/rejected": 0.00927734375,
      "step": 6
    },
    {
      "epoch": 0.008588957055214725,
      "grad_norm": 228.0843622624859,
      "learning_rate": 4.2000000000000006e-07,
      "logits/chosen": -0.2060546875,
      "logits/rejected": -0.38671875,
      "logps/chosen": -378.0,
      "logps/rejected": -266.0,
      "loss": 0.6962,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": 0.006683349609375,
      "rewards/margins": -0.001617431640625,
      "rewards/rejected": 0.00830078125,
      "step": 7
    },
    {
      "epoch": 0.0098159509202454,
      "grad_norm": 139.9677750400214,
      "learning_rate": 4.800000000000001e-07,
      "logits/chosen": -0.111328125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -432.0,
      "logps/rejected": -292.0,
      "loss": 0.6667,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": 0.054443359375,
      "rewards/margins": 0.06201171875,
      "rewards/rejected": -0.00762939453125,
      "step": 8
    },
    {
      "epoch": 0.011042944785276074,
      "grad_norm": 114.26865689975604,
      "learning_rate": 5.4e-07,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.359375,
      "logps/chosen": -402.0,
      "logps/rejected": -294.0,
      "loss": 0.6492,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 0.0654296875,
      "rewards/margins": 0.10986328125,
      "rewards/rejected": -0.044677734375,
      "step": 9
    },
    {
      "epoch": 0.012269938650306749,
      "grad_norm": 150.56729793626556,
      "learning_rate": 6.000000000000001e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -402.0,
      "logps/rejected": -278.0,
      "loss": 0.6146,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 0.1943359375,
      "rewards/rejected": -0.0361328125,
      "step": 10
    },
    {
      "epoch": 0.013496932515337423,
      "grad_norm": 85.94823037139018,
      "learning_rate": 6.6e-07,
      "logits/chosen": -0.375,
      "logits/rejected": -0.515625,
      "logps/chosen": -420.0,
      "logps/rejected": -298.0,
      "loss": 0.5535,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2451171875,
      "rewards/margins": 0.3984375,
      "rewards/rejected": -0.1533203125,
      "step": 11
    },
    {
      "epoch": 0.014723926380368098,
      "grad_norm": 60.96373680958018,
      "learning_rate": 7.2e-07,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.244140625,
      "logps/chosen": -372.0,
      "logps/rejected": -262.0,
      "loss": 0.548,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 0.435546875,
      "rewards/rejected": -0.1796875,
      "step": 12
    },
    {
      "epoch": 0.015950920245398775,
      "grad_norm": 92.70871898741228,
      "learning_rate": 7.8e-07,
      "logits/chosen": -0.240234375,
      "logits/rejected": -0.408203125,
      "logps/chosen": -390.0,
      "logps/rejected": -251.0,
      "loss": 0.5255,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.302734375,
      "rewards/margins": 0.56640625,
      "rewards/rejected": -0.26171875,
      "step": 13
    },
    {
      "epoch": 0.01717791411042945,
      "grad_norm": 48.59299098972598,
      "learning_rate": 8.400000000000001e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -466.0,
      "logps/rejected": -314.0,
      "loss": 0.4612,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.765625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -0.31640625,
      "step": 14
    },
    {
      "epoch": 0.018404907975460124,
      "grad_norm": 59.699183777930486,
      "learning_rate": 9e-07,
      "logits/chosen": -0.10400390625,
      "logits/rejected": -0.177734375,
      "logps/chosen": -414.0,
      "logps/rejected": -292.0,
      "loss": 0.5014,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 0.96875,
      "rewards/rejected": -0.31640625,
      "step": 15
    },
    {
      "epoch": 0.0196319018404908,
      "grad_norm": 52.10075632980986,
      "learning_rate": 9.600000000000001e-07,
      "logits/chosen": -0.25,
      "logits/rejected": -0.42578125,
      "logps/chosen": -400.0,
      "logps/rejected": -310.0,
      "loss": 0.4424,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.84375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -0.54296875,
      "step": 16
    },
    {
      "epoch": 0.020858895705521473,
      "grad_norm": 42.16097538458702,
      "learning_rate": 1.0200000000000002e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -426.0,
      "logps/rejected": -260.0,
      "loss": 0.4308,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -0.58984375,
      "step": 17
    },
    {
      "epoch": 0.022085889570552148,
      "grad_norm": 46.56295761289983,
      "learning_rate": 1.08e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.330078125,
      "logps/chosen": -398.0,
      "logps/rejected": -282.0,
      "loss": 0.432,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.875,
      "rewards/margins": 1.6875,
      "rewards/rejected": -0.8125,
      "step": 18
    },
    {
      "epoch": 0.023312883435582823,
      "grad_norm": 49.57615257325465,
      "learning_rate": 1.14e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.3671875,
      "logps/chosen": -382.0,
      "logps/rejected": -296.0,
      "loss": 0.4802,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -1.0546875,
      "step": 19
    },
    {
      "epoch": 0.024539877300613498,
      "grad_norm": 42.63506109619643,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.439453125,
      "logps/chosen": -456.0,
      "logps/rejected": -324.0,
      "loss": 0.3801,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.4375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -1.703125,
      "step": 20
    },
    {
      "epoch": 0.025766871165644172,
      "grad_norm": 34.456600308133275,
      "learning_rate": 1.26e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -404.0,
      "logps/rejected": -272.0,
      "loss": 0.3497,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 2.640625,
      "rewards/rejected": -1.53125,
      "step": 21
    },
    {
      "epoch": 0.026993865030674847,
      "grad_norm": 44.983878720566445,
      "learning_rate": 1.32e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.271484375,
      "logps/chosen": -374.0,
      "logps/rejected": -306.0,
      "loss": 0.4156,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -1.7890625,
      "step": 22
    },
    {
      "epoch": 0.02822085889570552,
      "grad_norm": 56.72818428668396,
      "learning_rate": 1.3800000000000001e-06,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.314453125,
      "logps/chosen": -370.0,
      "logps/rejected": -318.0,
      "loss": 0.4566,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.46875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -1.7265625,
      "step": 23
    },
    {
      "epoch": 0.029447852760736196,
      "grad_norm": 55.662509241103734,
      "learning_rate": 1.44e-06,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.375,
      "logps/chosen": -376.0,
      "logps/rejected": -288.0,
      "loss": 0.4763,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 2.265625,
      "rewards/rejected": -1.625,
      "step": 24
    },
    {
      "epoch": 0.03067484662576687,
      "grad_norm": 35.38065334196675,
      "learning_rate": 1.5e-06,
      "logits/chosen": -0.2578125,
      "logits/rejected": -0.4453125,
      "logps/chosen": -430.0,
      "logps/rejected": -312.0,
      "loss": 0.3378,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -2.21875,
      "step": 25
    },
    {
      "epoch": 0.03190184049079755,
      "grad_norm": 42.887450752414956,
      "learning_rate": 1.4993801652892562e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -380.0,
      "logps/rejected": -288.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.640625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -1.9609375,
      "step": 26
    },
    {
      "epoch": 0.033128834355828224,
      "grad_norm": 37.07120513953294,
      "learning_rate": 1.4987603305785125e-06,
      "logits/chosen": -0.095703125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -354.0,
      "logps/rejected": -292.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -2.34375,
      "step": 27
    },
    {
      "epoch": 0.0343558282208589,
      "grad_norm": 35.87136073029194,
      "learning_rate": 1.4981404958677687e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -374.0,
      "logps/rejected": -274.0,
      "loss": 0.3615,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -1.796875,
      "step": 28
    },
    {
      "epoch": 0.03558282208588957,
      "grad_norm": 38.17662218776474,
      "learning_rate": 1.4975206611570248e-06,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.4140625,
      "logps/chosen": -414.0,
      "logps/rejected": -334.0,
      "loss": 0.3367,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.765625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -2.296875,
      "step": 29
    },
    {
      "epoch": 0.03680981595092025,
      "grad_norm": 39.994932388621756,
      "learning_rate": 1.496900826446281e-06,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.248046875,
      "logps/chosen": -332.0,
      "logps/rejected": -278.0,
      "loss": 0.3648,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.203125,
      "rewards/margins": 3.375,
      "rewards/rejected": -2.171875,
      "step": 30
    },
    {
      "epoch": 0.03803680981595092,
      "grad_norm": 42.39396872212231,
      "learning_rate": 1.496280991735537e-06,
      "logits/chosen": -0.10986328125,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -378.0,
      "logps/rejected": -286.0,
      "loss": 0.364,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.9765625,
      "rewards/margins": 4.25,
      "rewards/rejected": -2.265625,
      "step": 31
    },
    {
      "epoch": 0.0392638036809816,
      "grad_norm": 43.84495796047598,
      "learning_rate": 1.4956611570247935e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -388.0,
      "logps/rejected": -298.0,
      "loss": 0.3861,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.71875,
      "rewards/margins": 4.0,
      "rewards/rejected": -2.28125,
      "step": 32
    },
    {
      "epoch": 0.04049079754601227,
      "grad_norm": 38.440307864705964,
      "learning_rate": 1.4950413223140496e-06,
      "logits/chosen": -0.04443359375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -328.0,
      "logps/rejected": -248.0,
      "loss": 0.3456,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.078125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -2.484375,
      "step": 33
    },
    {
      "epoch": 0.04171779141104295,
      "grad_norm": 42.669938783190766,
      "learning_rate": 1.4944214876033057e-06,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.296875,
      "logps/chosen": -384.0,
      "logps/rejected": -316.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -2.8125,
      "step": 34
    },
    {
      "epoch": 0.04294478527607362,
      "grad_norm": 39.82238013261856,
      "learning_rate": 1.493801652892562e-06,
      "logits/chosen": -0.0810546875,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -356.0,
      "logps/rejected": -278.0,
      "loss": 0.3621,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 4.125,
      "rewards/rejected": -2.9375,
      "step": 35
    },
    {
      "epoch": 0.044171779141104296,
      "grad_norm": 34.79756258065961,
      "learning_rate": 1.4931818181818182e-06,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.20703125,
      "logps/chosen": -398.0,
      "logps/rejected": -310.0,
      "loss": 0.3113,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.6640625,
      "rewards/margins": 4.78125,
      "rewards/rejected": -3.125,
      "step": 36
    },
    {
      "epoch": 0.04539877300613497,
      "grad_norm": 40.21812818356396,
      "learning_rate": 1.4925619834710746e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -384.0,
      "logps/rejected": -322.0,
      "loss": 0.3512,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 4.625,
      "rewards/rejected": -2.328125,
      "step": 37
    },
    {
      "epoch": 0.046625766871165646,
      "grad_norm": 38.815201724599156,
      "learning_rate": 1.4919421487603307e-06,
      "logits/chosen": -0.032470703125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -390.0,
      "logps/rejected": -312.0,
      "loss": 0.36,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.546875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -2.546875,
      "step": 38
    },
    {
      "epoch": 0.04785276073619632,
      "grad_norm": 36.01668225984001,
      "learning_rate": 1.4913223140495867e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.2890625,
      "logps/chosen": -418.0,
      "logps/rejected": -318.0,
      "loss": 0.3347,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.296875,
      "rewards/margins": 5.21875,
      "rewards/rejected": -2.921875,
      "step": 39
    },
    {
      "epoch": 0.049079754601226995,
      "grad_norm": 30.447104323671855,
      "learning_rate": 1.490702479338843e-06,
      "logits/chosen": -0.0712890625,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -370.0,
      "logps/rejected": -296.0,
      "loss": 0.2729,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.09375,
      "rewards/margins": 4.90625,
      "rewards/rejected": -2.796875,
      "step": 40
    },
    {
      "epoch": 0.05030674846625767,
      "grad_norm": 36.96390592719149,
      "learning_rate": 1.4900826446280991e-06,
      "logits/chosen": -0.0751953125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -374.0,
      "logps/rejected": -286.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -2.671875,
      "step": 41
    },
    {
      "epoch": 0.051533742331288344,
      "grad_norm": 33.41932525699284,
      "learning_rate": 1.4894628099173555e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -360.0,
      "logps/rejected": -294.0,
      "loss": 0.3014,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.75,
      "rewards/margins": 4.3125,
      "rewards/rejected": -2.5625,
      "step": 42
    },
    {
      "epoch": 0.05276073619631902,
      "grad_norm": 38.41247549359973,
      "learning_rate": 1.4888429752066116e-06,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.248046875,
      "logps/chosen": -412.0,
      "logps/rejected": -336.0,
      "loss": 0.2954,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.09375,
      "rewards/margins": 5.09375,
      "rewards/rejected": -3.0,
      "step": 43
    },
    {
      "epoch": 0.053987730061349694,
      "grad_norm": 43.07116536752478,
      "learning_rate": 1.4882231404958678e-06,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -384.0,
      "logps/rejected": -304.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.578125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -2.96875,
      "step": 44
    },
    {
      "epoch": 0.05521472392638037,
      "grad_norm": 39.15973652461684,
      "learning_rate": 1.4876033057851241e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.294921875,
      "logps/chosen": -350.0,
      "logps/rejected": -300.0,
      "loss": 0.3339,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.828125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -3.109375,
      "step": 45
    },
    {
      "epoch": 0.05644171779141104,
      "grad_norm": 37.261369579802675,
      "learning_rate": 1.4869834710743803e-06,
      "logits/chosen": -0.09716796875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -416.0,
      "logps/rejected": -302.0,
      "loss": 0.2861,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.46875,
      "rewards/margins": 5.375,
      "rewards/rejected": -2.890625,
      "step": 46
    },
    {
      "epoch": 0.05766871165644172,
      "grad_norm": 54.78531831058221,
      "learning_rate": 1.4863636363636364e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -390.0,
      "logps/rejected": -372.0,
      "loss": 0.5073,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 1.5234375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -2.6875,
      "step": 47
    },
    {
      "epoch": 0.05889570552147239,
      "grad_norm": 43.69804279742604,
      "learning_rate": 1.4857438016528926e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.26171875,
      "logps/chosen": -352.0,
      "logps/rejected": -288.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -2.734375,
      "step": 48
    },
    {
      "epoch": 0.06012269938650307,
      "grad_norm": 34.89468290879934,
      "learning_rate": 1.4851239669421487e-06,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.328125,
      "logps/chosen": -400.0,
      "logps/rejected": -300.0,
      "loss": 0.2721,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.25,
      "rewards/margins": 5.46875,
      "rewards/rejected": -3.21875,
      "step": 49
    },
    {
      "epoch": 0.06134969325153374,
      "grad_norm": 54.15414768657872,
      "learning_rate": 1.484504132231405e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.296875,
      "logps/chosen": -404.0,
      "logps/rejected": -324.0,
      "loss": 0.4387,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.515625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -3.171875,
      "step": 50
    },
    {
      "epoch": 0.06257668711656442,
      "grad_norm": 34.25072100129564,
      "learning_rate": 1.4838842975206612e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -440.0,
      "logps/rejected": -364.0,
      "loss": 0.2872,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.984375,
      "rewards/margins": 5.09375,
      "rewards/rejected": -3.125,
      "step": 51
    },
    {
      "epoch": 0.0638036809815951,
      "grad_norm": 38.20166040651749,
      "learning_rate": 1.4832644628099173e-06,
      "logits/chosen": -0.10693359375,
      "logits/rejected": -0.265625,
      "logps/chosen": -396.0,
      "logps/rejected": -338.0,
      "loss": 0.3424,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.65625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.171875,
      "step": 52
    },
    {
      "epoch": 0.06503067484662577,
      "grad_norm": 37.089420431960214,
      "learning_rate": 1.4826446280991737e-06,
      "logits/chosen": -0.055908203125,
      "logits/rejected": -0.189453125,
      "logps/chosen": -398.0,
      "logps/rejected": -326.0,
      "loss": 0.3026,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.953125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -2.75,
      "step": 53
    },
    {
      "epoch": 0.06625766871165645,
      "grad_norm": 35.67709547242796,
      "learning_rate": 1.4820247933884298e-06,
      "logits/chosen": 0.045654296875,
      "logits/rejected": -0.1162109375,
      "logps/chosen": -336.0,
      "logps/rejected": -266.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 4.625,
      "rewards/rejected": -3.296875,
      "step": 54
    },
    {
      "epoch": 0.06748466257668712,
      "grad_norm": 40.58362122970572,
      "learning_rate": 1.481404958677686e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -376.0,
      "logps/rejected": -302.0,
      "loss": 0.3515,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.8046875,
      "rewards/margins": 5.03125,
      "rewards/rejected": -3.234375,
      "step": 55
    },
    {
      "epoch": 0.0687116564417178,
      "grad_norm": 43.576103099929334,
      "learning_rate": 1.4807851239669423e-06,
      "logits/chosen": -0.034912109375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -384.0,
      "logps/rejected": -334.0,
      "loss": 0.4139,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.015625,
      "rewards/margins": 4.8125,
      "rewards/rejected": -2.796875,
      "step": 56
    },
    {
      "epoch": 0.06993865030674846,
      "grad_norm": 51.562196128944976,
      "learning_rate": 1.4801652892561983e-06,
      "logits/chosen": -0.042724609375,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -352.0,
      "logps/rejected": -326.0,
      "loss": 0.4667,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.6015625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -2.984375,
      "step": 57
    },
    {
      "epoch": 0.07116564417177915,
      "grad_norm": 49.38086728368548,
      "learning_rate": 1.4795454545454546e-06,
      "logits/chosen": -0.046630859375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.640625,
      "rewards/margins": 4.875,
      "rewards/rejected": -3.234375,
      "step": 58
    },
    {
      "epoch": 0.07239263803680981,
      "grad_norm": 43.27222220894176,
      "learning_rate": 1.4789256198347107e-06,
      "logits/chosen": 0.0240478515625,
      "logits/rejected": -0.051025390625,
      "logps/chosen": -378.0,
      "logps/rejected": -302.0,
      "loss": 0.3983,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.3125,
      "rewards/margins": 4.25,
      "rewards/rejected": -2.953125,
      "step": 59
    },
    {
      "epoch": 0.0736196319018405,
      "grad_norm": 39.673404465674906,
      "learning_rate": 1.4783057851239669e-06,
      "logits/chosen": -0.049072265625,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -386.0,
      "logps/rejected": -294.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -3.1875,
      "step": 60
    },
    {
      "epoch": 0.07484662576687116,
      "grad_norm": 40.98812217259802,
      "learning_rate": 1.4776859504132232e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.173828125,
      "logps/chosen": -418.0,
      "logps/rejected": -328.0,
      "loss": 0.3496,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.890625,
      "rewards/margins": 5.65625,
      "rewards/rejected": -3.765625,
      "step": 61
    },
    {
      "epoch": 0.07607361963190185,
      "grad_norm": 40.09146702222396,
      "learning_rate": 1.4770661157024794e-06,
      "logits/chosen": -0.051513671875,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -426.0,
      "logps/rejected": -316.0,
      "loss": 0.3416,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -3.640625,
      "step": 62
    },
    {
      "epoch": 0.07730061349693251,
      "grad_norm": 46.30805404508898,
      "learning_rate": 1.4764462809917355e-06,
      "logits/chosen": 0.031982421875,
      "logits/rejected": -0.11767578125,
      "logps/chosen": -418.0,
      "logps/rejected": -314.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.515625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.328125,
      "step": 63
    },
    {
      "epoch": 0.0785276073619632,
      "grad_norm": 47.96974437982722,
      "learning_rate": 1.4758264462809919e-06,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.27734375,
      "logps/chosen": -402.0,
      "logps/rejected": -336.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.046875,
      "rewards/margins": 5.46875,
      "rewards/rejected": -3.4375,
      "step": 64
    },
    {
      "epoch": 0.07975460122699386,
      "grad_norm": 35.57381547040094,
      "learning_rate": 1.475206611570248e-06,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.25390625,
      "logps/chosen": -390.0,
      "logps/rejected": -310.0,
      "loss": 0.2899,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.046875,
      "rewards/margins": 5.53125,
      "rewards/rejected": -3.484375,
      "step": 65
    },
    {
      "epoch": 0.08098159509202454,
      "grad_norm": 33.414611619044024,
      "learning_rate": 1.4745867768595042e-06,
      "logits/chosen": -0.162109375,
      "logits/rejected": -0.287109375,
      "logps/chosen": -314.0,
      "logps/rejected": -270.0,
      "loss": 0.2704,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.78125,
      "step": 66
    },
    {
      "epoch": 0.08220858895705521,
      "grad_norm": 42.668537708095855,
      "learning_rate": 1.4739669421487603e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.1767578125,
      "logps/chosen": -332.0,
      "logps/rejected": -302.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.2578125,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.578125,
      "step": 67
    },
    {
      "epoch": 0.0834355828220859,
      "grad_norm": 40.70660622767979,
      "learning_rate": 1.4733471074380164e-06,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.193359375,
      "logps/chosen": -358.0,
      "logps/rejected": -338.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.625,
      "rewards/margins": 4.96875,
      "rewards/rejected": -3.359375,
      "step": 68
    },
    {
      "epoch": 0.08466257668711656,
      "grad_norm": 42.29365611407333,
      "learning_rate": 1.4727272727272728e-06,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.36328125,
      "logps/chosen": -406.0,
      "logps/rejected": -290.0,
      "loss": 0.3163,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.484375,
      "rewards/margins": 6.0,
      "rewards/rejected": -3.515625,
      "step": 69
    },
    {
      "epoch": 0.08588957055214724,
      "grad_norm": 45.16433653526153,
      "learning_rate": 1.472107438016529e-06,
      "logits/chosen": -0.08447265625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -406.0,
      "logps/rejected": -346.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.515625,
      "rewards/margins": 5.6875,
      "rewards/rejected": -3.15625,
      "step": 70
    },
    {
      "epoch": 0.08711656441717791,
      "grad_norm": 37.819952615726415,
      "learning_rate": 1.4714876033057853e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.1005859375,
      "logps/chosen": -344.0,
      "logps/rejected": -286.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.46875,
      "rewards/margins": 4.625,
      "rewards/rejected": -3.15625,
      "step": 71
    },
    {
      "epoch": 0.08834355828220859,
      "grad_norm": 41.96140458025261,
      "learning_rate": 1.4708677685950414e-06,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.33203125,
      "logps/chosen": -428.0,
      "logps/rejected": -354.0,
      "loss": 0.3187,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.125,
      "rewards/margins": 5.1875,
      "rewards/rejected": -3.0625,
      "step": 72
    },
    {
      "epoch": 0.08957055214723926,
      "grad_norm": 45.003627256411235,
      "learning_rate": 1.4702479338842976e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.220703125,
      "logps/chosen": -346.0,
      "logps/rejected": -308.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 5.25,
      "rewards/rejected": -3.625,
      "step": 73
    },
    {
      "epoch": 0.09079754601226994,
      "grad_norm": 37.795821367857755,
      "learning_rate": 1.4696280991735537e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.212890625,
      "logps/chosen": -352.0,
      "logps/rejected": -338.0,
      "loss": 0.2989,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 5.0625,
      "rewards/rejected": -3.625,
      "step": 74
    },
    {
      "epoch": 0.09202453987730061,
      "grad_norm": 38.6325533255864,
      "learning_rate": 1.4690082644628099e-06,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -376.0,
      "logps/rejected": -326.0,
      "loss": 0.3138,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.265625,
      "rewards/margins": 5.9375,
      "rewards/rejected": -3.671875,
      "step": 75
    },
    {
      "epoch": 0.09325153374233129,
      "grad_norm": 41.500214372239896,
      "learning_rate": 1.4683884297520662e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -394.0,
      "logps/rejected": -296.0,
      "loss": 0.3196,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.28125,
      "rewards/margins": 5.625,
      "rewards/rejected": -3.359375,
      "step": 76
    },
    {
      "epoch": 0.09447852760736196,
      "grad_norm": 37.661301568651126,
      "learning_rate": 1.4677685950413223e-06,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -350.0,
      "logps/rejected": -308.0,
      "loss": 0.3298,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.0625,
      "step": 77
    },
    {
      "epoch": 0.09570552147239264,
      "grad_norm": 37.401749661903814,
      "learning_rate": 1.4671487603305785e-06,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -414.0,
      "logps/rejected": -324.0,
      "loss": 0.3295,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.015625,
      "rewards/margins": 5.90625,
      "rewards/rejected": -3.90625,
      "step": 78
    },
    {
      "epoch": 0.09693251533742331,
      "grad_norm": 41.34926796893681,
      "learning_rate": 1.4665289256198348e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2109375,
      "logps/chosen": -354.0,
      "logps/rejected": -300.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.25,
      "rewards/margins": 5.53125,
      "rewards/rejected": -4.28125,
      "step": 79
    },
    {
      "epoch": 0.09815950920245399,
      "grad_norm": 40.52807889159641,
      "learning_rate": 1.465909090909091e-06,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -404.0,
      "logps/rejected": -340.0,
      "loss": 0.3387,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.484375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -4.75,
      "step": 80
    },
    {
      "epoch": 0.09938650306748466,
      "grad_norm": 35.605812619698284,
      "learning_rate": 1.4652892561983471e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": -0.365234375,
      "logps/chosen": -436.0,
      "logps/rejected": -356.0,
      "loss": 0.3191,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.8828125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.78125,
      "step": 81
    },
    {
      "epoch": 0.10061349693251534,
      "grad_norm": 47.4617133344908,
      "learning_rate": 1.4646694214876035e-06,
      "logits/chosen": -0.1171875,
      "logits/rejected": -0.228515625,
      "logps/chosen": -430.0,
      "logps/rejected": -348.0,
      "loss": 0.3954,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.84375,
      "step": 82
    },
    {
      "epoch": 0.10184049079754601,
      "grad_norm": 42.72902928212931,
      "learning_rate": 1.4640495867768594e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.255859375,
      "logps/chosen": -352.0,
      "logps/rejected": -320.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.71875,
      "step": 83
    },
    {
      "epoch": 0.10306748466257669,
      "grad_norm": 59.351571106495314,
      "learning_rate": 1.4634297520661158e-06,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.310546875,
      "logps/chosen": -448.0,
      "logps/rejected": -360.0,
      "loss": 0.2887,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.0625,
      "step": 84
    },
    {
      "epoch": 0.10429447852760736,
      "grad_norm": 40.67504361329859,
      "learning_rate": 1.462809917355372e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -376.0,
      "logps/rejected": -338.0,
      "loss": 0.2933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.96875,
      "step": 85
    },
    {
      "epoch": 0.10552147239263804,
      "grad_norm": 44.45491286199163,
      "learning_rate": 1.462190082644628e-06,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.224609375,
      "logps/chosen": -424.0,
      "logps/rejected": -314.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.625,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.25,
      "step": 86
    },
    {
      "epoch": 0.1067484662576687,
      "grad_norm": 37.230987939956755,
      "learning_rate": 1.4615702479338844e-06,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.3046875,
      "logps/chosen": -354.0,
      "logps/rejected": -274.0,
      "loss": 0.3433,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0,
      "step": 87
    },
    {
      "epoch": 0.10797546012269939,
      "grad_norm": 35.435756690174514,
      "learning_rate": 1.4609504132231405e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -362.0,
      "logps/rejected": -286.0,
      "loss": 0.2796,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.5,
      "step": 88
    },
    {
      "epoch": 0.10920245398773006,
      "grad_norm": 39.218068339324205,
      "learning_rate": 1.4603305785123967e-06,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.2734375,
      "logps/chosen": -392.0,
      "logps/rejected": -340.0,
      "loss": 0.3661,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.609375,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.28125,
      "step": 89
    },
    {
      "epoch": 0.11042944785276074,
      "grad_norm": 35.86351693081174,
      "learning_rate": 1.459710743801653e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.375,
      "logps/chosen": -378.0,
      "logps/rejected": -334.0,
      "loss": 0.3305,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.0625,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.625,
      "step": 90
    },
    {
      "epoch": 0.1116564417177914,
      "grad_norm": 39.260039435042906,
      "learning_rate": 1.4590909090909092e-06,
      "logits/chosen": -0.028076171875,
      "logits/rejected": -0.28125,
      "logps/chosen": -392.0,
      "logps/rejected": -308.0,
      "loss": 0.3104,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.5234375,
      "rewards/margins": 5.9375,
      "rewards/rejected": -4.40625,
      "step": 91
    },
    {
      "epoch": 0.11288343558282209,
      "grad_norm": 35.99771043781269,
      "learning_rate": 1.4584710743801653e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.236328125,
      "logps/chosen": -390.0,
      "logps/rejected": -316.0,
      "loss": 0.304,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.484375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.8125,
      "step": 92
    },
    {
      "epoch": 0.11411042944785275,
      "grad_norm": 38.41867053252873,
      "learning_rate": 1.4578512396694215e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.36328125,
      "logps/chosen": -400.0,
      "logps/rejected": -310.0,
      "loss": 0.3311,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 6.625,
      "rewards/rejected": -4.3125,
      "step": 93
    },
    {
      "epoch": 0.11533742331288344,
      "grad_norm": 37.099757310606805,
      "learning_rate": 1.4572314049586776e-06,
      "logits/chosen": -0.171875,
      "logits/rejected": -0.390625,
      "logps/chosen": -426.0,
      "logps/rejected": -354.0,
      "loss": 0.2709,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.765625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -4.3125,
      "step": 94
    },
    {
      "epoch": 0.1165644171779141,
      "grad_norm": 36.042450420967036,
      "learning_rate": 1.456611570247934e-06,
      "logits/chosen": -0.2490234375,
      "logits/rejected": -0.392578125,
      "logps/chosen": -378.0,
      "logps/rejected": -320.0,
      "loss": 0.3138,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.0625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -4.625,
      "step": 95
    },
    {
      "epoch": 0.11779141104294479,
      "grad_norm": 37.51605196372948,
      "learning_rate": 1.45599173553719e-06,
      "logits/chosen": -0.2392578125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -366.0,
      "logps/rejected": -320.0,
      "loss": 0.2944,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.625,
      "rewards/margins": 7.125,
      "rewards/rejected": -4.5,
      "step": 96
    },
    {
      "epoch": 0.11901840490797547,
      "grad_norm": 47.87658843319023,
      "learning_rate": 1.4553719008264462e-06,
      "logits/chosen": -0.234375,
      "logits/rejected": -0.373046875,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4367,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.25,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.859375,
      "step": 97
    },
    {
      "epoch": 0.12024539877300613,
      "grad_norm": 37.61598969810632,
      "learning_rate": 1.4547520661157026e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.236328125,
      "logps/chosen": -354.0,
      "logps/rejected": -296.0,
      "loss": 0.3071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.984375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -4.09375,
      "step": 98
    },
    {
      "epoch": 0.12147239263803682,
      "grad_norm": 35.33546250741774,
      "learning_rate": 1.4541322314049587e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -338.0,
      "logps/rejected": -258.0,
      "loss": 0.2669,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.359375,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.15625,
      "step": 99
    },
    {
      "epoch": 0.12269938650306748,
      "grad_norm": 32.14334216613253,
      "learning_rate": 1.453512396694215e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.404296875,
      "logps/chosen": -394.0,
      "logps/rejected": -332.0,
      "loss": 0.2858,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.8359375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.625,
      "step": 100
    },
    {
      "epoch": 0.12392638036809817,
      "grad_norm": 45.60783475518701,
      "learning_rate": 1.452892561983471e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -412.0,
      "logps/rejected": -352.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.375,
      "step": 101
    },
    {
      "epoch": 0.12515337423312883,
      "grad_norm": 39.740396682339004,
      "learning_rate": 1.4522727272727272e-06,
      "logits/chosen": -0.1865234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -392.0,
      "logps/rejected": -330.0,
      "loss": 0.3534,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 6.0,
      "rewards/rejected": -3.9375,
      "step": 102
    },
    {
      "epoch": 0.1263803680981595,
      "grad_norm": 35.16122653572647,
      "learning_rate": 1.4516528925619835e-06,
      "logits/chosen": -0.17578125,
      "logits/rejected": -0.326171875,
      "logps/chosen": -350.0,
      "logps/rejected": -310.0,
      "loss": 0.3153,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.4765625,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.28125,
      "step": 103
    },
    {
      "epoch": 0.1276073619631902,
      "grad_norm": 46.63941850097279,
      "learning_rate": 1.4510330578512396e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.2255859375,
      "logps/chosen": -414.0,
      "logps/rejected": -348.0,
      "loss": 0.381,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.6484375,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.125,
      "step": 104
    },
    {
      "epoch": 0.12883435582822086,
      "grad_norm": 35.04246201772006,
      "learning_rate": 1.450413223140496e-06,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.3828125,
      "logps/chosen": -374.0,
      "logps/rejected": -322.0,
      "loss": 0.2864,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.03125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -4.78125,
      "step": 105
    },
    {
      "epoch": 0.13006134969325153,
      "grad_norm": 39.36563074587024,
      "learning_rate": 1.4497933884297521e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -392.0,
      "logps/rejected": -322.0,
      "loss": 0.3462,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.7890625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.25,
      "step": 106
    },
    {
      "epoch": 0.1312883435582822,
      "grad_norm": 35.07229699022507,
      "learning_rate": 1.4491735537190083e-06,
      "logits/chosen": -0.1865234375,
      "logits/rejected": -0.26953125,
      "logps/chosen": -352.0,
      "logps/rejected": -312.0,
      "loss": 0.2577,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.3515625,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.5,
      "step": 107
    },
    {
      "epoch": 0.1325153374233129,
      "grad_norm": 28.019513861884207,
      "learning_rate": 1.4485537190082646e-06,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.2438,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -5.25,
      "step": 108
    },
    {
      "epoch": 0.13374233128834356,
      "grad_norm": 35.40605672013622,
      "learning_rate": 1.4479338842975208e-06,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.359375,
      "logps/chosen": -412.0,
      "logps/rejected": -326.0,
      "loss": 0.3584,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.5625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -4.625,
      "step": 109
    },
    {
      "epoch": 0.13496932515337423,
      "grad_norm": 38.14439771999889,
      "learning_rate": 1.447314049586777e-06,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.310546875,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.3481,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.53125,
      "step": 110
    },
    {
      "epoch": 0.1361963190184049,
      "grad_norm": 36.779042619503116,
      "learning_rate": 1.446694214876033e-06,
      "logits/chosen": -0.09716796875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -334.0,
      "logps/rejected": -282.0,
      "loss": 0.3363,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3671875,
      "rewards/margins": 5.4375,
      "rewards/rejected": -4.0625,
      "step": 111
    },
    {
      "epoch": 0.1374233128834356,
      "grad_norm": 36.86435500620325,
      "learning_rate": 1.4460743801652892e-06,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.29296875,
      "logps/chosen": -342.0,
      "logps/rejected": -292.0,
      "loss": 0.343,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.4375,
      "step": 112
    },
    {
      "epoch": 0.13865030674846626,
      "grad_norm": 38.139474814179394,
      "learning_rate": 1.4454545454545456e-06,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.169921875,
      "logps/chosen": -374.0,
      "logps/rejected": -312.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.859375,
      "rewards/margins": 5.75,
      "rewards/rejected": -3.90625,
      "step": 113
    },
    {
      "epoch": 0.13987730061349693,
      "grad_norm": 38.81539599706057,
      "learning_rate": 1.4448347107438017e-06,
      "logits/chosen": -0.1220703125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -372.0,
      "logps/rejected": -316.0,
      "loss": 0.3854,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 5.375,
      "rewards/rejected": -3.875,
      "step": 114
    },
    {
      "epoch": 0.1411042944785276,
      "grad_norm": 45.65683573404254,
      "learning_rate": 1.4442148760330578e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -338.0,
      "logps/rejected": -320.0,
      "loss": 0.4779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.046875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -3.65625,
      "step": 115
    },
    {
      "epoch": 0.1423312883435583,
      "grad_norm": 32.97644733528144,
      "learning_rate": 1.4435950413223142e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -376.0,
      "logps/rejected": -324.0,
      "loss": 0.2978,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.40625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -3.625,
      "step": 116
    },
    {
      "epoch": 0.14355828220858896,
      "grad_norm": 38.235071091905795,
      "learning_rate": 1.4429752066115703e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.26953125,
      "logps/chosen": -324.0,
      "logps/rejected": -270.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -4.21875,
      "step": 117
    },
    {
      "epoch": 0.14478527607361963,
      "grad_norm": 41.12118912071453,
      "learning_rate": 1.4423553719008265e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -418.0,
      "logps/rejected": -360.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 2.28125,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.84375,
      "step": 118
    },
    {
      "epoch": 0.1460122699386503,
      "grad_norm": 35.626041219957244,
      "learning_rate": 1.4417355371900826e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -352.0,
      "logps/rejected": -324.0,
      "loss": 0.3003,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -3.90625,
      "step": 119
    },
    {
      "epoch": 0.147239263803681,
      "grad_norm": 40.42176001046704,
      "learning_rate": 1.4411157024793388e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.37890625,
      "logps/chosen": -418.0,
      "logps/rejected": -354.0,
      "loss": 0.3506,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.6875,
      "rewards/margins": 6.625,
      "rewards/rejected": -3.9375,
      "step": 120
    },
    {
      "epoch": 0.14846625766871166,
      "grad_norm": 33.17627892781642,
      "learning_rate": 1.4404958677685951e-06,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.27734375,
      "logps/chosen": -404.0,
      "logps/rejected": -336.0,
      "loss": 0.2824,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.625,
      "rewards/margins": 6.75,
      "rewards/rejected": -4.125,
      "step": 121
    },
    {
      "epoch": 0.14969325153374233,
      "grad_norm": 41.361282319998544,
      "learning_rate": 1.4398760330578512e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -378.0,
      "logps/rejected": -298.0,
      "loss": 0.3454,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.03125,
      "step": 122
    },
    {
      "epoch": 0.150920245398773,
      "grad_norm": 39.0679541541933,
      "learning_rate": 1.4392561983471074e-06,
      "logits/chosen": -0.0191650390625,
      "logits/rejected": -0.1884765625,
      "logps/chosen": -380.0,
      "logps/rejected": -290.0,
      "loss": 0.384,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.2734375,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.375,
      "step": 123
    },
    {
      "epoch": 0.1521472392638037,
      "grad_norm": 30.841344702577548,
      "learning_rate": 1.4386363636363637e-06,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -348.0,
      "logps/rejected": -294.0,
      "loss": 0.2792,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 7.0,
      "rewards/rejected": -4.8125,
      "step": 124
    },
    {
      "epoch": 0.15337423312883436,
      "grad_norm": 40.57809799655024,
      "learning_rate": 1.4380165289256199e-06,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -434.0,
      "logps/rejected": -340.0,
      "loss": 0.3257,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.6015625,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.0,
      "step": 125
    },
    {
      "epoch": 0.15460122699386503,
      "grad_norm": 49.176910835221456,
      "learning_rate": 1.437396694214876e-06,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.30078125,
      "logps/chosen": -368.0,
      "logps/rejected": -320.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6328125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -4.96875,
      "step": 126
    },
    {
      "epoch": 0.1558282208588957,
      "grad_norm": 42.06253723411312,
      "learning_rate": 1.4367768595041324e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -388.0,
      "logps/rejected": -340.0,
      "loss": 0.4543,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.40625,
      "rewards/rejected": -4.65625,
      "step": 127
    },
    {
      "epoch": 0.1570552147239264,
      "grad_norm": 49.94695723772631,
      "learning_rate": 1.4361570247933883e-06,
      "logits/chosen": -0.1044921875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -374.0,
      "logps/rejected": -344.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.09375,
      "step": 128
    },
    {
      "epoch": 0.15828220858895706,
      "grad_norm": 31.69804344527413,
      "learning_rate": 1.4355371900826447e-06,
      "logits/chosen": -0.00439453125,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -342.0,
      "logps/rejected": -302.0,
      "loss": 0.3139,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.78125,
      "step": 129
    },
    {
      "epoch": 0.15950920245398773,
      "grad_norm": 39.02664413781982,
      "learning_rate": 1.4349173553719008e-06,
      "logits/chosen": -0.11328125,
      "logits/rejected": -0.251953125,
      "logps/chosen": -388.0,
      "logps/rejected": -324.0,
      "loss": 0.3918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.375,
      "step": 130
    },
    {
      "epoch": 0.1607361963190184,
      "grad_norm": 33.65136979573152,
      "learning_rate": 1.434297520661157e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.2429,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.3671875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -5.59375,
      "step": 131
    },
    {
      "epoch": 0.1619631901840491,
      "grad_norm": 40.70444309722836,
      "learning_rate": 1.4336776859504133e-06,
      "logits/chosen": -0.1845703125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -392.0,
      "logps/rejected": -336.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.15625,
      "step": 132
    },
    {
      "epoch": 0.16319018404907976,
      "grad_norm": 38.39829631153826,
      "learning_rate": 1.4330578512396694e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.375,
      "logps/chosen": -388.0,
      "logps/rejected": -342.0,
      "loss": 0.3065,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.46875,
      "step": 133
    },
    {
      "epoch": 0.16441717791411042,
      "grad_norm": 54.071557291057594,
      "learning_rate": 1.4324380165289258e-06,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -458.0,
      "logps/rejected": -374.0,
      "loss": 0.4193,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -4.625,
      "step": 134
    },
    {
      "epoch": 0.1656441717791411,
      "grad_norm": 42.09400341898127,
      "learning_rate": 1.431818181818182e-06,
      "logits/chosen": -0.0140380859375,
      "logits/rejected": -0.11767578125,
      "logps/chosen": -378.0,
      "logps/rejected": -330.0,
      "loss": 0.4491,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.16015625,
      "rewards/margins": 5.21875,
      "rewards/rejected": -5.0625,
      "step": 135
    },
    {
      "epoch": 0.1668711656441718,
      "grad_norm": 34.790669543589566,
      "learning_rate": 1.431198347107438e-06,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.3203125,
      "logps/chosen": -392.0,
      "logps/rejected": -322.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.87890625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.46875,
      "step": 136
    },
    {
      "epoch": 0.16809815950920245,
      "grad_norm": 46.051678091055635,
      "learning_rate": 1.4305785123966942e-06,
      "logits/chosen": -0.020751953125,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -398.0,
      "logps/rejected": -320.0,
      "loss": 0.4495,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.609375,
      "rewards/margins": 5.21875,
      "rewards/rejected": -4.625,
      "step": 137
    },
    {
      "epoch": 0.16932515337423312,
      "grad_norm": 39.60892351140069,
      "learning_rate": 1.4299586776859504e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.2421875,
      "logps/chosen": -406.0,
      "logps/rejected": -360.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 5.59375,
      "rewards/rejected": -5.0,
      "step": 138
    },
    {
      "epoch": 0.1705521472392638,
      "grad_norm": 44.067268759137896,
      "learning_rate": 1.4293388429752067e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.2734375,
      "logps/chosen": -418.0,
      "logps/rejected": -380.0,
      "loss": 0.4261,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.59375,
      "step": 139
    },
    {
      "epoch": 0.17177914110429449,
      "grad_norm": 46.60040949853179,
      "learning_rate": 1.4287190082644628e-06,
      "logits/chosen": -0.07177734375,
      "logits/rejected": -0.251953125,
      "logps/chosen": -426.0,
      "logps/rejected": -376.0,
      "loss": 0.4219,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.1875,
      "step": 140
    },
    {
      "epoch": 0.17300613496932515,
      "grad_norm": 34.208928830023055,
      "learning_rate": 1.428099173553719e-06,
      "logits/chosen": -0.05126953125,
      "logits/rejected": -0.173828125,
      "logps/chosen": -358.0,
      "logps/rejected": -338.0,
      "loss": 0.3019,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -5.28125,
      "step": 141
    },
    {
      "epoch": 0.17423312883435582,
      "grad_norm": 41.96948447879753,
      "learning_rate": 1.4274793388429753e-06,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -400.0,
      "logps/rejected": -354.0,
      "loss": 0.4327,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.98828125,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.375,
      "step": 142
    },
    {
      "epoch": 0.1754601226993865,
      "grad_norm": 39.736356252739405,
      "learning_rate": 1.4268595041322315e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.33984375,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.3314,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -5.625,
      "step": 143
    },
    {
      "epoch": 0.17668711656441718,
      "grad_norm": 40.38397393664774,
      "learning_rate": 1.4262396694214876e-06,
      "logits/chosen": -0.224609375,
      "logits/rejected": -0.3984375,
      "logps/chosen": -420.0,
      "logps/rejected": -356.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.1875,
      "step": 144
    },
    {
      "epoch": 0.17791411042944785,
      "grad_norm": 36.78367346174047,
      "learning_rate": 1.425619834710744e-06,
      "logits/chosen": -0.20703125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -380.0,
      "logps/rejected": -348.0,
      "loss": 0.2992,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.21875,
      "step": 145
    },
    {
      "epoch": 0.17914110429447852,
      "grad_norm": 30.78038936070514,
      "learning_rate": 1.425e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.30859375,
      "logps/chosen": -388.0,
      "logps/rejected": -334.0,
      "loss": 0.2674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.53125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.5,
      "step": 146
    },
    {
      "epoch": 0.18036809815950922,
      "grad_norm": 33.88526702598482,
      "learning_rate": 1.4243801652892563e-06,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -388.0,
      "logps/rejected": -332.0,
      "loss": 0.3151,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.3125,
      "step": 147
    },
    {
      "epoch": 0.18159509202453988,
      "grad_norm": 49.54475790087749,
      "learning_rate": 1.4237603305785124e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -416.0,
      "logps/rejected": -368.0,
      "loss": 0.4146,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.34375,
      "step": 148
    },
    {
      "epoch": 0.18282208588957055,
      "grad_norm": 43.24095049009772,
      "learning_rate": 1.4231404958677685e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.244140625,
      "logps/chosen": -406.0,
      "logps/rejected": -384.0,
      "loss": 0.3173,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.03125,
      "step": 149
    },
    {
      "epoch": 0.18404907975460122,
      "grad_norm": 37.7299116363035,
      "learning_rate": 1.422520661157025e-06,
      "logits/chosen": -0.1181640625,
      "logits/rejected": -0.291015625,
      "logps/chosen": -424.0,
      "logps/rejected": -346.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.53125,
      "step": 150
    },
    {
      "epoch": 0.18527607361963191,
      "grad_norm": 36.98087652686202,
      "learning_rate": 1.421900826446281e-06,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -382.0,
      "logps/rejected": -326.0,
      "loss": 0.3467,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.625,
      "step": 151
    },
    {
      "epoch": 0.18650306748466258,
      "grad_norm": 33.96526048608977,
      "learning_rate": 1.4212809917355372e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.462890625,
      "logps/chosen": -402.0,
      "logps/rejected": -316.0,
      "loss": 0.322,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.5625,
      "rewards/rejected": -5.71875,
      "step": 152
    },
    {
      "epoch": 0.18773006134969325,
      "grad_norm": 34.59925431133825,
      "learning_rate": 1.4206611570247935e-06,
      "logits/chosen": -0.1875,
      "logits/rejected": -0.298828125,
      "logps/chosen": -350.0,
      "logps/rejected": -320.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.390625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -4.875,
      "step": 153
    },
    {
      "epoch": 0.18895705521472392,
      "grad_norm": 45.111496339674666,
      "learning_rate": 1.4200413223140497e-06,
      "logits/chosen": -0.060546875,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -424.0,
      "logps/rejected": -340.0,
      "loss": 0.3836,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.703125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -4.375,
      "step": 154
    },
    {
      "epoch": 0.1901840490797546,
      "grad_norm": 28.559326947209748,
      "learning_rate": 1.4194214876033058e-06,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.41796875,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.2127,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.359375,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.03125,
      "step": 155
    },
    {
      "epoch": 0.19141104294478528,
      "grad_norm": 41.969901464316806,
      "learning_rate": 1.418801652892562e-06,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -370.0,
      "logps/rejected": -350.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 5.59375,
      "rewards/rejected": -4.46875,
      "step": 156
    },
    {
      "epoch": 0.19263803680981595,
      "grad_norm": 35.115688822084685,
      "learning_rate": 1.418181818181818e-06,
      "logits/chosen": -0.05810546875,
      "logits/rejected": -0.1796875,
      "logps/chosen": -394.0,
      "logps/rejected": -310.0,
      "loss": 0.3364,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.625,
      "step": 157
    },
    {
      "epoch": 0.19386503067484662,
      "grad_norm": 29.216146857412586,
      "learning_rate": 1.4175619834710745e-06,
      "logits/chosen": -0.2734375,
      "logits/rejected": -0.44921875,
      "logps/chosen": -388.0,
      "logps/rejected": -324.0,
      "loss": 0.2283,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.390625,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.21875,
      "step": 158
    },
    {
      "epoch": 0.1950920245398773,
      "grad_norm": 39.305920966554595,
      "learning_rate": 1.4169421487603306e-06,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.28125,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.3649,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.359375,
      "rewards/margins": 5.71875,
      "rewards/rejected": -4.34375,
      "step": 159
    },
    {
      "epoch": 0.19631901840490798,
      "grad_norm": 26.74081618489141,
      "learning_rate": 1.416322314049587e-06,
      "logits/chosen": -0.047607421875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -354.0,
      "logps/rejected": -310.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.15625,
      "step": 160
    },
    {
      "epoch": 0.19754601226993865,
      "grad_norm": 32.344289601166764,
      "learning_rate": 1.415702479338843e-06,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.4296875,
      "logps/chosen": -386.0,
      "logps/rejected": -302.0,
      "loss": 0.2831,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.9453125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -4.71875,
      "step": 161
    },
    {
      "epoch": 0.19877300613496932,
      "grad_norm": 39.80838608230206,
      "learning_rate": 1.4150826446280992e-06,
      "logits/chosen": -0.1357421875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -350.0,
      "logps/rejected": -340.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.375,
      "step": 162
    },
    {
      "epoch": 0.2,
      "grad_norm": 38.00229232317159,
      "learning_rate": 1.4144628099173554e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.220703125,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.421875,
      "rewards/margins": 6.25,
      "rewards/rejected": -4.8125,
      "step": 163
    },
    {
      "epoch": 0.20122699386503068,
      "grad_norm": 31.02665679742822,
      "learning_rate": 1.4138429752066115e-06,
      "logits/chosen": -0.236328125,
      "logits/rejected": -0.466796875,
      "logps/chosen": -392.0,
      "logps/rejected": -310.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.15625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -5.1875,
      "step": 164
    },
    {
      "epoch": 0.20245398773006135,
      "grad_norm": 43.699969414714815,
      "learning_rate": 1.4132231404958679e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.212890625,
      "logps/chosen": -444.0,
      "logps/rejected": -326.0,
      "loss": 0.377,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.78125,
      "step": 165
    },
    {
      "epoch": 0.20368098159509201,
      "grad_norm": 38.30377721360019,
      "learning_rate": 1.412603305785124e-06,
      "logits/chosen": -0.09375,
      "logits/rejected": -0.314453125,
      "logps/chosen": -388.0,
      "logps/rejected": -336.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.53125,
      "rewards/rejected": -4.625,
      "step": 166
    },
    {
      "epoch": 0.2049079754601227,
      "grad_norm": 38.75636778509963,
      "learning_rate": 1.4119834710743801e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -398.0,
      "logps/rejected": -318.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.21875,
      "rewards/margins": 6.875,
      "rewards/rejected": -4.65625,
      "step": 167
    },
    {
      "epoch": 0.20613496932515338,
      "grad_norm": 37.779783793801464,
      "learning_rate": 1.4113636363636365e-06,
      "logits/chosen": -0.049560546875,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -348.0,
      "logps/rejected": -292.0,
      "loss": 0.3337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.46875,
      "step": 168
    },
    {
      "epoch": 0.20736196319018405,
      "grad_norm": 34.571770475148426,
      "learning_rate": 1.4107438016528926e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.27734375,
      "logps/chosen": -372.0,
      "logps/rejected": -366.0,
      "loss": 0.3006,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.25,
      "step": 169
    },
    {
      "epoch": 0.2085889570552147,
      "grad_norm": 42.205138483103795,
      "learning_rate": 1.4101239669421488e-06,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -450.0,
      "logps/rejected": -346.0,
      "loss": 0.338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -5.46875,
      "step": 170
    },
    {
      "epoch": 0.2098159509202454,
      "grad_norm": 29.828895289514126,
      "learning_rate": 1.4095041322314051e-06,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -384.0,
      "logps/rejected": -322.0,
      "loss": 0.3086,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -5.90625,
      "step": 171
    },
    {
      "epoch": 0.21104294478527608,
      "grad_norm": 30.980573058674597,
      "learning_rate": 1.408884297520661e-06,
      "logits/chosen": -0.062255859375,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -438.0,
      "logps/rejected": -354.0,
      "loss": 0.2599,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.84375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.0,
      "step": 172
    },
    {
      "epoch": 0.21226993865030674,
      "grad_norm": 25.22446362010358,
      "learning_rate": 1.4082644628099174e-06,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.353515625,
      "logps/chosen": -428.0,
      "logps/rejected": -362.0,
      "loss": 0.1997,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.703125,
      "rewards/margins": 8.25,
      "rewards/rejected": -6.5625,
      "step": 173
    },
    {
      "epoch": 0.2134969325153374,
      "grad_norm": 43.79333405262764,
      "learning_rate": 1.4076446280991736e-06,
      "logits/chosen": 0.05126953125,
      "logits/rejected": -0.158203125,
      "logps/chosen": -384.0,
      "logps/rejected": -348.0,
      "loss": 0.5204,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.78125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.03125,
      "step": 174
    },
    {
      "epoch": 0.2147239263803681,
      "grad_norm": 45.27680571467687,
      "learning_rate": 1.4070247933884297e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.220703125,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.09375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.25,
      "step": 175
    },
    {
      "epoch": 0.21595092024539878,
      "grad_norm": 35.16824036294685,
      "learning_rate": 1.406404958677686e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -434.0,
      "logps/rejected": -378.0,
      "loss": 0.3443,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -6.3125,
      "step": 176
    },
    {
      "epoch": 0.21717791411042944,
      "grad_norm": 27.30910780814845,
      "learning_rate": 1.4057851239669422e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -416.0,
      "logps/rejected": -356.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -6.71875,
      "step": 177
    },
    {
      "epoch": 0.2184049079754601,
      "grad_norm": 37.84132418939681,
      "learning_rate": 1.4051652892561983e-06,
      "logits/chosen": -0.07275390625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -330.0,
      "logps/rejected": -318.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.1806640625,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.59375,
      "step": 178
    },
    {
      "epoch": 0.2196319018404908,
      "grad_norm": 32.98852515847511,
      "learning_rate": 1.4045454545454547e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.31640625,
      "logps/chosen": -372.0,
      "logps/rejected": -324.0,
      "loss": 0.3252,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.671875,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.1875,
      "step": 179
    },
    {
      "epoch": 0.22085889570552147,
      "grad_norm": 38.78908549807395,
      "learning_rate": 1.4039256198347108e-06,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.21484375,
      "logps/chosen": -356.0,
      "logps/rejected": -314.0,
      "loss": 0.3669,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.234375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.21875,
      "step": 180
    },
    {
      "epoch": 0.22208588957055214,
      "grad_norm": 35.32530991965649,
      "learning_rate": 1.403305785123967e-06,
      "logits/chosen": -0.09423828125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -410.0,
      "logps/rejected": -386.0,
      "loss": 0.3373,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.25,
      "step": 181
    },
    {
      "epoch": 0.2233128834355828,
      "grad_norm": 36.9736807793252,
      "learning_rate": 1.4026859504132231e-06,
      "logits/chosen": -0.12890625,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.3661,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.109375,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.75,
      "step": 182
    },
    {
      "epoch": 0.2245398773006135,
      "grad_norm": 40.8878376560248,
      "learning_rate": 1.4020661157024793e-06,
      "logits/chosen": 0.00762939453125,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -412.0,
      "logps/rejected": -378.0,
      "loss": 0.3866,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.95703125,
      "rewards/margins": 4.90625,
      "rewards/rejected": -3.953125,
      "step": 183
    },
    {
      "epoch": 0.22576687116564417,
      "grad_norm": 42.156241310731595,
      "learning_rate": 1.4014462809917356e-06,
      "logits/chosen": -0.0255126953125,
      "logits/rejected": -0.1328125,
      "logps/chosen": -390.0,
      "logps/rejected": -340.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 5.1875,
      "rewards/rejected": -4.3125,
      "step": 184
    },
    {
      "epoch": 0.22699386503067484,
      "grad_norm": 44.943258114663124,
      "learning_rate": 1.4008264462809917e-06,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.1220703125,
      "logps/chosen": -382.0,
      "logps/rejected": -350.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.9765625,
      "rewards/margins": 5.8125,
      "rewards/rejected": -3.828125,
      "step": 185
    },
    {
      "epoch": 0.2282208588957055,
      "grad_norm": 33.47267300169336,
      "learning_rate": 1.4002066115702479e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -336.0,
      "logps/rejected": -312.0,
      "loss": 0.3163,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.3125,
      "step": 186
    },
    {
      "epoch": 0.2294478527607362,
      "grad_norm": 32.03885108074069,
      "learning_rate": 1.3995867768595042e-06,
      "logits/chosen": -0.07080078125,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -370.0,
      "logps/rejected": -340.0,
      "loss": 0.258,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.6328125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.78125,
      "step": 187
    },
    {
      "epoch": 0.23067484662576687,
      "grad_norm": 35.158897439200416,
      "learning_rate": 1.3989669421487604e-06,
      "logits/chosen": 0.0228271484375,
      "logits/rejected": -0.0615234375,
      "logps/chosen": -330.0,
      "logps/rejected": -292.0,
      "loss": 0.3609,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.40625,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.46875,
      "step": 188
    },
    {
      "epoch": 0.23190184049079754,
      "grad_norm": 32.15934523587848,
      "learning_rate": 1.3983471074380167e-06,
      "logits/chosen": -0.080078125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -396.0,
      "logps/rejected": -322.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 2.046875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.71875,
      "step": 189
    },
    {
      "epoch": 0.2331288343558282,
      "grad_norm": 32.11586724143221,
      "learning_rate": 1.3977272727272727e-06,
      "logits/chosen": 0.041748046875,
      "logits/rejected": -0.10693359375,
      "logps/chosen": -340.0,
      "logps/rejected": -316.0,
      "loss": 0.3271,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.4140625,
      "rewards/margins": 6.0,
      "rewards/rejected": -4.59375,
      "step": 190
    },
    {
      "epoch": 0.2343558282208589,
      "grad_norm": 46.26761231273777,
      "learning_rate": 1.3971074380165288e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.220703125,
      "logps/chosen": -400.0,
      "logps/rejected": -306.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.828125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -5.0625,
      "step": 191
    },
    {
      "epoch": 0.23558282208588957,
      "grad_norm": 39.15535099753523,
      "learning_rate": 1.3964876033057852e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -388.0,
      "logps/rejected": -302.0,
      "loss": 0.3379,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 5.65625,
      "rewards/rejected": -4.5,
      "step": 192
    },
    {
      "epoch": 0.23680981595092024,
      "grad_norm": 31.3365855799527,
      "learning_rate": 1.3958677685950413e-06,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.423828125,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.2485,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.625,
      "step": 193
    },
    {
      "epoch": 0.23803680981595093,
      "grad_norm": 38.5490731428379,
      "learning_rate": 1.3952479338842977e-06,
      "logits/chosen": -0.21875,
      "logits/rejected": -0.326171875,
      "logps/chosen": -394.0,
      "logps/rejected": -336.0,
      "loss": 0.296,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.96875,
      "step": 194
    },
    {
      "epoch": 0.2392638036809816,
      "grad_norm": 35.180225705298604,
      "learning_rate": 1.3946280991735538e-06,
      "logits/chosen": -0.031982421875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -408.0,
      "logps/rejected": -348.0,
      "loss": 0.2727,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 2.046875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.125,
      "step": 195
    },
    {
      "epoch": 0.24049079754601227,
      "grad_norm": 37.22166797602668,
      "learning_rate": 1.39400826446281e-06,
      "logits/chosen": 0.0157470703125,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -394.0,
      "logps/rejected": -308.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.53125,
      "step": 196
    },
    {
      "epoch": 0.24171779141104294,
      "grad_norm": 28.562917988218086,
      "learning_rate": 1.3933884297520663e-06,
      "logits/chosen": -0.115234375,
      "logits/rejected": -0.34765625,
      "logps/chosen": -380.0,
      "logps/rejected": -334.0,
      "loss": 0.3009,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.125,
      "rewards/rejected": -5.78125,
      "step": 197
    },
    {
      "epoch": 0.24294478527607363,
      "grad_norm": 36.54466818998776,
      "learning_rate": 1.3927685950413224e-06,
      "logits/chosen": -0.0174560546875,
      "logits/rejected": -0.10205078125,
      "logps/chosen": -346.0,
      "logps/rejected": -276.0,
      "loss": 0.4465,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -5.4375,
      "step": 198
    },
    {
      "epoch": 0.2441717791411043,
      "grad_norm": 25.79094354827764,
      "learning_rate": 1.3921487603305786e-06,
      "logits/chosen": -0.05615234375,
      "logits/rejected": -0.26171875,
      "logps/chosen": -434.0,
      "logps/rejected": -330.0,
      "loss": 0.1969,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.7265625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.125,
      "step": 199
    },
    {
      "epoch": 0.24539877300613497,
      "grad_norm": 34.674086597448465,
      "learning_rate": 1.3915289256198347e-06,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.25390625,
      "logps/chosen": -442.0,
      "logps/rejected": -392.0,
      "loss": 0.3066,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.2734375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.15625,
      "step": 200
    },
    {
      "epoch": 0.24662576687116564,
      "grad_norm": 29.903922318940893,
      "learning_rate": 1.3909090909090909e-06,
      "logits/chosen": -0.03662109375,
      "logits/rejected": -0.259765625,
      "logps/chosen": -404.0,
      "logps/rejected": -330.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -5.9375,
      "step": 201
    },
    {
      "epoch": 0.24785276073619633,
      "grad_norm": 33.833661458600005,
      "learning_rate": 1.3902892561983472e-06,
      "logits/chosen": -0.02001953125,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -432.0,
      "logps/rejected": -340.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -5.875,
      "step": 202
    },
    {
      "epoch": 0.249079754601227,
      "grad_norm": 37.642944458655066,
      "learning_rate": 1.3896694214876034e-06,
      "logits/chosen": 0.0272216796875,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -392.0,
      "logps/rejected": -354.0,
      "loss": 0.482,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.96875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.0625,
      "step": 203
    },
    {
      "epoch": 0.25030674846625767,
      "grad_norm": 35.44282157848665,
      "learning_rate": 1.3890495867768595e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.1005859375,
      "logps/chosen": -356.0,
      "logps/rejected": -290.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.78125,
      "step": 204
    },
    {
      "epoch": 0.25153374233128833,
      "grad_norm": 38.48812661842802,
      "learning_rate": 1.3884297520661158e-06,
      "logits/chosen": 0.08984375,
      "logits/rejected": -0.04833984375,
      "logps/chosen": -390.0,
      "logps/rejected": -316.0,
      "loss": 0.4435,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.125,
      "step": 205
    },
    {
      "epoch": 0.252760736196319,
      "grad_norm": 26.640868179250116,
      "learning_rate": 1.387809917355372e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.3125,
      "logps/chosen": -434.0,
      "logps/rejected": -350.0,
      "loss": 0.2375,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 1.203125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.28125,
      "step": 206
    },
    {
      "epoch": 0.25398773006134967,
      "grad_norm": 37.72822731879787,
      "learning_rate": 1.3871900826446281e-06,
      "logits/chosen": -0.16015625,
      "logits/rejected": -0.353515625,
      "logps/chosen": -434.0,
      "logps/rejected": -376.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.21875,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.53125,
      "step": 207
    },
    {
      "epoch": 0.2552147239263804,
      "grad_norm": 39.99822195882057,
      "learning_rate": 1.3865702479338843e-06,
      "logits/chosen": -0.12255859375,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -396.0,
      "logps/rejected": -340.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -5.6875,
      "step": 208
    },
    {
      "epoch": 0.25644171779141106,
      "grad_norm": 31.995520167951547,
      "learning_rate": 1.3859504132231404e-06,
      "logits/chosen": -0.125,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -398.0,
      "logps/rejected": -352.0,
      "loss": 0.3203,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.5625,
      "step": 209
    },
    {
      "epoch": 0.25766871165644173,
      "grad_norm": 42.71588448177596,
      "learning_rate": 1.3853305785123968e-06,
      "logits/chosen": -0.0732421875,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -400.0,
      "logps/rejected": -382.0,
      "loss": 0.4413,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.314453125,
      "rewards/margins": 5.90625,
      "rewards/rejected": -5.59375,
      "step": 210
    },
    {
      "epoch": 0.2588957055214724,
      "grad_norm": 35.38452290000429,
      "learning_rate": 1.384710743801653e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.375,
      "logps/chosen": -438.0,
      "logps/rejected": -362.0,
      "loss": 0.3649,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.90625,
      "step": 211
    },
    {
      "epoch": 0.26012269938650306,
      "grad_norm": 35.00437697538439,
      "learning_rate": 1.384090909090909e-06,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.251953125,
      "logps/chosen": -404.0,
      "logps/rejected": -328.0,
      "loss": 0.3606,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.5625,
      "step": 212
    },
    {
      "epoch": 0.26134969325153373,
      "grad_norm": 40.33706711227177,
      "learning_rate": 1.3834710743801654e-06,
      "logits/chosen": -0.031982421875,
      "logits/rejected": -0.0888671875,
      "logps/chosen": -352.0,
      "logps/rejected": -296.0,
      "loss": 0.3034,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.4765625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -5.3125,
      "step": 213
    },
    {
      "epoch": 0.2625766871165644,
      "grad_norm": 36.278922089821435,
      "learning_rate": 1.3828512396694215e-06,
      "logits/chosen": -0.0152587890625,
      "logits/rejected": -0.171875,
      "logps/chosen": -376.0,
      "logps/rejected": -326.0,
      "loss": 0.3538,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.25,
      "step": 214
    },
    {
      "epoch": 0.26380368098159507,
      "grad_norm": 39.61373791294869,
      "learning_rate": 1.3822314049586777e-06,
      "logits/chosen": 0.0224609375,
      "logits/rejected": -0.06689453125,
      "logps/chosen": -356.0,
      "logps/rejected": -316.0,
      "loss": 0.3393,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.55859375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -5.0,
      "step": 215
    },
    {
      "epoch": 0.2650306748466258,
      "grad_norm": 36.52111248304287,
      "learning_rate": 1.381611570247934e-06,
      "logits/chosen": -0.04443359375,
      "logits/rejected": -0.1572265625,
      "logps/chosen": -334.0,
      "logps/rejected": -302.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.42578125,
      "rewards/margins": 5.375,
      "rewards/rejected": -4.96875,
      "step": 216
    },
    {
      "epoch": 0.26625766871165646,
      "grad_norm": 35.893159477690816,
      "learning_rate": 1.38099173553719e-06,
      "logits/chosen": 0.05517578125,
      "logits/rejected": -0.08154296875,
      "logps/chosen": -402.0,
      "logps/rejected": -316.0,
      "loss": 0.4174,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -4.78125,
      "step": 217
    },
    {
      "epoch": 0.2674846625766871,
      "grad_norm": 43.28791050351139,
      "learning_rate": 1.3803719008264463e-06,
      "logits/chosen": 0.0108642578125,
      "logits/rejected": -0.0908203125,
      "logps/chosen": -384.0,
      "logps/rejected": -344.0,
      "loss": 0.514,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.79296875,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.8125,
      "step": 218
    },
    {
      "epoch": 0.2687116564417178,
      "grad_norm": 30.96899305966375,
      "learning_rate": 1.3797520661157025e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.189453125,
      "logps/chosen": -354.0,
      "logps/rejected": -310.0,
      "loss": 0.3339,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.28125,
      "step": 219
    },
    {
      "epoch": 0.26993865030674846,
      "grad_norm": 49.60415252304962,
      "learning_rate": 1.3791322314049586e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": 0.0047607421875,
      "logps/chosen": -366.0,
      "logps/rejected": -372.0,
      "loss": 0.4563,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.416015625,
      "rewards/margins": 5.0625,
      "rewards/rejected": -4.65625,
      "step": 220
    },
    {
      "epoch": 0.27116564417177913,
      "grad_norm": 35.817382292638634,
      "learning_rate": 1.378512396694215e-06,
      "logits/chosen": -0.049560546875,
      "logits/rejected": -0.25390625,
      "logps/chosen": -364.0,
      "logps/rejected": -306.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 5.75,
      "rewards/rejected": -5.28125,
      "step": 221
    },
    {
      "epoch": 0.2723926380368098,
      "grad_norm": 33.842292905676736,
      "learning_rate": 1.377892561983471e-06,
      "logits/chosen": -0.240234375,
      "logits/rejected": -0.3671875,
      "logps/chosen": -388.0,
      "logps/rejected": -318.0,
      "loss": 0.3321,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.9375,
      "step": 222
    },
    {
      "epoch": 0.27361963190184047,
      "grad_norm": 34.24344157812324,
      "learning_rate": 1.3772727272727274e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -390.0,
      "logps/rejected": -338.0,
      "loss": 0.3389,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.58984375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.59375,
      "step": 223
    },
    {
      "epoch": 0.2748466257668712,
      "grad_norm": 36.32910023908209,
      "learning_rate": 1.3766528925619836e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.33203125,
      "logps/chosen": -452.0,
      "logps/rejected": -352.0,
      "loss": 0.3172,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.28125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -5.90625,
      "step": 224
    },
    {
      "epoch": 0.27607361963190186,
      "grad_norm": 33.96451254717418,
      "learning_rate": 1.3760330578512397e-06,
      "logits/chosen": -0.021484375,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -344.0,
      "logps/rejected": -322.0,
      "loss": 0.2991,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4609375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.0625,
      "step": 225
    },
    {
      "epoch": 0.2773006134969325,
      "grad_norm": 37.753965128366424,
      "learning_rate": 1.3754132231404959e-06,
      "logits/chosen": -0.035888671875,
      "logits/rejected": -0.1689453125,
      "logps/chosen": -378.0,
      "logps/rejected": -328.0,
      "loss": 0.4435,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.625,
      "step": 226
    },
    {
      "epoch": 0.2785276073619632,
      "grad_norm": 37.97162675437385,
      "learning_rate": 1.374793388429752e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -362.0,
      "logps/rejected": -332.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -6.0,
      "step": 227
    },
    {
      "epoch": 0.27975460122699386,
      "grad_norm": 42.90390749582614,
      "learning_rate": 1.3741735537190084e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -366.0,
      "logps/rejected": -338.0,
      "loss": 0.3568,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.46875,
      "step": 228
    },
    {
      "epoch": 0.2809815950920245,
      "grad_norm": 35.198650610347336,
      "learning_rate": 1.3735537190082645e-06,
      "logits/chosen": 0.0174560546875,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -422.0,
      "logps/rejected": -344.0,
      "loss": 0.3812,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.640625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.375,
      "step": 229
    },
    {
      "epoch": 0.2822085889570552,
      "grad_norm": 31.76904887089288,
      "learning_rate": 1.3729338842975206e-06,
      "logits/chosen": 0.0027618408203125,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.2964,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.375,
      "step": 230
    },
    {
      "epoch": 0.28343558282208586,
      "grad_norm": 38.2475164063325,
      "learning_rate": 1.372314049586777e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -370.0,
      "logps/rejected": -320.0,
      "loss": 0.4068,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.65625,
      "step": 231
    },
    {
      "epoch": 0.2846625766871166,
      "grad_norm": 34.785350120825576,
      "learning_rate": 1.3716942148760331e-06,
      "logits/chosen": -0.1474609375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -384.0,
      "logps/rejected": -342.0,
      "loss": 0.3319,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 5.5625,
      "rewards/rejected": -4.875,
      "step": 232
    },
    {
      "epoch": 0.28588957055214725,
      "grad_norm": 33.915962784771814,
      "learning_rate": 1.3710743801652893e-06,
      "logits/chosen": -0.04052734375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -372.0,
      "logps/rejected": -296.0,
      "loss": 0.3377,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -4.84375,
      "step": 233
    },
    {
      "epoch": 0.2871165644171779,
      "grad_norm": 36.944834789970336,
      "learning_rate": 1.3704545454545456e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -374.0,
      "logps/rejected": -318.0,
      "loss": 0.3474,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.203125,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.40625,
      "step": 234
    },
    {
      "epoch": 0.2883435582822086,
      "grad_norm": 36.36585753307406,
      "learning_rate": 1.3698347107438016e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.34765625,
      "logps/chosen": -362.0,
      "logps/rejected": -326.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -4.78125,
      "step": 235
    },
    {
      "epoch": 0.28957055214723926,
      "grad_norm": 39.75324625491103,
      "learning_rate": 1.369214876033058e-06,
      "logits/chosen": 0.058837890625,
      "logits/rejected": -0.1123046875,
      "logps/chosen": -396.0,
      "logps/rejected": -374.0,
      "loss": 0.4644,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 5.625,
      "rewards/rejected": -3.890625,
      "step": 236
    },
    {
      "epoch": 0.2907975460122699,
      "grad_norm": 33.93837750645907,
      "learning_rate": 1.368595041322314e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.248046875,
      "logps/chosen": -368.0,
      "logps/rejected": -314.0,
      "loss": 0.3597,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.5,
      "rewards/margins": 6.1875,
      "rewards/rejected": -4.6875,
      "step": 237
    },
    {
      "epoch": 0.2920245398773006,
      "grad_norm": 40.82126535762802,
      "learning_rate": 1.3679752066115702e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.5098,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.25,
      "step": 238
    },
    {
      "epoch": 0.29325153374233126,
      "grad_norm": 37.96435040796285,
      "learning_rate": 1.3673553719008266e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -382.0,
      "logps/rejected": -314.0,
      "loss": 0.371,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.375,
      "step": 239
    },
    {
      "epoch": 0.294478527607362,
      "grad_norm": 34.11140908926358,
      "learning_rate": 1.3667355371900827e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.28515625,
      "logps/chosen": -418.0,
      "logps/rejected": -364.0,
      "loss": 0.2283,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.890625,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.0,
      "step": 240
    },
    {
      "epoch": 0.29570552147239265,
      "grad_norm": 33.461868421672946,
      "learning_rate": 1.3661157024793388e-06,
      "logits/chosen": -0.076171875,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -366.0,
      "logps/rejected": -336.0,
      "loss": 0.2828,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.8359375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.59375,
      "step": 241
    },
    {
      "epoch": 0.2969325153374233,
      "grad_norm": 32.77289894542164,
      "learning_rate": 1.3654958677685952e-06,
      "logits/chosen": -0.10009765625,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -350.0,
      "logps/rejected": -324.0,
      "loss": 0.2975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.2578125,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.34375,
      "step": 242
    },
    {
      "epoch": 0.298159509202454,
      "grad_norm": 38.07813065801456,
      "learning_rate": 1.3648760330578511e-06,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.328125,
      "logps/chosen": -390.0,
      "logps/rejected": -364.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.328125,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.3125,
      "step": 243
    },
    {
      "epoch": 0.29938650306748466,
      "grad_norm": 39.871497711059654,
      "learning_rate": 1.3642561983471075e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.39453125,
      "logps/chosen": -386.0,
      "logps/rejected": -346.0,
      "loss": 0.5523,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.265625,
      "rewards/margins": 5.46875,
      "rewards/rejected": -4.1875,
      "step": 244
    },
    {
      "epoch": 0.3006134969325153,
      "grad_norm": 32.221505039188294,
      "learning_rate": 1.3636363636363636e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.2890625,
      "logps/chosen": -398.0,
      "logps/rejected": -354.0,
      "loss": 0.2967,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.53125,
      "step": 245
    },
    {
      "epoch": 0.301840490797546,
      "grad_norm": 34.1015902082894,
      "learning_rate": 1.3630165289256198e-06,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -374.0,
      "logps/rejected": -346.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.171875,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.3125,
      "step": 246
    },
    {
      "epoch": 0.3030674846625767,
      "grad_norm": 32.83309359401401,
      "learning_rate": 1.3623966942148761e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.337890625,
      "logps/chosen": -372.0,
      "logps/rejected": -308.0,
      "loss": 0.3534,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -4.78125,
      "step": 247
    },
    {
      "epoch": 0.3042944785276074,
      "grad_norm": 33.089465845458236,
      "learning_rate": 1.3617768595041322e-06,
      "logits/chosen": -0.076171875,
      "logits/rejected": -0.3203125,
      "logps/chosen": -388.0,
      "logps/rejected": -332.0,
      "loss": 0.2682,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.703125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.3125,
      "step": 248
    },
    {
      "epoch": 0.30552147239263805,
      "grad_norm": 39.03476607944447,
      "learning_rate": 1.3611570247933884e-06,
      "logits/chosen": -0.20703125,
      "logits/rejected": -0.396484375,
      "logps/chosen": -474.0,
      "logps/rejected": -368.0,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.640625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.15625,
      "step": 249
    },
    {
      "epoch": 0.3067484662576687,
      "grad_norm": 38.29235099515543,
      "learning_rate": 1.3605371900826447e-06,
      "logits/chosen": -0.30078125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -340.0,
      "logps/rejected": -324.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.34375,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.4375,
      "step": 250
    },
    {
      "epoch": 0.3079754601226994,
      "grad_norm": 37.34357400280288,
      "learning_rate": 1.3599173553719009e-06,
      "logits/chosen": 0.0289306640625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -376.0,
      "logps/rejected": -308.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.421875,
      "rewards/margins": 5.375,
      "rewards/rejected": -3.9375,
      "step": 251
    },
    {
      "epoch": 0.30920245398773005,
      "grad_norm": 37.841749969537275,
      "learning_rate": 1.359297520661157e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.29296875,
      "logps/chosen": -392.0,
      "logps/rejected": -310.0,
      "loss": 0.4506,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.890625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -3.875,
      "step": 252
    },
    {
      "epoch": 0.3104294478527607,
      "grad_norm": 25.74895782175542,
      "learning_rate": 1.3586776859504132e-06,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.392578125,
      "logps/chosen": -336.0,
      "logps/rejected": -298.0,
      "loss": 0.2694,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.46875,
      "rewards/margins": 6.0625,
      "rewards/rejected": -4.59375,
      "step": 253
    },
    {
      "epoch": 0.3116564417177914,
      "grad_norm": 36.56021325822385,
      "learning_rate": 1.3580578512396693e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.25,
      "logps/chosen": -352.0,
      "logps/rejected": -320.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.8671875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -4.3125,
      "step": 254
    },
    {
      "epoch": 0.3128834355828221,
      "grad_norm": 37.45936986772829,
      "learning_rate": 1.3574380165289257e-06,
      "logits/chosen": -0.07958984375,
      "logits/rejected": -0.228515625,
      "logps/chosen": -384.0,
      "logps/rejected": -312.0,
      "loss": 0.439,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.8515625,
      "rewards/margins": 5.71875,
      "rewards/rejected": -3.859375,
      "step": 255
    },
    {
      "epoch": 0.3141104294478528,
      "grad_norm": 33.113238785557265,
      "learning_rate": 1.3568181818181818e-06,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.328125,
      "logps/chosen": -348.0,
      "logps/rejected": -316.0,
      "loss": 0.3148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 7.0,
      "rewards/rejected": -4.6875,
      "step": 256
    },
    {
      "epoch": 0.31533742331288345,
      "grad_norm": 34.825667619250744,
      "learning_rate": 1.3561983471074382e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.2890625,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.3455,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.25,
      "rewards/margins": 6.5625,
      "rewards/rejected": -4.3125,
      "step": 257
    },
    {
      "epoch": 0.3165644171779141,
      "grad_norm": 35.08263037322888,
      "learning_rate": 1.3555785123966943e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.28125,
      "logps/chosen": -396.0,
      "logps/rejected": -342.0,
      "loss": 0.319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.75,
      "step": 258
    },
    {
      "epoch": 0.3177914110429448,
      "grad_norm": 33.547677568046986,
      "learning_rate": 1.3549586776859504e-06,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -344.0,
      "logps/rejected": -356.0,
      "loss": 0.3553,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.5,
      "rewards/rejected": -5.1875,
      "step": 259
    },
    {
      "epoch": 0.31901840490797545,
      "grad_norm": 35.38305104419636,
      "learning_rate": 1.3543388429752068e-06,
      "logits/chosen": -0.1904296875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -390.0,
      "logps/rejected": -326.0,
      "loss": 0.3657,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.15625,
      "step": 260
    },
    {
      "epoch": 0.3202453987730061,
      "grad_norm": 37.62925536561278,
      "learning_rate": 1.3537190082644627e-06,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.263671875,
      "logps/chosen": -380.0,
      "logps/rejected": -330.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.53125,
      "step": 261
    },
    {
      "epoch": 0.3214723926380368,
      "grad_norm": 45.47620412654006,
      "learning_rate": 1.353099173553719e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.3359375,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -5.0625,
      "step": 262
    },
    {
      "epoch": 0.3226993865030675,
      "grad_norm": 43.246204937096174,
      "learning_rate": 1.3524793388429752e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.4521,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0,
      "step": 263
    },
    {
      "epoch": 0.3239263803680982,
      "grad_norm": 37.571005698421835,
      "learning_rate": 1.3518595041322314e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.20703125,
      "logps/chosen": -354.0,
      "logps/rejected": -308.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.15625,
      "step": 264
    },
    {
      "epoch": 0.32515337423312884,
      "grad_norm": 45.351424976255885,
      "learning_rate": 1.3512396694214877e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.265625,
      "logps/chosen": -360.0,
      "logps/rejected": -288.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.875,
      "step": 265
    },
    {
      "epoch": 0.3263803680981595,
      "grad_norm": 44.31098246879602,
      "learning_rate": 1.3506198347107439e-06,
      "logits/chosen": -0.07861328125,
      "logits/rejected": -0.228515625,
      "logps/chosen": -326.0,
      "logps/rejected": -316.0,
      "loss": 0.5312,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.27734375,
      "rewards/margins": 5.25,
      "rewards/rejected": -5.53125,
      "step": 266
    },
    {
      "epoch": 0.3276073619631902,
      "grad_norm": 43.337206726154484,
      "learning_rate": 1.35e-06,
      "logits/chosen": -0.0966796875,
      "logits/rejected": -0.31640625,
      "logps/chosen": -406.0,
      "logps/rejected": -322.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.625,
      "step": 267
    },
    {
      "epoch": 0.32883435582822085,
      "grad_norm": 28.822383795146568,
      "learning_rate": 1.3493801652892563e-06,
      "logits/chosen": -0.25390625,
      "logits/rejected": -0.365234375,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.209,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.65625,
      "step": 268
    },
    {
      "epoch": 0.3300613496932515,
      "grad_norm": 30.291051549986957,
      "learning_rate": 1.3487603305785125e-06,
      "logits/chosen": -0.1240234375,
      "logits/rejected": -0.43359375,
      "logps/chosen": -412.0,
      "logps/rejected": -322.0,
      "loss": 0.237,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.3125,
      "step": 269
    },
    {
      "epoch": 0.3312883435582822,
      "grad_norm": 38.1594801225088,
      "learning_rate": 1.3481404958677686e-06,
      "logits/chosen": -0.09912109375,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -334.0,
      "logps/rejected": -308.0,
      "loss": 0.4766,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 5.625,
      "rewards/rejected": -5.15625,
      "step": 270
    },
    {
      "epoch": 0.3325153374233129,
      "grad_norm": 28.374821367974555,
      "learning_rate": 1.3475206611570248e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.240234375,
      "logps/chosen": -390.0,
      "logps/rejected": -344.0,
      "loss": 0.3004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3984375,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.09375,
      "step": 271
    },
    {
      "epoch": 0.3337423312883436,
      "grad_norm": 34.17489484053231,
      "learning_rate": 1.346900826446281e-06,
      "logits/chosen": -0.00823974609375,
      "logits/rejected": -0.171875,
      "logps/chosen": -366.0,
      "logps/rejected": -318.0,
      "loss": 0.2861,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.423828125,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.34375,
      "step": 272
    },
    {
      "epoch": 0.33496932515337424,
      "grad_norm": 41.21053534408477,
      "learning_rate": 1.3462809917355373e-06,
      "logits/chosen": 0.06640625,
      "logits/rejected": -0.007110595703125,
      "logps/chosen": -344.0,
      "logps/rejected": -314.0,
      "loss": 0.5165,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 0.166015625,
      "rewards/margins": 5.75,
      "rewards/rejected": -5.5625,
      "step": 273
    },
    {
      "epoch": 0.3361963190184049,
      "grad_norm": 34.571787806623156,
      "learning_rate": 1.3456611570247934e-06,
      "logits/chosen": 0.0419921875,
      "logits/rejected": -0.1025390625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.5625,
      "step": 274
    },
    {
      "epoch": 0.3374233128834356,
      "grad_norm": 31.979565052575825,
      "learning_rate": 1.3450413223140495e-06,
      "logits/chosen": -0.0302734375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -326.0,
      "logps/rejected": -326.0,
      "loss": 0.4175,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.412109375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.6875,
      "step": 275
    },
    {
      "epoch": 0.33865030674846625,
      "grad_norm": 28.189301031425032,
      "learning_rate": 1.344421487603306e-06,
      "logits/chosen": 0.0498046875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -390.0,
      "logps/rejected": -330.0,
      "loss": 0.3505,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.96875,
      "step": 276
    },
    {
      "epoch": 0.3398773006134969,
      "grad_norm": 29.004541701793432,
      "learning_rate": 1.343801652892562e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.337890625,
      "logps/chosen": -414.0,
      "logps/rejected": -344.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.375,
      "step": 277
    },
    {
      "epoch": 0.3411042944785276,
      "grad_norm": 40.163399091060114,
      "learning_rate": 1.3431818181818184e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -384.0,
      "logps/rejected": -356.0,
      "loss": 0.4008,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.75,
      "step": 278
    },
    {
      "epoch": 0.3423312883435583,
      "grad_norm": 21.8515947233153,
      "learning_rate": 1.3425619834710743e-06,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -392.0,
      "logps/rejected": -326.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.8125,
      "step": 279
    },
    {
      "epoch": 0.34355828220858897,
      "grad_norm": 34.753521492334585,
      "learning_rate": 1.3419421487603305e-06,
      "logits/chosen": -0.09228515625,
      "logits/rejected": -0.234375,
      "logps/chosen": -408.0,
      "logps/rejected": -372.0,
      "loss": 0.4097,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.15625,
      "step": 280
    },
    {
      "epoch": 0.34478527607361964,
      "grad_norm": 35.589195433139196,
      "learning_rate": 1.3413223140495868e-06,
      "logits/chosen": -0.00921630859375,
      "logits/rejected": -0.201171875,
      "logps/chosen": -382.0,
      "logps/rejected": -330.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0625,
      "step": 281
    },
    {
      "epoch": 0.3460122699386503,
      "grad_norm": 29.302596470256102,
      "learning_rate": 1.340702479338843e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -344.0,
      "logps/rejected": -324.0,
      "loss": 0.2481,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.09375,
      "step": 282
    },
    {
      "epoch": 0.347239263803681,
      "grad_norm": 33.7375126879184,
      "learning_rate": 1.3400826446280993e-06,
      "logits/chosen": -0.2265625,
      "logits/rejected": -0.3125,
      "logps/chosen": -354.0,
      "logps/rejected": -298.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.59765625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.5625,
      "step": 283
    },
    {
      "epoch": 0.34846625766871164,
      "grad_norm": 35.97047400633379,
      "learning_rate": 1.3394628099173555e-06,
      "logits/chosen": -0.034423828125,
      "logits/rejected": -0.166015625,
      "logps/chosen": -370.0,
      "logps/rejected": -320.0,
      "loss": 0.2954,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.859375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.84375,
      "step": 284
    },
    {
      "epoch": 0.3496932515337423,
      "grad_norm": 29.423070534885003,
      "learning_rate": 1.3388429752066116e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.322265625,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.2845,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.80078125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.875,
      "step": 285
    },
    {
      "epoch": 0.350920245398773,
      "grad_norm": 35.376791661357736,
      "learning_rate": 1.338223140495868e-06,
      "logits/chosen": -0.04736328125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -422.0,
      "logps/rejected": -356.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.0625,
      "step": 286
    },
    {
      "epoch": 0.3521472392638037,
      "grad_norm": 38.51049635159653,
      "learning_rate": 1.337603305785124e-06,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.283203125,
      "logps/chosen": -382.0,
      "logps/rejected": -330.0,
      "loss": 0.3715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.8125,
      "step": 287
    },
    {
      "epoch": 0.35337423312883437,
      "grad_norm": 31.395092155644413,
      "learning_rate": 1.3369834710743802e-06,
      "logits/chosen": 0.057373046875,
      "logits/rejected": -0.0888671875,
      "logps/chosen": -386.0,
      "logps/rejected": -336.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.298828125,
      "rewards/margins": 5.9375,
      "rewards/rejected": -5.625,
      "step": 288
    },
    {
      "epoch": 0.35460122699386504,
      "grad_norm": 25.699158986974325,
      "learning_rate": 1.3363636363636364e-06,
      "logits/chosen": -0.11083984375,
      "logits/rejected": -0.248046875,
      "logps/chosen": -404.0,
      "logps/rejected": -330.0,
      "loss": 0.1963,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -6.71875,
      "step": 289
    },
    {
      "epoch": 0.3558282208588957,
      "grad_norm": 26.172712857085628,
      "learning_rate": 1.3357438016528925e-06,
      "logits/chosen": -0.283203125,
      "logits/rejected": -0.41796875,
      "logps/chosen": -380.0,
      "logps/rejected": -350.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.0,
      "step": 290
    },
    {
      "epoch": 0.3570552147239264,
      "grad_norm": 35.706855965144605,
      "learning_rate": 1.3351239669421489e-06,
      "logits/chosen": -0.2275390625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -394.0,
      "logps/rejected": -340.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.5625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.53125,
      "step": 291
    },
    {
      "epoch": 0.35828220858895704,
      "grad_norm": 32.40631116792141,
      "learning_rate": 1.334504132231405e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.390625,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2821,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.40625,
      "step": 292
    },
    {
      "epoch": 0.3595092024539877,
      "grad_norm": 27.724555696310663,
      "learning_rate": 1.3338842975206611e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -382.0,
      "logps/rejected": -332.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.033203125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.71875,
      "step": 293
    },
    {
      "epoch": 0.36073619631901843,
      "grad_norm": 26.756162333522596,
      "learning_rate": 1.3332644628099175e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.4921875,
      "logps/chosen": -472.0,
      "logps/rejected": -368.0,
      "loss": 0.3094,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.0,
      "step": 294
    },
    {
      "epoch": 0.3619631901840491,
      "grad_norm": 27.599547481570436,
      "learning_rate": 1.3326446280991736e-06,
      "logits/chosen": -0.04345703125,
      "logits/rejected": -0.1884765625,
      "logps/chosen": -356.0,
      "logps/rejected": -332.0,
      "loss": 0.2495,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.2734375,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.59375,
      "step": 295
    },
    {
      "epoch": 0.36319018404907977,
      "grad_norm": 35.1081345543023,
      "learning_rate": 1.3320247933884298e-06,
      "logits/chosen": -0.007598876953125,
      "logits/rejected": -0.154296875,
      "logps/chosen": -420.0,
      "logps/rejected": -338.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.5,
      "step": 296
    },
    {
      "epoch": 0.36441717791411044,
      "grad_norm": 31.495501539039623,
      "learning_rate": 1.331404958677686e-06,
      "logits/chosen": -0.154296875,
      "logits/rejected": -0.30859375,
      "logps/chosen": -360.0,
      "logps/rejected": -322.0,
      "loss": 0.2553,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.79296875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.25,
      "step": 297
    },
    {
      "epoch": 0.3656441717791411,
      "grad_norm": 37.56929706779854,
      "learning_rate": 1.330785123966942e-06,
      "logits/chosen": -0.009521484375,
      "logits/rejected": -0.09716796875,
      "logps/chosen": -384.0,
      "logps/rejected": -336.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.498046875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.53125,
      "step": 298
    },
    {
      "epoch": 0.36687116564417177,
      "grad_norm": 36.46646666012725,
      "learning_rate": 1.3301652892561984e-06,
      "logits/chosen": -0.058837890625,
      "logits/rejected": -0.228515625,
      "logps/chosen": -400.0,
      "logps/rejected": -334.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1650390625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -5.9375,
      "step": 299
    },
    {
      "epoch": 0.36809815950920244,
      "grad_norm": 37.01196046043553,
      "learning_rate": 1.3295454545454546e-06,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.4644,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.482421875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.15625,
      "step": 300
    },
    {
      "epoch": 0.3693251533742331,
      "grad_norm": 32.17417313364543,
      "learning_rate": 1.3289256198347107e-06,
      "logits/chosen": -0.11865234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -460.0,
      "logps/rejected": -386.0,
      "loss": 0.3417,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.34375,
      "step": 301
    },
    {
      "epoch": 0.37055214723926383,
      "grad_norm": 25.798891470824255,
      "learning_rate": 1.328305785123967e-06,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -382.0,
      "logps/rejected": -316.0,
      "loss": 0.2452,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.40625,
      "step": 302
    },
    {
      "epoch": 0.3717791411042945,
      "grad_norm": 40.25697610802204,
      "learning_rate": 1.3276859504132232e-06,
      "logits/chosen": -0.1474609375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -388.0,
      "logps/rejected": -376.0,
      "loss": 0.4569,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.466796875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.65625,
      "step": 303
    },
    {
      "epoch": 0.37300613496932516,
      "grad_norm": 30.538056754282714,
      "learning_rate": 1.3270661157024793e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -396.0,
      "logps/rejected": -334.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.140625,
      "rewards/margins": 6.875,
      "rewards/rejected": -7.0,
      "step": 304
    },
    {
      "epoch": 0.37423312883435583,
      "grad_norm": 27.228440712710828,
      "learning_rate": 1.3264462809917357e-06,
      "logits/chosen": -0.109375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -416.0,
      "logps/rejected": -344.0,
      "loss": 0.2623,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.341796875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.375,
      "step": 305
    },
    {
      "epoch": 0.3754601226993865,
      "grad_norm": 34.02132219179046,
      "learning_rate": 1.3258264462809916e-06,
      "logits/chosen": -0.2177734375,
      "logits/rejected": -0.37890625,
      "logps/chosen": -430.0,
      "logps/rejected": -364.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.33203125,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.0625,
      "step": 306
    },
    {
      "epoch": 0.37668711656441717,
      "grad_norm": 34.60907155238812,
      "learning_rate": 1.325206611570248e-06,
      "logits/chosen": -0.06103515625,
      "logits/rejected": -0.19140625,
      "logps/chosen": -382.0,
      "logps/rejected": -344.0,
      "loss": 0.4791,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.18359375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -7.09375,
      "step": 307
    },
    {
      "epoch": 0.37791411042944784,
      "grad_norm": 34.523248358542865,
      "learning_rate": 1.3245867768595041e-06,
      "logits/chosen": -0.00148773193359375,
      "logits/rejected": -0.162109375,
      "logps/chosen": -398.0,
      "logps/rejected": -360.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.5625,
      "step": 308
    },
    {
      "epoch": 0.3791411042944785,
      "grad_norm": 29.962650665710807,
      "learning_rate": 1.3239669421487603e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": -0.31640625,
      "logps/chosen": -388.0,
      "logps/rejected": -354.0,
      "loss": 0.2543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.9375,
      "step": 309
    },
    {
      "epoch": 0.3803680981595092,
      "grad_norm": 33.20481159292772,
      "learning_rate": 1.3233471074380166e-06,
      "logits/chosen": 0.051513671875,
      "logits/rejected": -0.037353515625,
      "logps/chosen": -322.0,
      "logps/rejected": -312.0,
      "loss": 0.4174,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.875,
      "rewards/margins": 5.5,
      "rewards/rejected": -6.375,
      "step": 310
    },
    {
      "epoch": 0.3815950920245399,
      "grad_norm": 32.36453885140526,
      "learning_rate": 1.3227272727272727e-06,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -392.0,
      "logps/rejected": -348.0,
      "loss": 0.3525,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.486328125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.28125,
      "step": 311
    },
    {
      "epoch": 0.38282208588957056,
      "grad_norm": 39.08052846369179,
      "learning_rate": 1.322107438016529e-06,
      "logits/chosen": 0.0162353515625,
      "logits/rejected": -0.1142578125,
      "logps/chosen": -392.0,
      "logps/rejected": -378.0,
      "loss": 0.5608,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": 0.0712890625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.28125,
      "step": 312
    },
    {
      "epoch": 0.38404907975460123,
      "grad_norm": 35.15009453427041,
      "learning_rate": 1.3214876033057852e-06,
      "logits/chosen": -0.0235595703125,
      "logits/rejected": -0.298828125,
      "logps/chosen": -410.0,
      "logps/rejected": -372.0,
      "loss": 0.383,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -6.9375,
      "step": 313
    },
    {
      "epoch": 0.3852760736196319,
      "grad_norm": 28.038472021341548,
      "learning_rate": 1.3208677685950414e-06,
      "logits/chosen": -0.05517578125,
      "logits/rejected": -0.2421875,
      "logps/chosen": -458.0,
      "logps/rejected": -360.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 2.03125,
      "rewards/margins": 9.375,
      "rewards/rejected": -7.34375,
      "step": 314
    },
    {
      "epoch": 0.38650306748466257,
      "grad_norm": 31.702508477036716,
      "learning_rate": 1.3202479338842975e-06,
      "logits/chosen": 0.076171875,
      "logits/rejected": -0.11865234375,
      "logps/chosen": -350.0,
      "logps/rejected": -300.0,
      "loss": 0.3293,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.2431640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.59375,
      "step": 315
    },
    {
      "epoch": 0.38773006134969323,
      "grad_norm": 27.443273044054905,
      "learning_rate": 1.3196280991735537e-06,
      "logits/chosen": 0.05419921875,
      "logits/rejected": -0.08544921875,
      "logps/chosen": -360.0,
      "logps/rejected": -308.0,
      "loss": 0.3129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.8125,
      "step": 316
    },
    {
      "epoch": 0.3889570552147239,
      "grad_norm": 28.339536052474916,
      "learning_rate": 1.31900826446281e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -400.0,
      "logps/rejected": -368.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.15625,
      "step": 317
    },
    {
      "epoch": 0.3901840490797546,
      "grad_norm": 38.792002326692455,
      "learning_rate": 1.3183884297520662e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.296875,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4379,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.91796875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.125,
      "step": 318
    },
    {
      "epoch": 0.3914110429447853,
      "grad_norm": 17.992013049840406,
      "learning_rate": 1.3177685950413223e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -414.0,
      "logps/rejected": -320.0,
      "loss": 0.1471,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.65625,
      "step": 319
    },
    {
      "epoch": 0.39263803680981596,
      "grad_norm": 31.193757820598883,
      "learning_rate": 1.3171487603305787e-06,
      "logits/chosen": -0.056884765625,
      "logits/rejected": -0.166015625,
      "logps/chosen": -320.0,
      "logps/rejected": -286.0,
      "loss": 0.325,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.53125,
      "step": 320
    },
    {
      "epoch": 0.39386503067484663,
      "grad_norm": 39.71075166734815,
      "learning_rate": 1.3165289256198348e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.35546875,
      "logps/chosen": -410.0,
      "logps/rejected": -398.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.375,
      "step": 321
    },
    {
      "epoch": 0.3950920245398773,
      "grad_norm": 26.7824487712344,
      "learning_rate": 1.315909090909091e-06,
      "logits/chosen": -0.107421875,
      "logits/rejected": -0.2578125,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.203125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.65625,
      "step": 322
    },
    {
      "epoch": 0.39631901840490796,
      "grad_norm": 36.45438287117724,
      "learning_rate": 1.3152892561983473e-06,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.265625,
      "logps/chosen": -354.0,
      "logps/rejected": -334.0,
      "loss": 0.4474,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6875,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.46875,
      "step": 323
    },
    {
      "epoch": 0.39754601226993863,
      "grad_norm": 31.15880722522094,
      "learning_rate": 1.3146694214876032e-06,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -356.0,
      "logps/rejected": -328.0,
      "loss": 0.3486,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.8125,
      "step": 324
    },
    {
      "epoch": 0.3987730061349693,
      "grad_norm": 36.47850348671778,
      "learning_rate": 1.3140495867768596e-06,
      "logits/chosen": -0.0272216796875,
      "logits/rejected": -0.09326171875,
      "logps/chosen": -330.0,
      "logps/rejected": -304.0,
      "loss": 0.5542,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.33203125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.84375,
      "step": 325
    },
    {
      "epoch": 0.4,
      "grad_norm": 26.62936328977598,
      "learning_rate": 1.3134297520661157e-06,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.369140625,
      "logps/chosen": -398.0,
      "logps/rejected": -374.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 8.375,
      "rewards/rejected": -6.84375,
      "step": 326
    },
    {
      "epoch": 0.4012269938650307,
      "grad_norm": 39.76264342514329,
      "learning_rate": 1.3128099173553719e-06,
      "logits/chosen": -0.1015625,
      "logits/rejected": -0.271484375,
      "logps/chosen": -430.0,
      "logps/rejected": -352.0,
      "loss": 0.4557,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.921875,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.1875,
      "step": 327
    },
    {
      "epoch": 0.40245398773006136,
      "grad_norm": 33.62855666124103,
      "learning_rate": 1.3121900826446282e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -362.0,
      "logps/rejected": -288.0,
      "loss": 0.363,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.228515625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.9375,
      "step": 328
    },
    {
      "epoch": 0.403680981595092,
      "grad_norm": 36.1359556726495,
      "learning_rate": 1.3115702479338844e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -466.0,
      "logps/rejected": -386.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.078125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.25,
      "step": 329
    },
    {
      "epoch": 0.4049079754601227,
      "grad_norm": 27.601952208856858,
      "learning_rate": 1.3109504132231405e-06,
      "logits/chosen": -0.22265625,
      "logits/rejected": -0.423828125,
      "logps/chosen": -418.0,
      "logps/rejected": -352.0,
      "loss": 0.2524,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.25,
      "step": 330
    },
    {
      "epoch": 0.40613496932515336,
      "grad_norm": 26.526762211128467,
      "learning_rate": 1.3103305785123968e-06,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -408.0,
      "logps/rejected": -356.0,
      "loss": 0.2755,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.09375,
      "rewards/margins": 7.625,
      "rewards/rejected": -6.53125,
      "step": 331
    },
    {
      "epoch": 0.40736196319018403,
      "grad_norm": 24.05844405102659,
      "learning_rate": 1.3097107438016528e-06,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.37109375,
      "logps/chosen": -430.0,
      "logps/rejected": -332.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -6.84375,
      "step": 332
    },
    {
      "epoch": 0.4085889570552147,
      "grad_norm": 36.41468851723906,
      "learning_rate": 1.3090909090909091e-06,
      "logits/chosen": 0.017822265625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -372.0,
      "logps/rejected": -346.0,
      "loss": 0.4229,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.48046875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.90625,
      "step": 333
    },
    {
      "epoch": 0.4098159509202454,
      "grad_norm": 30.33603786358444,
      "learning_rate": 1.3084710743801653e-06,
      "logits/chosen": -0.1181640625,
      "logits/rejected": -0.349609375,
      "logps/chosen": -382.0,
      "logps/rejected": -366.0,
      "loss": 0.3109,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.458984375,
      "rewards/margins": 7.75,
      "rewards/rejected": -7.28125,
      "step": 334
    },
    {
      "epoch": 0.4110429447852761,
      "grad_norm": 33.01078113652063,
      "learning_rate": 1.3078512396694214e-06,
      "logits/chosen": -0.0213623046875,
      "logits/rejected": -0.28125,
      "logps/chosen": -432.0,
      "logps/rejected": -332.0,
      "loss": 0.2319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.34375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.71875,
      "step": 335
    },
    {
      "epoch": 0.41226993865030676,
      "grad_norm": 31.965240510538873,
      "learning_rate": 1.3072314049586778e-06,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.326171875,
      "logps/chosen": -402.0,
      "logps/rejected": -352.0,
      "loss": 0.3352,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.8125,
      "step": 336
    },
    {
      "epoch": 0.4134969325153374,
      "grad_norm": 36.98349242868308,
      "learning_rate": 1.306611570247934e-06,
      "logits/chosen": 0.0264892578125,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -358.0,
      "logps/rejected": -352.0,
      "loss": 0.4393,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.75,
      "step": 337
    },
    {
      "epoch": 0.4147239263803681,
      "grad_norm": 31.08894150655132,
      "learning_rate": 1.30599173553719e-06,
      "logits/chosen": -0.005157470703125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -392.0,
      "logps/rejected": -354.0,
      "loss": 0.3686,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.75,
      "step": 338
    },
    {
      "epoch": 0.41595092024539876,
      "grad_norm": 34.40525022311832,
      "learning_rate": 1.3053719008264464e-06,
      "logits/chosen": 0.0654296875,
      "logits/rejected": -0.06103515625,
      "logps/chosen": -378.0,
      "logps/rejected": -346.0,
      "loss": 0.3699,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.5625,
      "step": 339
    },
    {
      "epoch": 0.4171779141104294,
      "grad_norm": 35.90111385254676,
      "learning_rate": 1.3047520661157025e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -366.0,
      "logps/rejected": -350.0,
      "loss": 0.3448,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.058349609375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.1875,
      "step": 340
    },
    {
      "epoch": 0.41840490797546015,
      "grad_norm": 27.984626902771293,
      "learning_rate": 1.3041322314049587e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.3125,
      "logps/chosen": -398.0,
      "logps/rejected": -334.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.75,
      "step": 341
    },
    {
      "epoch": 0.4196319018404908,
      "grad_norm": 34.66561631382139,
      "learning_rate": 1.3035123966942148e-06,
      "logits/chosen": -0.021240234375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -340.0,
      "logps/rejected": -318.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.049072265625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.03125,
      "step": 342
    },
    {
      "epoch": 0.4208588957055215,
      "grad_norm": 34.35806991693024,
      "learning_rate": 1.302892561983471e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.1513671875,
      "logps/chosen": -386.0,
      "logps/rejected": -356.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.251953125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -5.8125,
      "step": 343
    },
    {
      "epoch": 0.42208588957055215,
      "grad_norm": 36.57762286698119,
      "learning_rate": 1.3022727272727273e-06,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -408.0,
      "logps/rejected": -340.0,
      "loss": 0.4341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.036376953125,
      "rewards/margins": 5.875,
      "rewards/rejected": -5.84375,
      "step": 344
    },
    {
      "epoch": 0.4233128834355828,
      "grad_norm": 32.698706365667604,
      "learning_rate": 1.3016528925619835e-06,
      "logits/chosen": -0.04931640625,
      "logits/rejected": -0.2265625,
      "logps/chosen": -414.0,
      "logps/rejected": -340.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2177734375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.21875,
      "step": 345
    },
    {
      "epoch": 0.4245398773006135,
      "grad_norm": 34.95248105852957,
      "learning_rate": 1.3010330578512398e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.23046875,
      "logps/chosen": -344.0,
      "logps/rejected": -306.0,
      "loss": 0.4481,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.19921875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -6.03125,
      "step": 346
    },
    {
      "epoch": 0.42576687116564416,
      "grad_norm": 27.628512071923325,
      "learning_rate": 1.300413223140496e-06,
      "logits/chosen": -0.06884765625,
      "logits/rejected": -0.1875,
      "logps/chosen": -370.0,
      "logps/rejected": -356.0,
      "loss": 0.317,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.20703125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.90625,
      "step": 347
    },
    {
      "epoch": 0.4269938650306748,
      "grad_norm": 24.80952384589789,
      "learning_rate": 1.299793388429752e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.318359375,
      "logps/chosen": -418.0,
      "logps/rejected": -328.0,
      "loss": 0.2563,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 7.4375,
      "rewards/rejected": -7.3125,
      "step": 348
    },
    {
      "epoch": 0.42822085889570555,
      "grad_norm": 38.377577445863466,
      "learning_rate": 1.2991735537190084e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -406.0,
      "logps/rejected": -322.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.046875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.84375,
      "step": 349
    },
    {
      "epoch": 0.4294478527607362,
      "grad_norm": 37.75213306544608,
      "learning_rate": 1.2985537190082644e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": -0.12890625,
      "logps/chosen": -412.0,
      "logps/rejected": -318.0,
      "loss": 0.3697,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.045654296875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.0,
      "step": 350
    },
    {
      "epoch": 0.4306748466257669,
      "grad_norm": 35.53586893463626,
      "learning_rate": 1.2979338842975207e-06,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -342.0,
      "logps/rejected": -326.0,
      "loss": 0.4086,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1337890625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.1875,
      "step": 351
    },
    {
      "epoch": 0.43190184049079755,
      "grad_norm": 30.59296136721169,
      "learning_rate": 1.2973140495867769e-06,
      "logits/chosen": -0.0230712890625,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -386.0,
      "logps/rejected": -334.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.0625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.28125,
      "step": 352
    },
    {
      "epoch": 0.4331288343558282,
      "grad_norm": 24.76896622328251,
      "learning_rate": 1.296694214876033e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.244140625,
      "logps/chosen": -360.0,
      "logps/rejected": -330.0,
      "loss": 0.2411,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.921875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.5625,
      "step": 353
    },
    {
      "epoch": 0.4343558282208589,
      "grad_norm": 31.01854459387497,
      "learning_rate": 1.2960743801652894e-06,
      "logits/chosen": 0.01043701171875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -444.0,
      "logps/rejected": -378.0,
      "loss": 0.3525,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -5.53125,
      "step": 354
    },
    {
      "epoch": 0.43558282208588955,
      "grad_norm": 32.456178590454805,
      "learning_rate": 1.2954545454545455e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.24609375,
      "logps/chosen": -348.0,
      "logps/rejected": -356.0,
      "loss": 0.4542,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.296875,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.0,
      "step": 355
    },
    {
      "epoch": 0.4368098159509202,
      "grad_norm": 31.595656958351967,
      "learning_rate": 1.2948347107438016e-06,
      "logits/chosen": 0.06396484375,
      "logits/rejected": -0.0859375,
      "logps/chosen": -396.0,
      "logps/rejected": -332.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -5.59375,
      "step": 356
    },
    {
      "epoch": 0.43803680981595094,
      "grad_norm": 28.72400215925028,
      "learning_rate": 1.294214876033058e-06,
      "logits/chosen": -0.23046875,
      "logits/rejected": -0.384765625,
      "logps/chosen": -402.0,
      "logps/rejected": -340.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.78125,
      "step": 357
    },
    {
      "epoch": 0.4392638036809816,
      "grad_norm": 29.5841283121426,
      "learning_rate": 1.2935950413223141e-06,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.328125,
      "logps/chosen": -446.0,
      "logps/rejected": -330.0,
      "loss": 0.3109,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 7.125,
      "rewards/rejected": -5.90625,
      "step": 358
    },
    {
      "epoch": 0.4404907975460123,
      "grad_norm": 33.37778751879116,
      "learning_rate": 1.2929752066115703e-06,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.154296875,
      "logps/chosen": -424.0,
      "logps/rejected": -334.0,
      "loss": 0.3631,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.8125,
      "step": 359
    },
    {
      "epoch": 0.44171779141104295,
      "grad_norm": 32.72250588219352,
      "learning_rate": 1.2923553719008264e-06,
      "logits/chosen": -0.02587890625,
      "logits/rejected": -0.1484375,
      "logps/chosen": -396.0,
      "logps/rejected": -358.0,
      "loss": 0.2778,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.33984375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.59375,
      "step": 360
    },
    {
      "epoch": 0.4429447852760736,
      "grad_norm": 29.99311979639803,
      "learning_rate": 1.2917355371900826e-06,
      "logits/chosen": 0.010498046875,
      "logits/rejected": -0.177734375,
      "logps/chosen": -438.0,
      "logps/rejected": -382.0,
      "loss": 0.3143,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.875,
      "step": 361
    },
    {
      "epoch": 0.4441717791411043,
      "grad_norm": 30.82537911112007,
      "learning_rate": 1.291115702479339e-06,
      "logits/chosen": -0.11083984375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -418.0,
      "logps/rejected": -344.0,
      "loss": 0.3606,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.62109375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.59375,
      "step": 362
    },
    {
      "epoch": 0.44539877300613495,
      "grad_norm": 26.541431319509446,
      "learning_rate": 1.290495867768595e-06,
      "logits/chosen": -0.018798828125,
      "logits/rejected": -0.171875,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.2712,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.77734375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.03125,
      "step": 363
    },
    {
      "epoch": 0.4466257668711656,
      "grad_norm": 36.761947246861894,
      "learning_rate": 1.2898760330578512e-06,
      "logits/chosen": 0.06591796875,
      "logits/rejected": -0.109375,
      "logps/chosen": -392.0,
      "logps/rejected": -314.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.765625,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.0,
      "step": 364
    },
    {
      "epoch": 0.44785276073619634,
      "grad_norm": 27.87437703363369,
      "learning_rate": 1.2892561983471076e-06,
      "logits/chosen": -0.0018463134765625,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -352.0,
      "logps/rejected": -314.0,
      "loss": 0.3086,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.53125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.125,
      "step": 365
    },
    {
      "epoch": 0.449079754601227,
      "grad_norm": 32.787747807253844,
      "learning_rate": 1.2886363636363637e-06,
      "logits/chosen": 0.0291748046875,
      "logits/rejected": -0.06787109375,
      "logps/chosen": -366.0,
      "logps/rejected": -332.0,
      "loss": 0.447,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.87890625,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.125,
      "step": 366
    },
    {
      "epoch": 0.4503067484662577,
      "grad_norm": 26.662012319210124,
      "learning_rate": 1.2880165289256198e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.326171875,
      "logps/chosen": -394.0,
      "logps/rejected": -336.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.71875,
      "step": 367
    },
    {
      "epoch": 0.45153374233128835,
      "grad_norm": 36.83996177643189,
      "learning_rate": 1.287396694214876e-06,
      "logits/chosen": 0.06787109375,
      "logits/rejected": 0.0234375,
      "logps/chosen": -368.0,
      "logps/rejected": -328.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.3125,
      "step": 368
    },
    {
      "epoch": 0.452760736196319,
      "grad_norm": 35.726856908761434,
      "learning_rate": 1.2867768595041321e-06,
      "logits/chosen": 0.05322265625,
      "logits/rejected": -0.06591796875,
      "logps/chosen": -344.0,
      "logps/rejected": -298.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.169921875,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.5625,
      "step": 369
    },
    {
      "epoch": 0.4539877300613497,
      "grad_norm": 31.108323501043053,
      "learning_rate": 1.2861570247933885e-06,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.328125,
      "logps/chosen": -402.0,
      "logps/rejected": -348.0,
      "loss": 0.369,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.90625,
      "step": 370
    },
    {
      "epoch": 0.45521472392638035,
      "grad_norm": 31.06340590217051,
      "learning_rate": 1.2855371900826446e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -460.0,
      "logps/rejected": -364.0,
      "loss": 0.3181,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.0,
      "rewards/rejected": -4.375,
      "step": 371
    },
    {
      "epoch": 0.456441717791411,
      "grad_norm": 28.189166139003635,
      "learning_rate": 1.2849173553719008e-06,
      "logits/chosen": -0.0301513671875,
      "logits/rejected": -0.1875,
      "logps/chosen": -410.0,
      "logps/rejected": -324.0,
      "loss": 0.2419,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.9375,
      "step": 372
    },
    {
      "epoch": 0.45766871165644174,
      "grad_norm": 33.21287873989905,
      "learning_rate": 1.2842975206611571e-06,
      "logits/chosen": -0.053466796875,
      "logits/rejected": -0.1591796875,
      "logps/chosen": -362.0,
      "logps/rejected": -322.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.58203125,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.125,
      "step": 373
    },
    {
      "epoch": 0.4588957055214724,
      "grad_norm": 26.151862774833457,
      "learning_rate": 1.2836776859504132e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -382.0,
      "logps/rejected": -316.0,
      "loss": 0.2534,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 2.0,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.78125,
      "step": 374
    },
    {
      "epoch": 0.4601226993865031,
      "grad_norm": 32.35488575008133,
      "learning_rate": 1.2830578512396696e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -392.0,
      "logps/rejected": -350.0,
      "loss": 0.2771,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.7109375,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.09375,
      "step": 375
    },
    {
      "epoch": 0.46134969325153374,
      "grad_norm": 34.001922340864844,
      "learning_rate": 1.2824380165289257e-06,
      "logits/chosen": -0.00946044921875,
      "logits/rejected": -0.15625,
      "logps/chosen": -376.0,
      "logps/rejected": -344.0,
      "loss": 0.3408,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.03125,
      "step": 376
    },
    {
      "epoch": 0.4625766871165644,
      "grad_norm": 35.73955039729585,
      "learning_rate": 1.2818181818181817e-06,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.150390625,
      "logps/chosen": -352.0,
      "logps/rejected": -318.0,
      "loss": 0.4247,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.0625,
      "step": 377
    },
    {
      "epoch": 0.4638036809815951,
      "grad_norm": 33.82458737973786,
      "learning_rate": 1.281198347107438e-06,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.376953125,
      "logps/chosen": -376.0,
      "logps/rejected": -332.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.0,
      "step": 378
    },
    {
      "epoch": 0.46503067484662575,
      "grad_norm": 31.86429872362659,
      "learning_rate": 1.2805785123966942e-06,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.359375,
      "logps/chosen": -408.0,
      "logps/rejected": -344.0,
      "loss": 0.3028,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.71875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -5.46875,
      "step": 379
    },
    {
      "epoch": 0.4662576687116564,
      "grad_norm": 31.446849732786838,
      "learning_rate": 1.2799586776859505e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -382.0,
      "logps/rejected": -326.0,
      "loss": 0.3068,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.15625,
      "step": 380
    },
    {
      "epoch": 0.46748466257668714,
      "grad_norm": 31.679856008810177,
      "learning_rate": 1.2793388429752067e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.374,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.046875,
      "rewards/margins": 7.0,
      "rewards/rejected": -5.9375,
      "step": 381
    },
    {
      "epoch": 0.4687116564417178,
      "grad_norm": 31.436122523167693,
      "learning_rate": 1.2787190082644628e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.3158,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.078125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.8125,
      "step": 382
    },
    {
      "epoch": 0.4699386503067485,
      "grad_norm": 35.065632595795535,
      "learning_rate": 1.2780991735537192e-06,
      "logits/chosen": -0.09912109375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -402.0,
      "logps/rejected": -378.0,
      "loss": 0.4456,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.6875,
      "step": 383
    },
    {
      "epoch": 0.47116564417177914,
      "grad_norm": 33.79101177267741,
      "learning_rate": 1.2774793388429753e-06,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.435546875,
      "logps/chosen": -398.0,
      "logps/rejected": -342.0,
      "loss": 0.2961,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.640625,
      "rewards/margins": 7.6875,
      "rewards/rejected": -7.03125,
      "step": 384
    },
    {
      "epoch": 0.4723926380368098,
      "grad_norm": 36.518171637968166,
      "learning_rate": 1.2768595041322314e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.283203125,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.0625,
      "step": 385
    },
    {
      "epoch": 0.4736196319018405,
      "grad_norm": 34.95682432591222,
      "learning_rate": 1.2762396694214876e-06,
      "logits/chosen": -0.043701171875,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -342.0,
      "logps/rejected": -310.0,
      "loss": 0.369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3203125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.40625,
      "step": 386
    },
    {
      "epoch": 0.47484662576687114,
      "grad_norm": 64.7195376952873,
      "learning_rate": 1.2756198347107437e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.271484375,
      "logps/chosen": -422.0,
      "logps/rejected": -350.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6171875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.75,
      "step": 387
    },
    {
      "epoch": 0.47607361963190187,
      "grad_norm": 42.357455920572896,
      "learning_rate": 1.275e-06,
      "logits/chosen": -0.08349609375,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -406.0,
      "logps/rejected": -326.0,
      "loss": 0.5163,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.04736328125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -5.59375,
      "step": 388
    },
    {
      "epoch": 0.47730061349693254,
      "grad_norm": 38.953026072404185,
      "learning_rate": 1.2743801652892562e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.2236328125,
      "logps/chosen": -412.0,
      "logps/rejected": -340.0,
      "loss": 0.4455,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.099609375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.1875,
      "step": 389
    },
    {
      "epoch": 0.4785276073619632,
      "grad_norm": 38.5573431700193,
      "learning_rate": 1.2737603305785124e-06,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.34375,
      "logps/chosen": -372.0,
      "logps/rejected": -344.0,
      "loss": 0.492,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.62890625,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.65625,
      "step": 390
    },
    {
      "epoch": 0.47975460122699387,
      "grad_norm": 37.845045364551986,
      "learning_rate": 1.2731404958677687e-06,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -390.0,
      "logps/rejected": -344.0,
      "loss": 0.4807,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.181640625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.625,
      "step": 391
    },
    {
      "epoch": 0.48098159509202454,
      "grad_norm": 29.781954844950388,
      "learning_rate": 1.2725206611570249e-06,
      "logits/chosen": -0.060791015625,
      "logits/rejected": -0.208984375,
      "logps/chosen": -362.0,
      "logps/rejected": -330.0,
      "loss": 0.3103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -6.65625,
      "step": 392
    },
    {
      "epoch": 0.4822085889570552,
      "grad_norm": 33.5926238632541,
      "learning_rate": 1.271900826446281e-06,
      "logits/chosen": -0.0286865234375,
      "logits/rejected": -0.166015625,
      "logps/chosen": -416.0,
      "logps/rejected": -374.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.263671875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.6875,
      "step": 393
    },
    {
      "epoch": 0.4834355828220859,
      "grad_norm": 32.087849602750985,
      "learning_rate": 1.2712809917355373e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -398.0,
      "logps/rejected": -342.0,
      "loss": 0.262,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.166015625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.375,
      "step": 394
    },
    {
      "epoch": 0.48466257668711654,
      "grad_norm": 34.842476262528464,
      "learning_rate": 1.2706611570247933e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -448.0,
      "logps/rejected": -356.0,
      "loss": 0.3764,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.09375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.21875,
      "step": 395
    },
    {
      "epoch": 0.48588957055214727,
      "grad_norm": 22.42844686295793,
      "learning_rate": 1.2700413223140496e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.388671875,
      "logps/chosen": -348.0,
      "logps/rejected": -332.0,
      "loss": 0.2277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.54296875,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.875,
      "step": 396
    },
    {
      "epoch": 0.48711656441717793,
      "grad_norm": 33.97555781703141,
      "learning_rate": 1.2694214876033058e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.314453125,
      "logps/chosen": -388.0,
      "logps/rejected": -350.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.40625,
      "step": 397
    },
    {
      "epoch": 0.4883435582822086,
      "grad_norm": 39.08030474072834,
      "learning_rate": 1.268801652892562e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.27734375,
      "logps/chosen": -428.0,
      "logps/rejected": -346.0,
      "loss": 0.4427,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.08740234375,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.46875,
      "step": 398
    },
    {
      "epoch": 0.48957055214723927,
      "grad_norm": 33.83840388728663,
      "learning_rate": 1.2681818181818183e-06,
      "logits/chosen": -0.1611328125,
      "logits/rejected": -0.291015625,
      "logps/chosen": -394.0,
      "logps/rejected": -346.0,
      "loss": 0.2963,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.0035858154296875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.53125,
      "step": 399
    },
    {
      "epoch": 0.49079754601226994,
      "grad_norm": 41.49480537946823,
      "learning_rate": 1.2675619834710744e-06,
      "logits/chosen": -0.09130859375,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -384.0,
      "logps/rejected": -368.0,
      "loss": 0.5062,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 5.59375,
      "rewards/rejected": -6.15625,
      "step": 400
    },
    {
      "epoch": 0.4920245398773006,
      "grad_norm": 27.800803034272004,
      "learning_rate": 1.2669421487603308e-06,
      "logits/chosen": -0.23828125,
      "logits/rejected": -0.373046875,
      "logps/chosen": -456.0,
      "logps/rejected": -390.0,
      "loss": 0.2938,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.40625,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.1875,
      "step": 401
    },
    {
      "epoch": 0.49325153374233127,
      "grad_norm": 32.09882908595564,
      "learning_rate": 1.266322314049587e-06,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.1796875,
      "logps/chosen": -394.0,
      "logps/rejected": -350.0,
      "loss": 0.349,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.04443359375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.53125,
      "step": 402
    },
    {
      "epoch": 0.49447852760736194,
      "grad_norm": 29.930227321008882,
      "learning_rate": 1.265702479338843e-06,
      "logits/chosen": -0.07958984375,
      "logits/rejected": -0.15625,
      "logps/chosen": -396.0,
      "logps/rejected": -374.0,
      "loss": 0.341,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.033447265625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.8125,
      "step": 403
    },
    {
      "epoch": 0.49570552147239266,
      "grad_norm": 25.987833840695732,
      "learning_rate": 1.2650826446280992e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.2606,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.1669921875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.90625,
      "step": 404
    },
    {
      "epoch": 0.49693251533742333,
      "grad_norm": 38.14679307536266,
      "learning_rate": 1.2644628099173553e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.328125,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.03125,
      "step": 405
    },
    {
      "epoch": 0.498159509202454,
      "grad_norm": 37.82975021877776,
      "learning_rate": 1.2638429752066117e-06,
      "logits/chosen": 0.0166015625,
      "logits/rejected": -0.0625,
      "logps/chosen": -378.0,
      "logps/rejected": -340.0,
      "loss": 0.5427,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.75,
      "step": 406
    },
    {
      "epoch": 0.49938650306748467,
      "grad_norm": 38.332851373691135,
      "learning_rate": 1.2632231404958678e-06,
      "logits/chosen": 0.0115966796875,
      "logits/rejected": -0.134765625,
      "logps/chosen": -408.0,
      "logps/rejected": -344.0,
      "loss": 0.4382,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.34375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.0625,
      "step": 407
    },
    {
      "epoch": 0.5006134969325153,
      "grad_norm": 41.6208959235071,
      "learning_rate": 1.262603305785124e-06,
      "logits/chosen": -0.083984375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -426.0,
      "logps/rejected": -360.0,
      "loss": 0.5081,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.091796875,
      "rewards/margins": 5.1875,
      "rewards/rejected": -5.28125,
      "step": 408
    },
    {
      "epoch": 0.501840490797546,
      "grad_norm": 42.88666645295333,
      "learning_rate": 1.2619834710743803e-06,
      "logits/chosen": -0.00860595703125,
      "logits/rejected": -0.205078125,
      "logps/chosen": -426.0,
      "logps/rejected": -342.0,
      "loss": 0.4119,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -6.15625,
      "step": 409
    },
    {
      "epoch": 0.5030674846625767,
      "grad_norm": 36.428858153344706,
      "learning_rate": 1.2613636363636365e-06,
      "logits/chosen": -0.00095367431640625,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -388.0,
      "logps/rejected": -360.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.3828125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.75,
      "step": 410
    },
    {
      "epoch": 0.5042944785276073,
      "grad_norm": 33.21094894864319,
      "learning_rate": 1.2607438016528926e-06,
      "logits/chosen": 0.0140380859375,
      "logits/rejected": -0.12890625,
      "logps/chosen": -342.0,
      "logps/rejected": -320.0,
      "loss": 0.3058,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.1298828125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.34375,
      "step": 411
    },
    {
      "epoch": 0.505521472392638,
      "grad_norm": 42.15288062503197,
      "learning_rate": 1.260123966942149e-06,
      "logits/chosen": 0.007080078125,
      "logits/rejected": -0.1259765625,
      "logps/chosen": -428.0,
      "logps/rejected": -368.0,
      "loss": 0.4444,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.30078125,
      "rewards/margins": 5.96875,
      "rewards/rejected": -5.6875,
      "step": 412
    },
    {
      "epoch": 0.5067484662576687,
      "grad_norm": 37.795734491735246,
      "learning_rate": 1.2595041322314049e-06,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.26171875,
      "logps/chosen": -408.0,
      "logps/rejected": -332.0,
      "loss": 0.3878,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.0625,
      "step": 413
    },
    {
      "epoch": 0.5079754601226993,
      "grad_norm": 37.27154344231617,
      "learning_rate": 1.2588842975206612e-06,
      "logits/chosen": -0.0859375,
      "logits/rejected": -0.2265625,
      "logps/chosen": -406.0,
      "logps/rejected": -354.0,
      "loss": 0.3559,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.85546875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.84375,
      "step": 414
    },
    {
      "epoch": 0.50920245398773,
      "grad_norm": 30.678885373774655,
      "learning_rate": 1.2582644628099174e-06,
      "logits/chosen": -0.041748046875,
      "logits/rejected": -0.212890625,
      "logps/chosen": -350.0,
      "logps/rejected": -332.0,
      "loss": 0.3048,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.21875,
      "step": 415
    },
    {
      "epoch": 0.5104294478527608,
      "grad_norm": 26.882010464050307,
      "learning_rate": 1.2576446280991735e-06,
      "logits/chosen": -0.004730224609375,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -360.0,
      "logps/rejected": -324.0,
      "loss": 0.1899,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.458984375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.28125,
      "step": 416
    },
    {
      "epoch": 0.5116564417177915,
      "grad_norm": 30.354406082685717,
      "learning_rate": 1.2570247933884299e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.384765625,
      "logps/chosen": -438.0,
      "logps/rejected": -382.0,
      "loss": 0.3453,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.4765625,
      "rewards/margins": 8.125,
      "rewards/rejected": -6.65625,
      "step": 417
    },
    {
      "epoch": 0.5128834355828221,
      "grad_norm": 33.53763392215037,
      "learning_rate": 1.256404958677686e-06,
      "logits/chosen": -0.126953125,
      "logits/rejected": -0.2099609375,
      "logps/chosen": -368.0,
      "logps/rejected": -306.0,
      "loss": 0.3744,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.7890625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.375,
      "step": 418
    },
    {
      "epoch": 0.5141104294478528,
      "grad_norm": 28.10440726715766,
      "learning_rate": 1.2557851239669421e-06,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.21875,
      "logps/chosen": -380.0,
      "logps/rejected": -342.0,
      "loss": 0.3206,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.78125,
      "step": 419
    },
    {
      "epoch": 0.5153374233128835,
      "grad_norm": 30.627621824355007,
      "learning_rate": 1.2551652892561985e-06,
      "logits/chosen": 0.0262451171875,
      "logits/rejected": -0.134765625,
      "logps/chosen": -346.0,
      "logps/rejected": -312.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.796875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.65625,
      "step": 420
    },
    {
      "epoch": 0.5165644171779141,
      "grad_norm": 31.844076287237254,
      "learning_rate": 1.2545454545454544e-06,
      "logits/chosen": -0.01544189453125,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -364.0,
      "logps/rejected": -320.0,
      "loss": 0.4123,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.625,
      "step": 421
    },
    {
      "epoch": 0.5177914110429448,
      "grad_norm": 33.522822368907185,
      "learning_rate": 1.2539256198347108e-06,
      "logits/chosen": 0.0595703125,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -360.0,
      "logps/rejected": -302.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.15625,
      "step": 422
    },
    {
      "epoch": 0.5190184049079755,
      "grad_norm": 37.40841671544669,
      "learning_rate": 1.253305785123967e-06,
      "logits/chosen": -0.0595703125,
      "logits/rejected": -0.20703125,
      "logps/chosen": -398.0,
      "logps/rejected": -362.0,
      "loss": 0.4749,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.578125,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.4375,
      "step": 423
    },
    {
      "epoch": 0.5202453987730061,
      "grad_norm": 34.546118881756456,
      "learning_rate": 1.252685950413223e-06,
      "logits/chosen": 0.0235595703125,
      "logits/rejected": -0.166015625,
      "logps/chosen": -378.0,
      "logps/rejected": -342.0,
      "loss": 0.4503,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.875,
      "step": 424
    },
    {
      "epoch": 0.5214723926380368,
      "grad_norm": 37.84213132068162,
      "learning_rate": 1.2520661157024794e-06,
      "logits/chosen": -0.0123291015625,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -396.0,
      "logps/rejected": -348.0,
      "loss": 0.5144,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3203125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.53125,
      "step": 425
    },
    {
      "epoch": 0.5226993865030675,
      "grad_norm": 36.174280199739684,
      "learning_rate": 1.2514462809917356e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.166015625,
      "logps/chosen": -370.0,
      "logps/rejected": -348.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.015625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.0625,
      "step": 426
    },
    {
      "epoch": 0.5239263803680981,
      "grad_norm": 45.37106746020955,
      "learning_rate": 1.2508264462809917e-06,
      "logits/chosen": -0.01483154296875,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -378.0,
      "logps/rejected": -334.0,
      "loss": 0.4318,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.625,
      "step": 427
    },
    {
      "epoch": 0.5251533742331288,
      "grad_norm": 33.589023647082186,
      "learning_rate": 1.250206611570248e-06,
      "logits/chosen": -0.0419921875,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -364.0,
      "logps/rejected": -324.0,
      "loss": 0.4488,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.0,
      "step": 428
    },
    {
      "epoch": 0.5263803680981595,
      "grad_norm": 30.11424815064351,
      "learning_rate": 1.2495867768595042e-06,
      "logits/chosen": -0.044677734375,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -360.0,
      "logps/rejected": -376.0,
      "loss": 0.2639,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.373046875,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.59375,
      "step": 429
    },
    {
      "epoch": 0.5276073619631901,
      "grad_norm": 23.36575464672381,
      "learning_rate": 1.2489669421487603e-06,
      "logits/chosen": -0.11767578125,
      "logits/rejected": -0.28125,
      "logps/chosen": -394.0,
      "logps/rejected": -350.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.921875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.625,
      "step": 430
    },
    {
      "epoch": 0.5288343558282208,
      "grad_norm": 39.63938191210347,
      "learning_rate": 1.2483471074380165e-06,
      "logits/chosen": 0.01806640625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -386.0,
      "logps/rejected": -346.0,
      "loss": 0.4684,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.12255859375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.3125,
      "step": 431
    },
    {
      "epoch": 0.5300613496932516,
      "grad_norm": 35.90106246867024,
      "learning_rate": 1.2477272727272726e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": 0.035888671875,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.4356,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.78515625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.9375,
      "step": 432
    },
    {
      "epoch": 0.5312883435582823,
      "grad_norm": 43.12860297142818,
      "learning_rate": 1.247107438016529e-06,
      "logits/chosen": 0.08740234375,
      "logits/rejected": -0.1328125,
      "logps/chosen": -388.0,
      "logps/rejected": -320.0,
      "loss": 0.4803,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.12353515625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.5625,
      "step": 433
    },
    {
      "epoch": 0.5325153374233129,
      "grad_norm": 37.083326074229944,
      "learning_rate": 1.2464876033057851e-06,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -390.0,
      "logps/rejected": -356.0,
      "loss": 0.3206,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.07666015625,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.28125,
      "step": 434
    },
    {
      "epoch": 0.5337423312883436,
      "grad_norm": 37.88064967580844,
      "learning_rate": 1.2458677685950415e-06,
      "logits/chosen": -0.01458740234375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -394.0,
      "logps/rejected": -362.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.51953125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.65625,
      "step": 435
    },
    {
      "epoch": 0.5349693251533743,
      "grad_norm": 30.102357096787667,
      "learning_rate": 1.2452479338842976e-06,
      "logits/chosen": -0.0250244140625,
      "logits/rejected": -0.2255859375,
      "logps/chosen": -422.0,
      "logps/rejected": -358.0,
      "loss": 0.2562,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 7.75,
      "rewards/rejected": -6.625,
      "step": 436
    },
    {
      "epoch": 0.5361963190184049,
      "grad_norm": 25.03840429478704,
      "learning_rate": 1.2446280991735538e-06,
      "logits/chosen": -0.080078125,
      "logits/rejected": -0.25,
      "logps/chosen": -388.0,
      "logps/rejected": -350.0,
      "loss": 0.241,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.875,
      "step": 437
    },
    {
      "epoch": 0.5374233128834356,
      "grad_norm": 36.15225304248156,
      "learning_rate": 1.24400826446281e-06,
      "logits/chosen": 0.02001953125,
      "logits/rejected": -0.06982421875,
      "logps/chosen": -366.0,
      "logps/rejected": -334.0,
      "loss": 0.5276,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0250244140625,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.6875,
      "step": 438
    },
    {
      "epoch": 0.5386503067484663,
      "grad_norm": 34.58997013752512,
      "learning_rate": 1.243388429752066e-06,
      "logits/chosen": 0.0625,
      "logits/rejected": -0.126953125,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.3671,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.71875,
      "step": 439
    },
    {
      "epoch": 0.5398773006134969,
      "grad_norm": 43.86154514789445,
      "learning_rate": 1.2427685950413224e-06,
      "logits/chosen": 0.0218505859375,
      "logits/rejected": -0.0751953125,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.4825,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.515625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -5.96875,
      "step": 440
    },
    {
      "epoch": 0.5411042944785276,
      "grad_norm": 32.02976017157544,
      "learning_rate": 1.2421487603305785e-06,
      "logits/chosen": 0.09619140625,
      "logits/rejected": -0.0108642578125,
      "logps/chosen": -340.0,
      "logps/rejected": -314.0,
      "loss": 0.4408,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.0869140625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -6.21875,
      "step": 441
    },
    {
      "epoch": 0.5423312883435583,
      "grad_norm": 31.122024927447573,
      "learning_rate": 1.2415289256198347e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -390.0,
      "logps/rejected": -352.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.51171875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.0625,
      "step": 442
    },
    {
      "epoch": 0.5435582822085889,
      "grad_norm": 35.35206800809496,
      "learning_rate": 1.240909090909091e-06,
      "logits/chosen": -0.024169921875,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -372.0,
      "logps/rejected": -332.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.69140625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.125,
      "step": 443
    },
    {
      "epoch": 0.5447852760736196,
      "grad_norm": 35.692369145811284,
      "learning_rate": 1.2402892561983472e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.298828125,
      "logps/chosen": -454.0,
      "logps/rejected": -382.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 8.0,
      "rewards/rejected": -6.71875,
      "step": 444
    },
    {
      "epoch": 0.5460122699386503,
      "grad_norm": 32.62912831625137,
      "learning_rate": 1.2396694214876033e-06,
      "logits/chosen": -0.016357421875,
      "logits/rejected": -0.208984375,
      "logps/chosen": -358.0,
      "logps/rejected": -314.0,
      "loss": 0.4199,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.5625,
      "step": 445
    },
    {
      "epoch": 0.5472392638036809,
      "grad_norm": 25.78605234898617,
      "learning_rate": 1.2390495867768597e-06,
      "logits/chosen": -0.040283203125,
      "logits/rejected": -0.203125,
      "logps/chosen": -438.0,
      "logps/rejected": -370.0,
      "loss": 0.2432,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.65625,
      "step": 446
    },
    {
      "epoch": 0.5484662576687117,
      "grad_norm": 28.526549681854004,
      "learning_rate": 1.2384297520661158e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -414.0,
      "logps/rejected": -360.0,
      "loss": 0.3087,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.359375,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.84375,
      "step": 447
    },
    {
      "epoch": 0.5496932515337424,
      "grad_norm": 33.26832004918433,
      "learning_rate": 1.237809917355372e-06,
      "logits/chosen": 0.042724609375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -346.0,
      "logps/rejected": -314.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1796875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.84375,
      "step": 448
    },
    {
      "epoch": 0.550920245398773,
      "grad_norm": 29.824788916298026,
      "learning_rate": 1.237190082644628e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -400.0,
      "logps/rejected": -346.0,
      "loss": 0.3203,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.234375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.09375,
      "step": 449
    },
    {
      "epoch": 0.5521472392638037,
      "grad_norm": 29.646416826102993,
      "learning_rate": 1.2365702479338842e-06,
      "logits/chosen": -0.1103515625,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -350.0,
      "logps/rejected": -302.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.125,
      "step": 450
    },
    {
      "epoch": 0.5533742331288344,
      "grad_norm": 31.583737211583365,
      "learning_rate": 1.2359504132231406e-06,
      "logits/chosen": 0.03173828125,
      "logits/rejected": -0.1396484375,
      "logps/chosen": -348.0,
      "logps/rejected": -294.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.875,
      "step": 451
    },
    {
      "epoch": 0.554601226993865,
      "grad_norm": 27.43380959875228,
      "learning_rate": 1.2353305785123967e-06,
      "logits/chosen": -0.0439453125,
      "logits/rejected": -0.28125,
      "logps/chosen": -460.0,
      "logps/rejected": -342.0,
      "loss": 0.2556,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.9453125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -5.59375,
      "step": 452
    },
    {
      "epoch": 0.5558282208588957,
      "grad_norm": 24.613642812642805,
      "learning_rate": 1.2347107438016529e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -440.0,
      "logps/rejected": -378.0,
      "loss": 0.2702,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.0625,
      "step": 453
    },
    {
      "epoch": 0.5570552147239264,
      "grad_norm": 31.985648649098213,
      "learning_rate": 1.2340909090909092e-06,
      "logits/chosen": 0.046875,
      "logits/rejected": -0.0206298828125,
      "logps/chosen": -346.0,
      "logps/rejected": -320.0,
      "loss": 0.3305,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0859375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -6.0,
      "step": 454
    },
    {
      "epoch": 0.558282208588957,
      "grad_norm": 26.277923249806037,
      "learning_rate": 1.2334710743801654e-06,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.201171875,
      "logps/chosen": -372.0,
      "logps/rejected": -362.0,
      "loss": 0.2299,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.5,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.09375,
      "step": 455
    },
    {
      "epoch": 0.5595092024539877,
      "grad_norm": 35.777155080443045,
      "learning_rate": 1.2328512396694215e-06,
      "logits/chosen": 0.0419921875,
      "logits/rejected": -0.0576171875,
      "logps/chosen": -318.0,
      "logps/rejected": -314.0,
      "loss": 0.4054,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.412109375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.375,
      "step": 456
    },
    {
      "epoch": 0.5607361963190184,
      "grad_norm": 33.41411882209408,
      "learning_rate": 1.2322314049586776e-06,
      "logits/chosen": 0.0126953125,
      "logits/rejected": -0.1142578125,
      "logps/chosen": -360.0,
      "logps/rejected": -330.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.59375,
      "step": 457
    },
    {
      "epoch": 0.561963190184049,
      "grad_norm": 27.510163645115238,
      "learning_rate": 1.2316115702479338e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -376.0,
      "logps/rejected": -302.0,
      "loss": 0.2733,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.3125,
      "step": 458
    },
    {
      "epoch": 0.5631901840490797,
      "grad_norm": 32.20829218573613,
      "learning_rate": 1.2309917355371901e-06,
      "logits/chosen": -0.0172119140625,
      "logits/rejected": -0.1796875,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.3698,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.171875,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.1875,
      "step": 459
    },
    {
      "epoch": 0.5644171779141104,
      "grad_norm": 28.463767397560563,
      "learning_rate": 1.2303719008264463e-06,
      "logits/chosen": -0.0159912109375,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -354.0,
      "logps/rejected": -342.0,
      "loss": 0.2835,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 7.46875,
      "rewards/rejected": -7.3125,
      "step": 460
    },
    {
      "epoch": 0.5656441717791411,
      "grad_norm": 30.885477551125014,
      "learning_rate": 1.2297520661157024e-06,
      "logits/chosen": -0.2158203125,
      "logits/rejected": -0.330078125,
      "logps/chosen": -380.0,
      "logps/rejected": -352.0,
      "loss": 0.2845,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.1884765625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.375,
      "step": 461
    },
    {
      "epoch": 0.5668711656441717,
      "grad_norm": 44.21195903099332,
      "learning_rate": 1.2291322314049588e-06,
      "logits/chosen": 0.047607421875,
      "logits/rejected": -0.1552734375,
      "logps/chosen": -370.0,
      "logps/rejected": -310.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.1337890625,
      "rewards/margins": 5.875,
      "rewards/rejected": -6.03125,
      "step": 462
    },
    {
      "epoch": 0.5680981595092025,
      "grad_norm": 33.208183287178784,
      "learning_rate": 1.228512396694215e-06,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.18359375,
      "logps/chosen": -370.0,
      "logps/rejected": -372.0,
      "loss": 0.3252,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.4140625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.6875,
      "step": 463
    },
    {
      "epoch": 0.5693251533742332,
      "grad_norm": 37.13310236043479,
      "learning_rate": 1.2278925619834713e-06,
      "logits/chosen": 0.05224609375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -398.0,
      "logps/rejected": -334.0,
      "loss": 0.3492,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.3359375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.59375,
      "step": 464
    },
    {
      "epoch": 0.5705521472392638,
      "grad_norm": 21.80723319666503,
      "learning_rate": 1.2272727272727274e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.302734375,
      "logps/chosen": -404.0,
      "logps/rejected": -362.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.6171875,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.625,
      "step": 465
    },
    {
      "epoch": 0.5717791411042945,
      "grad_norm": 32.116293843093004,
      "learning_rate": 1.2266528925619833e-06,
      "logits/chosen": -0.016845703125,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -424.0,
      "logps/rejected": -356.0,
      "loss": 0.3118,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.0458984375,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.46875,
      "step": 466
    },
    {
      "epoch": 0.5730061349693252,
      "grad_norm": 37.63110345633159,
      "learning_rate": 1.2260330578512397e-06,
      "logits/chosen": -0.0279541015625,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -388.0,
      "logps/rejected": -364.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.8046875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.40625,
      "step": 467
    },
    {
      "epoch": 0.5742331288343558,
      "grad_norm": 33.08460200253717,
      "learning_rate": 1.2254132231404958e-06,
      "logits/chosen": -0.0419921875,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -372.0,
      "logps/rejected": -358.0,
      "loss": 0.3099,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.09375,
      "step": 468
    },
    {
      "epoch": 0.5754601226993865,
      "grad_norm": 22.648898471514507,
      "learning_rate": 1.2247933884297522e-06,
      "logits/chosen": 0.0230712890625,
      "logits/rejected": -0.15234375,
      "logps/chosen": -390.0,
      "logps/rejected": -320.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.796875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.0625,
      "step": 469
    },
    {
      "epoch": 0.5766871165644172,
      "grad_norm": 33.108021332496286,
      "learning_rate": 1.2241735537190083e-06,
      "logits/chosen": -0.010498046875,
      "logits/rejected": -0.07373046875,
      "logps/chosen": -356.0,
      "logps/rejected": -338.0,
      "loss": 0.367,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.8125,
      "step": 470
    },
    {
      "epoch": 0.5779141104294478,
      "grad_norm": 30.281427499200404,
      "learning_rate": 1.2235537190082645e-06,
      "logits/chosen": 0.1171875,
      "logits/rejected": -0.0458984375,
      "logps/chosen": -408.0,
      "logps/rejected": -350.0,
      "loss": 0.3469,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.328125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.5625,
      "step": 471
    },
    {
      "epoch": 0.5791411042944785,
      "grad_norm": 30.766178072522326,
      "learning_rate": 1.2229338842975208e-06,
      "logits/chosen": 0.0654296875,
      "logits/rejected": -0.15234375,
      "logps/chosen": -414.0,
      "logps/rejected": -336.0,
      "loss": 0.341,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.71875,
      "step": 472
    },
    {
      "epoch": 0.5803680981595092,
      "grad_norm": 35.82872232913366,
      "learning_rate": 1.222314049586777e-06,
      "logits/chosen": -0.10986328125,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -406.0,
      "logps/rejected": -394.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.5,
      "step": 473
    },
    {
      "epoch": 0.5815950920245399,
      "grad_norm": 38.235476934077774,
      "learning_rate": 1.221694214876033e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.271484375,
      "logps/chosen": -440.0,
      "logps/rejected": -362.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0673828125,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.8125,
      "step": 474
    },
    {
      "epoch": 0.5828220858895705,
      "grad_norm": 28.310340512331347,
      "learning_rate": 1.2210743801652892e-06,
      "logits/chosen": -0.06689453125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -434.0,
      "logps/rejected": -362.0,
      "loss": 0.2596,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.140625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.3125,
      "step": 475
    },
    {
      "epoch": 0.5840490797546012,
      "grad_norm": 31.21556050331037,
      "learning_rate": 1.2204545454545454e-06,
      "logits/chosen": 0.00927734375,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -420.0,
      "logps/rejected": -360.0,
      "loss": 0.3346,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -1.078125,
      "rewards/margins": 6.53125,
      "rewards/rejected": -7.625,
      "step": 476
    },
    {
      "epoch": 0.5852760736196319,
      "grad_norm": 31.773096981894017,
      "learning_rate": 1.2198347107438017e-06,
      "logits/chosen": -0.11474609375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -432.0,
      "logps/rejected": -372.0,
      "loss": 0.3232,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.1484375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.65625,
      "step": 477
    },
    {
      "epoch": 0.5865030674846625,
      "grad_norm": 30.507836762025494,
      "learning_rate": 1.2192148760330579e-06,
      "logits/chosen": -0.04150390625,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -430.0,
      "logps/rejected": -368.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.024658203125,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.65625,
      "step": 478
    },
    {
      "epoch": 0.5877300613496933,
      "grad_norm": 32.261887607989564,
      "learning_rate": 1.218595041322314e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -370.0,
      "logps/rejected": -352.0,
      "loss": 0.322,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 6.625,
      "rewards/rejected": -7.78125,
      "step": 479
    },
    {
      "epoch": 0.588957055214724,
      "grad_norm": 28.188949742159306,
      "learning_rate": 1.2179752066115704e-06,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.208984375,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.2558,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.8125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -8.0,
      "step": 480
    },
    {
      "epoch": 0.5901840490797546,
      "grad_norm": 37.17733184183342,
      "learning_rate": 1.2173553719008265e-06,
      "logits/chosen": -0.07568359375,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -438.0,
      "logps/rejected": -398.0,
      "loss": 0.4973,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.578125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.3125,
      "step": 481
    },
    {
      "epoch": 0.5914110429447853,
      "grad_norm": 34.97683920834337,
      "learning_rate": 1.2167355371900826e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.244140625,
      "logps/chosen": -424.0,
      "logps/rejected": -352.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 6.875,
      "rewards/rejected": -7.125,
      "step": 482
    },
    {
      "epoch": 0.592638036809816,
      "grad_norm": 32.698615963886375,
      "learning_rate": 1.216115702479339e-06,
      "logits/chosen": 0.07763671875,
      "logits/rejected": -0.0556640625,
      "logps/chosen": -358.0,
      "logps/rejected": -330.0,
      "loss": 0.3304,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.55859375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.34375,
      "step": 483
    },
    {
      "epoch": 0.5938650306748466,
      "grad_norm": 31.73235234166197,
      "learning_rate": 1.215495867768595e-06,
      "logits/chosen": 0.00390625,
      "logits/rejected": -0.142578125,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.1611328125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.3125,
      "step": 484
    },
    {
      "epoch": 0.5950920245398773,
      "grad_norm": 28.55956265398308,
      "learning_rate": 1.2148760330578513e-06,
      "logits/chosen": 0.023681640625,
      "logits/rejected": -0.12109375,
      "logps/chosen": -370.0,
      "logps/rejected": -324.0,
      "loss": 0.3173,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.36328125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.3125,
      "step": 485
    },
    {
      "epoch": 0.596319018404908,
      "grad_norm": 33.81986931883939,
      "learning_rate": 1.2142561983471074e-06,
      "logits/chosen": -0.0050048828125,
      "logits/rejected": -0.14453125,
      "logps/chosen": -414.0,
      "logps/rejected": -352.0,
      "loss": 0.3778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.455078125,
      "rewards/margins": 7.0,
      "rewards/rejected": -6.53125,
      "step": 486
    },
    {
      "epoch": 0.5975460122699386,
      "grad_norm": 30.028596310368773,
      "learning_rate": 1.2136363636363636e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.27734375,
      "logps/chosen": -410.0,
      "logps/rejected": -366.0,
      "loss": 0.2781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.03125,
      "step": 487
    },
    {
      "epoch": 0.5987730061349693,
      "grad_norm": 35.71061050092003,
      "learning_rate": 1.21301652892562e-06,
      "logits/chosen": -0.0693359375,
      "logits/rejected": -0.138671875,
      "logps/chosen": -424.0,
      "logps/rejected": -370.0,
      "loss": 0.3544,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5078125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.46875,
      "step": 488
    },
    {
      "epoch": 0.6,
      "grad_norm": 26.138379124419743,
      "learning_rate": 1.212396694214876e-06,
      "logits/chosen": 0.01324462890625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -384.0,
      "logps/rejected": -340.0,
      "loss": 0.3008,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 7.875,
      "rewards/rejected": -6.9375,
      "step": 489
    },
    {
      "epoch": 0.6012269938650306,
      "grad_norm": 38.68817736099095,
      "learning_rate": 1.2117768595041322e-06,
      "logits/chosen": 0.055419921875,
      "logits/rejected": -0.1689453125,
      "logps/chosen": -432.0,
      "logps/rejected": -364.0,
      "loss": 0.404,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.5625,
      "step": 490
    },
    {
      "epoch": 0.6024539877300613,
      "grad_norm": 30.239394074669093,
      "learning_rate": 1.2111570247933886e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.3068,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.53125,
      "step": 491
    },
    {
      "epoch": 0.603680981595092,
      "grad_norm": 33.80787533439265,
      "learning_rate": 1.2105371900826447e-06,
      "logits/chosen": -0.013427734375,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -398.0,
      "logps/rejected": -390.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.125,
      "step": 492
    },
    {
      "epoch": 0.6049079754601226,
      "grad_norm": 41.137878058973136,
      "learning_rate": 1.2099173553719008e-06,
      "logits/chosen": 0.14453125,
      "logits/rejected": 0.043701171875,
      "logps/chosen": -392.0,
      "logps/rejected": -340.0,
      "loss": 0.5049,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.19921875,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.4375,
      "step": 493
    },
    {
      "epoch": 0.6061349693251534,
      "grad_norm": 28.550910744803765,
      "learning_rate": 1.209297520661157e-06,
      "logits/chosen": 0.044921875,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -428.0,
      "logps/rejected": -340.0,
      "loss": 0.3041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.0625,
      "step": 494
    },
    {
      "epoch": 0.6073619631901841,
      "grad_norm": 32.18887165372733,
      "learning_rate": 1.2086776859504131e-06,
      "logits/chosen": 0.031494140625,
      "logits/rejected": -0.1064453125,
      "logps/chosen": -402.0,
      "logps/rejected": -350.0,
      "loss": 0.3067,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 7.25,
      "rewards/rejected": -6.15625,
      "step": 495
    },
    {
      "epoch": 0.6085889570552148,
      "grad_norm": 34.7968597037722,
      "learning_rate": 1.2080578512396695e-06,
      "logits/chosen": -0.0025634765625,
      "logits/rejected": -0.21484375,
      "logps/chosen": -382.0,
      "logps/rejected": -322.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1708984375,
      "rewards/margins": 6.25,
      "rewards/rejected": -6.09375,
      "step": 496
    },
    {
      "epoch": 0.6098159509202454,
      "grad_norm": 36.914820937247434,
      "learning_rate": 1.2074380165289256e-06,
      "logits/chosen": 0.0361328125,
      "logits/rejected": -0.125,
      "logps/chosen": -404.0,
      "logps/rejected": -366.0,
      "loss": 0.4718,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1337890625,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.375,
      "step": 497
    },
    {
      "epoch": 0.6110429447852761,
      "grad_norm": 27.898760612078714,
      "learning_rate": 1.206818181818182e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.265625,
      "logps/chosen": -400.0,
      "logps/rejected": -366.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.9765625,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.5,
      "step": 498
    },
    {
      "epoch": 0.6122699386503068,
      "grad_norm": 33.514362873417944,
      "learning_rate": 1.2061983471074381e-06,
      "logits/chosen": 0.041015625,
      "logits/rejected": -0.177734375,
      "logps/chosen": -426.0,
      "logps/rejected": -350.0,
      "loss": 0.3147,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.0,
      "step": 499
    },
    {
      "epoch": 0.6134969325153374,
      "grad_norm": 30.177881260758163,
      "learning_rate": 1.2055785123966943e-06,
      "logits/chosen": 0.0224609375,
      "logits/rejected": -0.09375,
      "logps/chosen": -418.0,
      "logps/rejected": -338.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.28125,
      "step": 500
    },
    {
      "epoch": 0.6147239263803681,
      "grad_norm": 34.255808614960706,
      "learning_rate": 1.2049586776859504e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.171875,
      "logps/chosen": -410.0,
      "logps/rejected": -352.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.234375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.34375,
      "step": 501
    },
    {
      "epoch": 0.6159509202453988,
      "grad_norm": 28.29113723996875,
      "learning_rate": 1.2043388429752065e-06,
      "logits/chosen": -0.01165771484375,
      "logits/rejected": -0.28125,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.3467,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.03125,
      "step": 502
    },
    {
      "epoch": 0.6171779141104294,
      "grad_norm": 29.752413229458206,
      "learning_rate": 1.2037190082644629e-06,
      "logits/chosen": 0.1904296875,
      "logits/rejected": -0.0013275146484375,
      "logps/chosen": -334.0,
      "logps/rejected": -324.0,
      "loss": 0.2996,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.28515625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.40625,
      "step": 503
    },
    {
      "epoch": 0.6184049079754601,
      "grad_norm": 24.506669229979327,
      "learning_rate": 1.203099173553719e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.234375,
      "logps/chosen": -388.0,
      "logps/rejected": -340.0,
      "loss": 0.2718,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.1025390625,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.3125,
      "step": 504
    },
    {
      "epoch": 0.6196319018404908,
      "grad_norm": 38.20086418972538,
      "learning_rate": 1.2024793388429752e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -434.0,
      "logps/rejected": -406.0,
      "loss": 0.4423,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.609375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.375,
      "step": 505
    },
    {
      "epoch": 0.6208588957055214,
      "grad_norm": 67.15816433272404,
      "learning_rate": 1.2018595041322315e-06,
      "logits/chosen": 0.06982421875,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -428.0,
      "logps/rejected": -340.0,
      "loss": 0.3369,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.0081787109375,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.03125,
      "step": 506
    },
    {
      "epoch": 0.6220858895705521,
      "grad_norm": 25.2127178538785,
      "learning_rate": 1.2012396694214877e-06,
      "logits/chosen": -0.04296875,
      "logits/rejected": -0.150390625,
      "logps/chosen": -394.0,
      "logps/rejected": -318.0,
      "loss": 0.1989,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.11474609375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -7.21875,
      "step": 507
    },
    {
      "epoch": 0.6233128834355828,
      "grad_norm": 31.70942946416519,
      "learning_rate": 1.2006198347107438e-06,
      "logits/chosen": 0.14453125,
      "logits/rejected": -0.033935546875,
      "logps/chosen": -396.0,
      "logps/rejected": -320.0,
      "loss": 0.3214,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1494140625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -6.5625,
      "step": 508
    },
    {
      "epoch": 0.6245398773006134,
      "grad_norm": 39.83330773441419,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -0.0225830078125,
      "logits/rejected": -0.2578125,
      "logps/chosen": -474.0,
      "logps/rejected": -396.0,
      "loss": 0.569,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.21875,
      "step": 509
    },
    {
      "epoch": 0.6257668711656442,
      "grad_norm": 29.863730506340346,
      "learning_rate": 1.199380165289256e-06,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.291015625,
      "logps/chosen": -368.0,
      "logps/rejected": -314.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.6875,
      "step": 510
    },
    {
      "epoch": 0.6269938650306749,
      "grad_norm": 35.188221718693796,
      "learning_rate": 1.1987603305785124e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0859375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -7.21875,
      "step": 511
    },
    {
      "epoch": 0.6282208588957056,
      "grad_norm": 35.23659404136466,
      "learning_rate": 1.1981404958677686e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.0654296875,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.4354,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.388671875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -6.28125,
      "step": 512
    },
    {
      "epoch": 0.6294478527607362,
      "grad_norm": 33.94396537230905,
      "learning_rate": 1.1975206611570247e-06,
      "logits/chosen": 0.07958984375,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -384.0,
      "logps/rejected": -326.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1328125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.8125,
      "step": 513
    },
    {
      "epoch": 0.6306748466257669,
      "grad_norm": 27.33280439239978,
      "learning_rate": 1.196900826446281e-06,
      "logits/chosen": 0.09765625,
      "logits/rejected": -0.0537109375,
      "logps/chosen": -340.0,
      "logps/rejected": -302.0,
      "loss": 0.2677,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -1.015625,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.53125,
      "step": 514
    },
    {
      "epoch": 0.6319018404907976,
      "grad_norm": 32.16724360242484,
      "learning_rate": 1.1962809917355372e-06,
      "logits/chosen": 0.07275390625,
      "logits/rejected": -0.07763671875,
      "logps/chosen": -358.0,
      "logps/rejected": -362.0,
      "loss": 0.3029,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.5625,
      "step": 515
    },
    {
      "epoch": 0.6331288343558282,
      "grad_norm": 26.23429258203521,
      "learning_rate": 1.1956611570247934e-06,
      "logits/chosen": 0.0279541015625,
      "logits/rejected": -0.07861328125,
      "logps/chosen": -378.0,
      "logps/rejected": -346.0,
      "loss": 0.2424,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.8828125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -7.53125,
      "step": 516
    },
    {
      "epoch": 0.6343558282208589,
      "grad_norm": 37.00771401511282,
      "learning_rate": 1.1950413223140497e-06,
      "logits/chosen": 0.0703125,
      "logits/rejected": -0.08642578125,
      "logps/chosen": -400.0,
      "logps/rejected": -352.0,
      "loss": 0.4735,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.48828125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.78125,
      "step": 517
    },
    {
      "epoch": 0.6355828220858896,
      "grad_norm": 28.05157133110184,
      "learning_rate": 1.1944214876033059e-06,
      "logits/chosen": -0.04541015625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -424.0,
      "logps/rejected": -360.0,
      "loss": 0.2475,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.21875,
      "step": 518
    },
    {
      "epoch": 0.6368098159509202,
      "grad_norm": 38.286039724245256,
      "learning_rate": 1.193801652892562e-06,
      "logits/chosen": -0.026123046875,
      "logits/rejected": -0.1630859375,
      "logps/chosen": -396.0,
      "logps/rejected": -354.0,
      "loss": 0.3562,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.60546875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.53125,
      "step": 519
    },
    {
      "epoch": 0.6380368098159509,
      "grad_norm": 34.57537616564543,
      "learning_rate": 1.1931818181818181e-06,
      "logits/chosen": -0.0517578125,
      "logits/rejected": -0.279296875,
      "logps/chosen": -380.0,
      "logps/rejected": -350.0,
      "loss": 0.37,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 7.71875,
      "rewards/rejected": -8.0,
      "step": 520
    },
    {
      "epoch": 0.6392638036809816,
      "grad_norm": 29.132838683315903,
      "learning_rate": 1.1925619834710743e-06,
      "logits/chosen": 0.158203125,
      "logits/rejected": 0.01312255859375,
      "logps/chosen": -428.0,
      "logps/rejected": -378.0,
      "loss": 0.2975,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.01263427734375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.8125,
      "step": 521
    },
    {
      "epoch": 0.6404907975460122,
      "grad_norm": 30.38859952385869,
      "learning_rate": 1.1919421487603306e-06,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -438.0,
      "logps/rejected": -374.0,
      "loss": 0.3466,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.439453125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -8.625,
      "step": 522
    },
    {
      "epoch": 0.6417177914110429,
      "grad_norm": 36.440630501558566,
      "learning_rate": 1.1913223140495868e-06,
      "logits/chosen": 0.0238037109375,
      "logits/rejected": -0.12890625,
      "logps/chosen": -380.0,
      "logps/rejected": -346.0,
      "loss": 0.4467,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.09375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -7.40625,
      "step": 523
    },
    {
      "epoch": 0.6429447852760736,
      "grad_norm": 39.231543678462224,
      "learning_rate": 1.1907024793388431e-06,
      "logits/chosen": 0.05615234375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -440.0,
      "logps/rejected": -380.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -7.59375,
      "step": 524
    },
    {
      "epoch": 0.6441717791411042,
      "grad_norm": 47.6608831234207,
      "learning_rate": 1.1900826446280993e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": 0.02587890625,
      "logps/chosen": -406.0,
      "logps/rejected": -382.0,
      "loss": 0.5676,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -1.3125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -6.84375,
      "step": 525
    },
    {
      "epoch": 0.645398773006135,
      "grad_norm": 31.7891594700196,
      "learning_rate": 1.1894628099173554e-06,
      "logits/chosen": 0.0458984375,
      "logits/rejected": -0.10107421875,
      "logps/chosen": -402.0,
      "logps/rejected": -356.0,
      "loss": 0.3316,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.6171875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.84375,
      "step": 526
    },
    {
      "epoch": 0.6466257668711657,
      "grad_norm": 32.21728261594982,
      "learning_rate": 1.1888429752066118e-06,
      "logits/chosen": 0.1201171875,
      "logits/rejected": -0.103515625,
      "logps/chosen": -438.0,
      "logps/rejected": -348.0,
      "loss": 0.3529,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.7265625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.75,
      "step": 527
    },
    {
      "epoch": 0.6478527607361964,
      "grad_norm": 26.60182420517136,
      "learning_rate": 1.1882231404958677e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.220703125,
      "logps/chosen": -368.0,
      "logps/rejected": -344.0,
      "loss": 0.2928,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.671875,
      "rewards/margins": 7.375,
      "rewards/rejected": -8.0625,
      "step": 528
    },
    {
      "epoch": 0.649079754601227,
      "grad_norm": 32.109755070015275,
      "learning_rate": 1.187603305785124e-06,
      "logits/chosen": 0.030029296875,
      "logits/rejected": -0.14453125,
      "logps/chosen": -352.0,
      "logps/rejected": -352.0,
      "loss": 0.418,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.3984375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.34375,
      "step": 529
    },
    {
      "epoch": 0.6503067484662577,
      "grad_norm": 30.441331665820965,
      "learning_rate": 1.1869834710743802e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -370.0,
      "logps/rejected": -366.0,
      "loss": 0.2962,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.1103515625,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.59375,
      "step": 530
    },
    {
      "epoch": 0.6515337423312884,
      "grad_norm": 25.97427159310545,
      "learning_rate": 1.1863636363636363e-06,
      "logits/chosen": -0.0005950927734375,
      "logits/rejected": -0.1767578125,
      "logps/chosen": -394.0,
      "logps/rejected": -338.0,
      "loss": 0.3013,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.2001953125,
      "rewards/margins": 7.0625,
      "rewards/rejected": -7.25,
      "step": 531
    },
    {
      "epoch": 0.652760736196319,
      "grad_norm": 29.45586726470446,
      "learning_rate": 1.1857438016528927e-06,
      "logits/chosen": 0.12060546875,
      "logits/rejected": -0.01611328125,
      "logps/chosen": -394.0,
      "logps/rejected": -342.0,
      "loss": 0.3256,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.1650390625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.71875,
      "step": 532
    },
    {
      "epoch": 0.6539877300613497,
      "grad_norm": 31.510396061786174,
      "learning_rate": 1.1851239669421488e-06,
      "logits/chosen": -0.0634765625,
      "logits/rejected": -0.265625,
      "logps/chosen": -414.0,
      "logps/rejected": -360.0,
      "loss": 0.322,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.09375,
      "step": 533
    },
    {
      "epoch": 0.6552147239263804,
      "grad_norm": 29.028681159770606,
      "learning_rate": 1.184504132231405e-06,
      "logits/chosen": 0.0908203125,
      "logits/rejected": -0.02685546875,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.3243,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.0341796875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -6.375,
      "step": 534
    },
    {
      "epoch": 0.656441717791411,
      "grad_norm": 33.385986566673,
      "learning_rate": 1.1838842975206613e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.125,
      "logps/chosen": -378.0,
      "logps/rejected": -348.0,
      "loss": 0.3502,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7421875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -7.125,
      "step": 535
    },
    {
      "epoch": 0.6576687116564417,
      "grad_norm": 27.982276879832543,
      "learning_rate": 1.1832644628099175e-06,
      "logits/chosen": 0.0201416015625,
      "logits/rejected": -0.07568359375,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.2519,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.78125,
      "step": 536
    },
    {
      "epoch": 0.6588957055214724,
      "grad_norm": 40.68442319965527,
      "learning_rate": 1.1826446280991736e-06,
      "logits/chosen": 0.06201171875,
      "logits/rejected": -0.0927734375,
      "logps/chosen": -386.0,
      "logps/rejected": -318.0,
      "loss": 0.5685,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 5.28125,
      "rewards/rejected": -5.78125,
      "step": 537
    },
    {
      "epoch": 0.660122699386503,
      "grad_norm": 35.32326255751808,
      "learning_rate": 1.1820247933884297e-06,
      "logits/chosen": 0.068359375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -402.0,
      "logps/rejected": -360.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.78515625,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.96875,
      "step": 538
    },
    {
      "epoch": 0.6613496932515337,
      "grad_norm": 32.525023600229474,
      "learning_rate": 1.1814049586776859e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.04541015625,
      "logps/chosen": -400.0,
      "logps/rejected": -346.0,
      "loss": 0.4519,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 5.84375,
      "rewards/rejected": -5.03125,
      "step": 539
    },
    {
      "epoch": 0.6625766871165644,
      "grad_norm": 35.156155067546834,
      "learning_rate": 1.1807851239669422e-06,
      "logits/chosen": 0.0986328125,
      "logits/rejected": -0.06787109375,
      "logps/chosen": -396.0,
      "logps/rejected": -332.0,
      "loss": 0.432,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.9375,
      "step": 540
    },
    {
      "epoch": 0.6638036809815951,
      "grad_norm": 45.099896848619814,
      "learning_rate": 1.1801652892561984e-06,
      "logits/chosen": 0.08447265625,
      "logits/rejected": -0.0064697265625,
      "logps/chosen": -390.0,
      "logps/rejected": -354.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.279296875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -5.5625,
      "step": 541
    },
    {
      "epoch": 0.6650306748466258,
      "grad_norm": 30.118145811004524,
      "learning_rate": 1.1795454545454545e-06,
      "logits/chosen": 0.031494140625,
      "logits/rejected": -0.053466796875,
      "logps/chosen": -356.0,
      "logps/rejected": -320.0,
      "loss": 0.3659,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.15625,
      "step": 542
    },
    {
      "epoch": 0.6662576687116565,
      "grad_norm": 32.83770736751292,
      "learning_rate": 1.1789256198347109e-06,
      "logits/chosen": -0.02294921875,
      "logits/rejected": -0.201171875,
      "logps/chosen": -450.0,
      "logps/rejected": -354.0,
      "loss": 0.3396,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.375,
      "step": 543
    },
    {
      "epoch": 0.6674846625766871,
      "grad_norm": 35.490916209773005,
      "learning_rate": 1.178305785123967e-06,
      "logits/chosen": 0.007598876953125,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -378.0,
      "logps/rejected": -318.0,
      "loss": 0.434,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.376953125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.78125,
      "step": 544
    },
    {
      "epoch": 0.6687116564417178,
      "grad_norm": 33.60719342971152,
      "learning_rate": 1.1776859504132231e-06,
      "logits/chosen": -0.054443359375,
      "logits/rejected": -0.21875,
      "logps/chosen": -458.0,
      "logps/rejected": -368.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.25,
      "step": 545
    },
    {
      "epoch": 0.6699386503067485,
      "grad_norm": 33.972456048977826,
      "learning_rate": 1.1770661157024793e-06,
      "logits/chosen": 0.0693359375,
      "logits/rejected": -0.028564453125,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.6953125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.71875,
      "step": 546
    },
    {
      "epoch": 0.6711656441717792,
      "grad_norm": 28.303815223441166,
      "learning_rate": 1.1764462809917354e-06,
      "logits/chosen": 0.0133056640625,
      "logits/rejected": -0.166015625,
      "logps/chosen": -386.0,
      "logps/rejected": -312.0,
      "loss": 0.2992,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.88671875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.53125,
      "step": 547
    },
    {
      "epoch": 0.6723926380368098,
      "grad_norm": 38.31133772907115,
      "learning_rate": 1.1758264462809918e-06,
      "logits/chosen": -0.06494140625,
      "logits/rejected": -0.265625,
      "logps/chosen": -402.0,
      "logps/rejected": -358.0,
      "loss": 0.3532,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.015625,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.125,
      "step": 548
    },
    {
      "epoch": 0.6736196319018405,
      "grad_norm": 33.292222723297044,
      "learning_rate": 1.175206611570248e-06,
      "logits/chosen": 0.047607421875,
      "logits/rejected": -0.1572265625,
      "logps/chosen": -414.0,
      "logps/rejected": -344.0,
      "loss": 0.3292,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.5625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -5.40625,
      "step": 549
    },
    {
      "epoch": 0.6748466257668712,
      "grad_norm": 30.807870745155437,
      "learning_rate": 1.174586776859504e-06,
      "logits/chosen": -0.024658203125,
      "logits/rejected": -0.08642578125,
      "logps/chosen": -402.0,
      "logps/rejected": -330.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.5625,
      "step": 550
    },
    {
      "epoch": 0.6760736196319018,
      "grad_norm": 28.384977078060157,
      "learning_rate": 1.1739669421487604e-06,
      "logits/chosen": 0.06787109375,
      "logits/rejected": -0.04541015625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.53125,
      "step": 551
    },
    {
      "epoch": 0.6773006134969325,
      "grad_norm": 29.63861597355486,
      "learning_rate": 1.1733471074380166e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.158203125,
      "logps/chosen": -416.0,
      "logps/rejected": -360.0,
      "loss": 0.3331,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.71875,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.53125,
      "step": 552
    },
    {
      "epoch": 0.6785276073619632,
      "grad_norm": 32.698792932077886,
      "learning_rate": 1.172727272727273e-06,
      "logits/chosen": 0.01806640625,
      "logits/rejected": -0.1259765625,
      "logps/chosen": -366.0,
      "logps/rejected": -342.0,
      "loss": 0.4129,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.1474609375,
      "rewards/margins": 5.625,
      "rewards/rejected": -5.78125,
      "step": 553
    },
    {
      "epoch": 0.6797546012269938,
      "grad_norm": 32.72020008420759,
      "learning_rate": 1.172107438016529e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.1171875,
      "logps/chosen": -434.0,
      "logps/rejected": -334.0,
      "loss": 0.3464,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.59375,
      "step": 554
    },
    {
      "epoch": 0.6809815950920245,
      "grad_norm": 30.608371906092913,
      "learning_rate": 1.171487603305785e-06,
      "logits/chosen": 0.2021484375,
      "logits/rejected": 0.09130859375,
      "logps/chosen": -364.0,
      "logps/rejected": -318.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.0,
      "step": 555
    },
    {
      "epoch": 0.6822085889570552,
      "grad_norm": 26.18493705591653,
      "learning_rate": 1.1708677685950413e-06,
      "logits/chosen": 0.053466796875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -424.0,
      "logps/rejected": -360.0,
      "loss": 0.2487,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.96875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.59375,
      "step": 556
    },
    {
      "epoch": 0.6834355828220859,
      "grad_norm": 27.830299010779843,
      "learning_rate": 1.1702479338842975e-06,
      "logits/chosen": 0.05029296875,
      "logits/rejected": -0.16796875,
      "logps/chosen": -398.0,
      "logps/rejected": -346.0,
      "loss": 0.3343,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.53515625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.15625,
      "step": 557
    },
    {
      "epoch": 0.6846625766871166,
      "grad_norm": 25.854408165941038,
      "learning_rate": 1.1696280991735538e-06,
      "logits/chosen": 0.087890625,
      "logits/rejected": -0.078125,
      "logps/chosen": -378.0,
      "logps/rejected": -340.0,
      "loss": 0.2782,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.3125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.25,
      "step": 558
    },
    {
      "epoch": 0.6858895705521473,
      "grad_norm": 33.3339739767785,
      "learning_rate": 1.16900826446281e-06,
      "logits/chosen": -0.072265625,
      "logits/rejected": -0.1591796875,
      "logps/chosen": -388.0,
      "logps/rejected": -354.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.78125,
      "step": 559
    },
    {
      "epoch": 0.6871165644171779,
      "grad_norm": 21.74474553758103,
      "learning_rate": 1.1683884297520661e-06,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -394.0,
      "logps/rejected": -344.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.6875,
      "step": 560
    },
    {
      "epoch": 0.6883435582822086,
      "grad_norm": 24.604934270788146,
      "learning_rate": 1.1677685950413225e-06,
      "logits/chosen": 0.00164031982421875,
      "logits/rejected": -0.177734375,
      "logps/chosen": -360.0,
      "logps/rejected": -314.0,
      "loss": 0.2132,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.171875,
      "rewards/margins": 7.25,
      "rewards/rejected": -7.09375,
      "step": 561
    },
    {
      "epoch": 0.6895705521472393,
      "grad_norm": 25.33619721263671,
      "learning_rate": 1.1671487603305786e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": -0.064453125,
      "logps/chosen": -382.0,
      "logps/rejected": -336.0,
      "loss": 0.2353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.55078125,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.3125,
      "step": 562
    },
    {
      "epoch": 0.69079754601227,
      "grad_norm": 28.2123146430669,
      "learning_rate": 1.1665289256198348e-06,
      "logits/chosen": -0.055908203125,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -398.0,
      "logps/rejected": -386.0,
      "loss": 0.3157,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.3125,
      "step": 563
    },
    {
      "epoch": 0.6920245398773006,
      "grad_norm": 24.610651917154996,
      "learning_rate": 1.1659090909090909e-06,
      "logits/chosen": -0.040283203125,
      "logits/rejected": -0.1552734375,
      "logps/chosen": -396.0,
      "logps/rejected": -350.0,
      "loss": 0.3066,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.21875,
      "rewards/margins": 6.625,
      "rewards/rejected": -6.84375,
      "step": 564
    },
    {
      "epoch": 0.6932515337423313,
      "grad_norm": 30.241252760906917,
      "learning_rate": 1.165289256198347e-06,
      "logits/chosen": 0.0081787109375,
      "logits/rejected": -0.1796875,
      "logps/chosen": -404.0,
      "logps/rejected": -354.0,
      "loss": 0.4933,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.396484375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.40625,
      "step": 565
    },
    {
      "epoch": 0.694478527607362,
      "grad_norm": 30.063355062368128,
      "learning_rate": 1.1646694214876034e-06,
      "logits/chosen": 0.004852294921875,
      "logits/rejected": -0.08544921875,
      "logps/chosen": -420.0,
      "logps/rejected": -348.0,
      "loss": 0.2711,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.765625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.53125,
      "step": 566
    },
    {
      "epoch": 0.6957055214723926,
      "grad_norm": 23.27581640569879,
      "learning_rate": 1.1640495867768595e-06,
      "logits/chosen": 0.11865234375,
      "logits/rejected": -0.0096435546875,
      "logps/chosen": -322.0,
      "logps/rejected": -288.0,
      "loss": 0.2144,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.30859375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.34375,
      "step": 567
    },
    {
      "epoch": 0.6969325153374233,
      "grad_norm": 30.55160170642105,
      "learning_rate": 1.1634297520661157e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.0869140625,
      "logps/chosen": -388.0,
      "logps/rejected": -334.0,
      "loss": 0.3414,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.357421875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -6.375,
      "step": 568
    },
    {
      "epoch": 0.698159509202454,
      "grad_norm": 29.293298679377934,
      "learning_rate": 1.162809917355372e-06,
      "logits/chosen": 0.04736328125,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -388.0,
      "logps/rejected": -316.0,
      "loss": 0.2723,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0361328125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.3125,
      "step": 569
    },
    {
      "epoch": 0.6993865030674846,
      "grad_norm": 29.217886094805987,
      "learning_rate": 1.1621900826446282e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2078,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.9375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.46875,
      "step": 570
    },
    {
      "epoch": 0.7006134969325153,
      "grad_norm": 29.496729411168463,
      "learning_rate": 1.1615702479338843e-06,
      "logits/chosen": 0.0179443359375,
      "logits/rejected": -0.10205078125,
      "logps/chosen": -332.0,
      "logps/rejected": -314.0,
      "loss": 0.3188,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0294189453125,
      "rewards/margins": 5.875,
      "rewards/rejected": -5.84375,
      "step": 571
    },
    {
      "epoch": 0.701840490797546,
      "grad_norm": 28.740254642158334,
      "learning_rate": 1.1609504132231407e-06,
      "logits/chosen": 0.03173828125,
      "logits/rejected": -0.11669921875,
      "logps/chosen": -388.0,
      "logps/rejected": -378.0,
      "loss": 0.2732,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.1484375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.25,
      "step": 572
    },
    {
      "epoch": 0.7030674846625767,
      "grad_norm": 35.19506105279461,
      "learning_rate": 1.1603305785123966e-06,
      "logits/chosen": 0.078125,
      "logits/rejected": -0.050048828125,
      "logps/chosen": -394.0,
      "logps/rejected": -346.0,
      "loss": 0.5289,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.150390625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.59375,
      "step": 573
    },
    {
      "epoch": 0.7042944785276074,
      "grad_norm": 30.319920980400575,
      "learning_rate": 1.159710743801653e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -328.0,
      "logps/rejected": -320.0,
      "loss": 0.34,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.28125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.5,
      "step": 574
    },
    {
      "epoch": 0.7055214723926381,
      "grad_norm": 29.615184087034994,
      "learning_rate": 1.159090909090909e-06,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -412.0,
      "logps/rejected": -388.0,
      "loss": 0.2626,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.34375,
      "step": 575
    },
    {
      "epoch": 0.7067484662576687,
      "grad_norm": 22.847983936498952,
      "learning_rate": 1.1584710743801652e-06,
      "logits/chosen": 0.03271484375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -386.0,
      "logps/rejected": -344.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.0625,
      "rewards/margins": 8.5,
      "rewards/rejected": -7.46875,
      "step": 576
    },
    {
      "epoch": 0.7079754601226994,
      "grad_norm": 33.54348915562207,
      "learning_rate": 1.1578512396694216e-06,
      "logits/chosen": 0.08935546875,
      "logits/rejected": -0.0296630859375,
      "logps/chosen": -368.0,
      "logps/rejected": -334.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.828125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.34375,
      "step": 577
    },
    {
      "epoch": 0.7092024539877301,
      "grad_norm": 31.17463611360663,
      "learning_rate": 1.1572314049586777e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.208984375,
      "logps/chosen": -420.0,
      "logps/rejected": -354.0,
      "loss": 0.3672,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.921875,
      "rewards/margins": 8.6875,
      "rewards/rejected": -7.75,
      "step": 578
    },
    {
      "epoch": 0.7104294478527607,
      "grad_norm": 37.58239380773895,
      "learning_rate": 1.1566115702479339e-06,
      "logits/chosen": -0.025390625,
      "logits/rejected": -0.1181640625,
      "logps/chosen": -440.0,
      "logps/rejected": -352.0,
      "loss": 0.4655,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -7.0,
      "step": 579
    },
    {
      "epoch": 0.7116564417177914,
      "grad_norm": 34.56148267033386,
      "learning_rate": 1.1559917355371902e-06,
      "logits/chosen": 0.0673828125,
      "logits/rejected": -0.10791015625,
      "logps/chosen": -390.0,
      "logps/rejected": -374.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.21484375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -8.3125,
      "step": 580
    },
    {
      "epoch": 0.7128834355828221,
      "grad_norm": 26.418081399921064,
      "learning_rate": 1.1553719008264464e-06,
      "logits/chosen": -0.0498046875,
      "logits/rejected": -0.244140625,
      "logps/chosen": -380.0,
      "logps/rejected": -338.0,
      "loss": 0.2267,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -8.0625,
      "step": 581
    },
    {
      "epoch": 0.7141104294478527,
      "grad_norm": 30.496748663730113,
      "learning_rate": 1.1547520661157025e-06,
      "logits/chosen": 0.197265625,
      "logits/rejected": -0.02587890625,
      "logps/chosen": -402.0,
      "logps/rejected": -350.0,
      "loss": 0.3169,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.01220703125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -7.0,
      "step": 582
    },
    {
      "epoch": 0.7153374233128834,
      "grad_norm": 31.96559757214927,
      "learning_rate": 1.1541322314049586e-06,
      "logits/chosen": -0.02880859375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -392.0,
      "logps/rejected": -364.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.3671875,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.125,
      "step": 583
    },
    {
      "epoch": 0.7165644171779141,
      "grad_norm": 36.407577582473756,
      "learning_rate": 1.1535123966942148e-06,
      "logits/chosen": 0.08154296875,
      "logits/rejected": 0.0218505859375,
      "logps/chosen": -380.0,
      "logps/rejected": -392.0,
      "loss": 0.431,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -6.375,
      "step": 584
    },
    {
      "epoch": 0.7177914110429447,
      "grad_norm": 41.799396220975545,
      "learning_rate": 1.1528925619834711e-06,
      "logits/chosen": 0.1796875,
      "logits/rejected": 0.0634765625,
      "logps/chosen": -410.0,
      "logps/rejected": -344.0,
      "loss": 0.5833,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 0.2255859375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.21875,
      "step": 585
    },
    {
      "epoch": 0.7190184049079754,
      "grad_norm": 32.85377702391839,
      "learning_rate": 1.1522727272727273e-06,
      "logits/chosen": 0.035888671875,
      "logits/rejected": -0.07373046875,
      "logps/chosen": -356.0,
      "logps/rejected": -324.0,
      "loss": 0.3732,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.00927734375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.90625,
      "step": 586
    },
    {
      "epoch": 0.7202453987730061,
      "grad_norm": 28.585914695021586,
      "learning_rate": 1.1516528925619836e-06,
      "logits/chosen": 0.01116943359375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -342.0,
      "logps/rejected": -332.0,
      "loss": 0.3594,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.08935546875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.96875,
      "step": 587
    },
    {
      "epoch": 0.7214723926380369,
      "grad_norm": 24.10336561473994,
      "learning_rate": 1.1510330578512398e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -388.0,
      "logps/rejected": -342.0,
      "loss": 0.2892,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -6.875,
      "step": 588
    },
    {
      "epoch": 0.7226993865030675,
      "grad_norm": 28.27312313115512,
      "learning_rate": 1.150413223140496e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -372.0,
      "logps/rejected": -354.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.37890625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.65625,
      "step": 589
    },
    {
      "epoch": 0.7239263803680982,
      "grad_norm": 30.343626428171632,
      "learning_rate": 1.149793388429752e-06,
      "logits/chosen": 0.00927734375,
      "logits/rejected": -0.103515625,
      "logps/chosen": -374.0,
      "logps/rejected": -330.0,
      "loss": 0.3324,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0986328125,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.78125,
      "step": 590
    },
    {
      "epoch": 0.7251533742331289,
      "grad_norm": 36.86660943740023,
      "learning_rate": 1.1491735537190082e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.373046875,
      "logps/chosen": -414.0,
      "logps/rejected": -354.0,
      "loss": 0.28,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.3125,
      "step": 591
    },
    {
      "epoch": 0.7263803680981595,
      "grad_norm": 27.76314433981488,
      "learning_rate": 1.1485537190082645e-06,
      "logits/chosen": -0.02490234375,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -378.0,
      "logps/rejected": -328.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.78125,
      "step": 592
    },
    {
      "epoch": 0.7276073619631902,
      "grad_norm": 36.82046387528082,
      "learning_rate": 1.1479338842975207e-06,
      "logits/chosen": 0.022705078125,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -454.0,
      "logps/rejected": -392.0,
      "loss": 0.4117,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.76953125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.09375,
      "step": 593
    },
    {
      "epoch": 0.7288343558282209,
      "grad_norm": 30.96499084941041,
      "learning_rate": 1.1473140495867768e-06,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.197265625,
      "logps/chosen": -366.0,
      "logps/rejected": -334.0,
      "loss": 0.3336,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.65625,
      "step": 594
    },
    {
      "epoch": 0.7300613496932515,
      "grad_norm": 33.881983550723454,
      "learning_rate": 1.1466942148760332e-06,
      "logits/chosen": -0.059814453125,
      "logits/rejected": -0.26953125,
      "logps/chosen": -454.0,
      "logps/rejected": -372.0,
      "loss": 0.2673,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.71875,
      "step": 595
    },
    {
      "epoch": 0.7312883435582822,
      "grad_norm": 34.4461650357885,
      "learning_rate": 1.1460743801652893e-06,
      "logits/chosen": -0.0286865234375,
      "logits/rejected": -0.18359375,
      "logps/chosen": -422.0,
      "logps/rejected": -368.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.004608154296875,
      "rewards/margins": 7.25,
      "rewards/rejected": -7.25,
      "step": 596
    },
    {
      "epoch": 0.7325153374233129,
      "grad_norm": 32.96451484117692,
      "learning_rate": 1.1454545454545455e-06,
      "logits/chosen": 0.0830078125,
      "logits/rejected": -0.09228515625,
      "logps/chosen": -388.0,
      "logps/rejected": -306.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.53125,
      "step": 597
    },
    {
      "epoch": 0.7337423312883435,
      "grad_norm": 25.225306353625232,
      "learning_rate": 1.1448347107438018e-06,
      "logits/chosen": -0.0211181640625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -446.0,
      "logps/rejected": -338.0,
      "loss": 0.2285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.296875,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.09375,
      "step": 598
    },
    {
      "epoch": 0.7349693251533742,
      "grad_norm": 30.51091893610521,
      "learning_rate": 1.1442148760330577e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.3125,
      "logps/chosen": -456.0,
      "logps/rejected": -386.0,
      "loss": 0.3423,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.53125,
      "step": 599
    },
    {
      "epoch": 0.7361963190184049,
      "grad_norm": 31.65374536927,
      "learning_rate": 1.143595041322314e-06,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.25390625,
      "logps/chosen": -364.0,
      "logps/rejected": -334.0,
      "loss": 0.437,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.1875,
      "step": 600
    },
    {
      "epoch": 0.7374233128834355,
      "grad_norm": 29.257508369320092,
      "learning_rate": 1.1429752066115702e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -348.0,
      "logps/rejected": -324.0,
      "loss": 0.3337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.71875,
      "step": 601
    },
    {
      "epoch": 0.7386503067484662,
      "grad_norm": 23.637703488224467,
      "learning_rate": 1.1423553719008264e-06,
      "logits/chosen": 0.1103515625,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -406.0,
      "logps/rejected": -350.0,
      "loss": 0.2192,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.78125,
      "step": 602
    },
    {
      "epoch": 0.7398773006134969,
      "grad_norm": 24.798455930230393,
      "learning_rate": 1.1417355371900827e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -382.0,
      "logps/rejected": -336.0,
      "loss": 0.3279,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.5,
      "step": 603
    },
    {
      "epoch": 0.7411042944785277,
      "grad_norm": 28.661443673376716,
      "learning_rate": 1.1411157024793389e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -420.0,
      "logps/rejected": -374.0,
      "loss": 0.2818,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.8125,
      "step": 604
    },
    {
      "epoch": 0.7423312883435583,
      "grad_norm": 28.682789953669307,
      "learning_rate": 1.140495867768595e-06,
      "logits/chosen": -0.057373046875,
      "logits/rejected": -0.24609375,
      "logps/chosen": -380.0,
      "logps/rejected": -348.0,
      "loss": 0.2877,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.349609375,
      "rewards/margins": 8.5,
      "rewards/rejected": -8.125,
      "step": 605
    },
    {
      "epoch": 0.743558282208589,
      "grad_norm": 31.866662459492442,
      "learning_rate": 1.1398760330578514e-06,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.263671875,
      "logps/chosen": -390.0,
      "logps/rejected": -348.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.21875,
      "step": 606
    },
    {
      "epoch": 0.7447852760736197,
      "grad_norm": 29.660350580126263,
      "learning_rate": 1.1392561983471075e-06,
      "logits/chosen": -0.0289306640625,
      "logits/rejected": -0.25,
      "logps/chosen": -404.0,
      "logps/rejected": -386.0,
      "loss": 0.3196,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.265625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -8.3125,
      "step": 607
    },
    {
      "epoch": 0.7460122699386503,
      "grad_norm": 31.21931912524627,
      "learning_rate": 1.1386363636363636e-06,
      "logits/chosen": 0.10498046875,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -404.0,
      "logps/rejected": -360.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.5,
      "step": 608
    },
    {
      "epoch": 0.747239263803681,
      "grad_norm": 35.278030303126314,
      "learning_rate": 1.1380165289256198e-06,
      "logits/chosen": -0.02978515625,
      "logits/rejected": -0.1953125,
      "logps/chosen": -412.0,
      "logps/rejected": -342.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 7.9375,
      "rewards/rejected": -8.1875,
      "step": 609
    },
    {
      "epoch": 0.7484662576687117,
      "grad_norm": 29.748859331814412,
      "learning_rate": 1.137396694214876e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.3203125,
      "logps/chosen": -452.0,
      "logps/rejected": -396.0,
      "loss": 0.2827,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 8.625,
      "rewards/rejected": -8.375,
      "step": 610
    },
    {
      "epoch": 0.7496932515337423,
      "grad_norm": 42.36265371910942,
      "learning_rate": 1.1367768595041323e-06,
      "logits/chosen": -0.050537109375,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -442.0,
      "logps/rejected": -384.0,
      "loss": 0.493,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.234375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -8.0625,
      "step": 611
    },
    {
      "epoch": 0.750920245398773,
      "grad_norm": 36.03030744089288,
      "learning_rate": 1.1361570247933884e-06,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.1796875,
      "logps/chosen": -402.0,
      "logps/rejected": -376.0,
      "loss": 0.4916,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.462890625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.25,
      "step": 612
    },
    {
      "epoch": 0.7521472392638037,
      "grad_norm": 35.23308877108843,
      "learning_rate": 1.1355371900826446e-06,
      "logits/chosen": -0.04052734375,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -388.0,
      "logps/rejected": -348.0,
      "loss": 0.4598,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -7.875,
      "step": 613
    },
    {
      "epoch": 0.7533742331288343,
      "grad_norm": 35.827377930375896,
      "learning_rate": 1.134917355371901e-06,
      "logits/chosen": -0.095703125,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -430.0,
      "logps/rejected": -364.0,
      "loss": 0.3803,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.71875,
      "step": 614
    },
    {
      "epoch": 0.754601226993865,
      "grad_norm": 31.70255072714647,
      "learning_rate": 1.134297520661157e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.166015625,
      "logps/chosen": -376.0,
      "logps/rejected": -364.0,
      "loss": 0.3435,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.037841796875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.8125,
      "step": 615
    },
    {
      "epoch": 0.7558282208588957,
      "grad_norm": 30.125086244839487,
      "learning_rate": 1.1336776859504134e-06,
      "logits/chosen": -0.004241943359375,
      "logits/rejected": -0.15625,
      "logps/chosen": -332.0,
      "logps/rejected": -310.0,
      "loss": 0.3151,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.6328125,
      "rewards/margins": 6.53125,
      "rewards/rejected": -7.15625,
      "step": 616
    },
    {
      "epoch": 0.7570552147239263,
      "grad_norm": 32.7474160937551,
      "learning_rate": 1.1330578512396693e-06,
      "logits/chosen": 0.030517578125,
      "logits/rejected": -0.0390625,
      "logps/chosen": -378.0,
      "logps/rejected": -336.0,
      "loss": 0.3705,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.15625,
      "step": 617
    },
    {
      "epoch": 0.758282208588957,
      "grad_norm": 45.056199177526004,
      "learning_rate": 1.1324380165289255e-06,
      "logits/chosen": 0.109375,
      "logits/rejected": -0.056396484375,
      "logps/chosen": -426.0,
      "logps/rejected": -332.0,
      "loss": 0.5324,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.625,
      "step": 618
    },
    {
      "epoch": 0.7595092024539877,
      "grad_norm": 29.95096958444537,
      "learning_rate": 1.1318181818181818e-06,
      "logits/chosen": 0.08056640625,
      "logits/rejected": -0.12060546875,
      "logps/chosen": -432.0,
      "logps/rejected": -350.0,
      "loss": 0.2823,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.36328125,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.34375,
      "step": 619
    },
    {
      "epoch": 0.7607361963190185,
      "grad_norm": 37.035466575479155,
      "learning_rate": 1.131198347107438e-06,
      "logits/chosen": 0.08349609375,
      "logits/rejected": -0.11181640625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3202,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.03271484375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.71875,
      "step": 620
    },
    {
      "epoch": 0.7619631901840491,
      "grad_norm": 30.546031931386054,
      "learning_rate": 1.1305785123966943e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -482.0,
      "logps/rejected": -380.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.40625,
      "step": 621
    },
    {
      "epoch": 0.7631901840490798,
      "grad_norm": 30.102626290429743,
      "learning_rate": 1.1299586776859505e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.0791015625,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.2060546875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.15625,
      "step": 622
    },
    {
      "epoch": 0.7644171779141105,
      "grad_norm": 31.272882227536257,
      "learning_rate": 1.1293388429752066e-06,
      "logits/chosen": 0.041259765625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -456.0,
      "logps/rejected": -390.0,
      "loss": 0.338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.78125,
      "step": 623
    },
    {
      "epoch": 0.7656441717791411,
      "grad_norm": 29.25638333606671,
      "learning_rate": 1.128719008264463e-06,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -382.0,
      "logps/rejected": -334.0,
      "loss": 0.302,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 8.5,
      "rewards/rejected": -7.59375,
      "step": 624
    },
    {
      "epoch": 0.7668711656441718,
      "grad_norm": 27.76458750020888,
      "learning_rate": 1.1280991735537191e-06,
      "logits/chosen": 0.01214599609375,
      "logits/rejected": -0.123046875,
      "logps/chosen": -408.0,
      "logps/rejected": -334.0,
      "loss": 0.2518,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.69140625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.46875,
      "step": 625
    },
    {
      "epoch": 0.7680981595092025,
      "grad_norm": 34.03667353357896,
      "learning_rate": 1.1274793388429753e-06,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -416.0,
      "logps/rejected": -358.0,
      "loss": 0.4042,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.140625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -5.875,
      "step": 626
    },
    {
      "epoch": 0.7693251533742331,
      "grad_norm": 30.346756983376576,
      "learning_rate": 1.1268595041322314e-06,
      "logits/chosen": 0.095703125,
      "logits/rejected": 0.0223388671875,
      "logps/chosen": -314.0,
      "logps/rejected": -332.0,
      "loss": 0.3854,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.65625,
      "rewards/margins": 5.28125,
      "rewards/rejected": -5.9375,
      "step": 627
    },
    {
      "epoch": 0.7705521472392638,
      "grad_norm": 31.196230115400443,
      "learning_rate": 1.1262396694214875e-06,
      "logits/chosen": 0.0732421875,
      "logits/rejected": -0.06884765625,
      "logps/chosen": -380.0,
      "logps/rejected": -304.0,
      "loss": 0.2959,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.70703125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.71875,
      "step": 628
    },
    {
      "epoch": 0.7717791411042945,
      "grad_norm": 27.109929558619076,
      "learning_rate": 1.1256198347107439e-06,
      "logits/chosen": 0.0262451171875,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -352.0,
      "logps/rejected": -292.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.353515625,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.15625,
      "step": 629
    },
    {
      "epoch": 0.7730061349693251,
      "grad_norm": 27.79699333558513,
      "learning_rate": 1.125e-06,
      "logits/chosen": 0.0947265625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -374.0,
      "logps/rejected": -372.0,
      "loss": 0.3309,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.21875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.875,
      "step": 630
    },
    {
      "epoch": 0.7742331288343558,
      "grad_norm": 32.55842929304895,
      "learning_rate": 1.1243801652892562e-06,
      "logits/chosen": -0.11474609375,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -394.0,
      "logps/rejected": -326.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.6875,
      "step": 631
    },
    {
      "epoch": 0.7754601226993865,
      "grad_norm": 30.882292248466843,
      "learning_rate": 1.1237603305785125e-06,
      "logits/chosen": -0.0111083984375,
      "logits/rejected": -0.134765625,
      "logps/chosen": -406.0,
      "logps/rejected": -344.0,
      "loss": 0.3282,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.125,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.09375,
      "step": 632
    },
    {
      "epoch": 0.7766871165644171,
      "grad_norm": 32.30019011289937,
      "learning_rate": 1.1231404958677687e-06,
      "logits/chosen": 0.072265625,
      "logits/rejected": -0.03857421875,
      "logps/chosen": -342.0,
      "logps/rejected": -338.0,
      "loss": 0.3515,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.39453125,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.34375,
      "step": 633
    },
    {
      "epoch": 0.7779141104294478,
      "grad_norm": 26.84260857948927,
      "learning_rate": 1.1225206611570248e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.248046875,
      "logps/chosen": -416.0,
      "logps/rejected": -328.0,
      "loss": 0.2366,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -5.78125,
      "step": 634
    },
    {
      "epoch": 0.7791411042944786,
      "grad_norm": 23.744218806065337,
      "learning_rate": 1.121900826446281e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -362.0,
      "logps/rejected": -302.0,
      "loss": 0.2194,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.5625,
      "step": 635
    },
    {
      "epoch": 0.7803680981595092,
      "grad_norm": 34.66381046216169,
      "learning_rate": 1.121280991735537e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.166015625,
      "logps/chosen": -414.0,
      "logps/rejected": -368.0,
      "loss": 0.4205,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.46875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.1875,
      "step": 636
    },
    {
      "epoch": 0.7815950920245399,
      "grad_norm": 41.24072249858344,
      "learning_rate": 1.1206611570247934e-06,
      "logits/chosen": 0.11962890625,
      "logits/rejected": -0.0615234375,
      "logps/chosen": -382.0,
      "logps/rejected": -306.0,
      "loss": 0.4805,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.6875,
      "step": 637
    },
    {
      "epoch": 0.7828220858895706,
      "grad_norm": 31.604533876936294,
      "learning_rate": 1.1200413223140496e-06,
      "logits/chosen": -0.044677734375,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -406.0,
      "logps/rejected": -340.0,
      "loss": 0.2858,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -5.625,
      "step": 638
    },
    {
      "epoch": 0.7840490797546013,
      "grad_norm": 34.862589865848754,
      "learning_rate": 1.1194214876033057e-06,
      "logits/chosen": 0.0869140625,
      "logits/rejected": -0.080078125,
      "logps/chosen": -408.0,
      "logps/rejected": -342.0,
      "loss": 0.4829,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.046875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.28125,
      "step": 639
    },
    {
      "epoch": 0.7852760736196319,
      "grad_norm": 29.526720756728196,
      "learning_rate": 1.118801652892562e-06,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -446.0,
      "logps/rejected": -394.0,
      "loss": 0.2982,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.6328125,
      "rewards/margins": 7.5,
      "rewards/rejected": -5.875,
      "step": 640
    },
    {
      "epoch": 0.7865030674846626,
      "grad_norm": 38.16790195191018,
      "learning_rate": 1.1181818181818182e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -398.0,
      "logps/rejected": -360.0,
      "loss": 0.5882,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.90625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.25,
      "step": 641
    },
    {
      "epoch": 0.7877300613496933,
      "grad_norm": 25.77138922359556,
      "learning_rate": 1.1175619834710746e-06,
      "logits/chosen": -0.01422119140625,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -352.0,
      "logps/rejected": -304.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.75,
      "step": 642
    },
    {
      "epoch": 0.7889570552147239,
      "grad_norm": 35.220692662985385,
      "learning_rate": 1.1169421487603307e-06,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.333984375,
      "logps/chosen": -406.0,
      "logps/rejected": -386.0,
      "loss": 0.327,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.9375,
      "step": 643
    },
    {
      "epoch": 0.7901840490797546,
      "grad_norm": 30.676171080381682,
      "learning_rate": 1.1163223140495866e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.21875,
      "logps/chosen": -392.0,
      "logps/rejected": -338.0,
      "loss": 0.3619,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.25,
      "rewards/rejected": -6.5,
      "step": 644
    },
    {
      "epoch": 0.7914110429447853,
      "grad_norm": 29.15888390878358,
      "learning_rate": 1.115702479338843e-06,
      "logits/chosen": -0.03955078125,
      "logits/rejected": -0.234375,
      "logps/chosen": -336.0,
      "logps/rejected": -346.0,
      "loss": 0.2816,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.98046875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.5625,
      "step": 645
    },
    {
      "epoch": 0.7926380368098159,
      "grad_norm": 28.593454046054227,
      "learning_rate": 1.1150826446280991e-06,
      "logits/chosen": 0.047119140625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.2765,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.59375,
      "step": 646
    },
    {
      "epoch": 0.7938650306748466,
      "grad_norm": 32.656529890149606,
      "learning_rate": 1.1144628099173555e-06,
      "logits/chosen": -0.033935546875,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -386.0,
      "logps/rejected": -334.0,
      "loss": 0.3573,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.9296875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.875,
      "step": 647
    },
    {
      "epoch": 0.7950920245398773,
      "grad_norm": 36.18303049965862,
      "learning_rate": 1.1138429752066116e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.08837890625,
      "logps/chosen": -334.0,
      "logps/rejected": -336.0,
      "loss": 0.438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0986328125,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.46875,
      "step": 648
    },
    {
      "epoch": 0.7963190184049079,
      "grad_norm": 33.937125436202074,
      "learning_rate": 1.1132231404958678e-06,
      "logits/chosen": 0.061279296875,
      "logits/rejected": -0.027587890625,
      "logps/chosen": -372.0,
      "logps/rejected": -342.0,
      "loss": 0.5003,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0732421875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.9375,
      "step": 649
    },
    {
      "epoch": 0.7975460122699386,
      "grad_norm": 36.01649609906715,
      "learning_rate": 1.1126033057851241e-06,
      "logits/chosen": 0.0264892578125,
      "logits/rejected": -0.1298828125,
      "logps/chosen": -432.0,
      "logps/rejected": -364.0,
      "loss": 0.4749,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.71875,
      "rewards/margins": 6.5,
      "rewards/rejected": -5.78125,
      "step": 650
    },
    {
      "epoch": 0.7987730061349694,
      "grad_norm": 37.059100709512336,
      "learning_rate": 1.1119834710743803e-06,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -376.0,
      "logps/rejected": -362.0,
      "loss": 0.4208,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.125,
      "step": 651
    },
    {
      "epoch": 0.8,
      "grad_norm": 40.43433181753938,
      "learning_rate": 1.1113636363636364e-06,
      "logits/chosen": -0.031494140625,
      "logits/rejected": -0.1455078125,
      "logps/chosen": -376.0,
      "logps/rejected": -360.0,
      "loss": 0.4169,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 6.625,
      "rewards/rejected": -7.125,
      "step": 652
    },
    {
      "epoch": 0.8012269938650307,
      "grad_norm": 32.30744337627645,
      "learning_rate": 1.1107438016528925e-06,
      "logits/chosen": 0.032958984375,
      "logits/rejected": -0.142578125,
      "logps/chosen": -358.0,
      "logps/rejected": -346.0,
      "loss": 0.2863,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0498046875,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.59375,
      "step": 653
    },
    {
      "epoch": 0.8024539877300614,
      "grad_norm": 23.943395122229603,
      "learning_rate": 1.1101239669421487e-06,
      "logits/chosen": 0.050537109375,
      "logits/rejected": -0.15234375,
      "logps/chosen": -400.0,
      "logps/rejected": -336.0,
      "loss": 0.226,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.333984375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.625,
      "step": 654
    },
    {
      "epoch": 0.803680981595092,
      "grad_norm": 31.423267487900187,
      "learning_rate": 1.109504132231405e-06,
      "logits/chosen": -0.007659912109375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -396.0,
      "logps/rejected": -366.0,
      "loss": 0.3332,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.1943359375,
      "rewards/margins": 7.125,
      "rewards/rejected": -7.3125,
      "step": 655
    },
    {
      "epoch": 0.8049079754601227,
      "grad_norm": 35.483670047647635,
      "learning_rate": 1.1088842975206612e-06,
      "logits/chosen": 0.055419921875,
      "logits/rejected": -0.08740234375,
      "logps/chosen": -380.0,
      "logps/rejected": -344.0,
      "loss": 0.4163,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.5,
      "step": 656
    },
    {
      "epoch": 0.8061349693251534,
      "grad_norm": 27.21341640433181,
      "learning_rate": 1.1082644628099173e-06,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -372.0,
      "logps/rejected": -354.0,
      "loss": 0.3384,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.388671875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -7.5625,
      "step": 657
    },
    {
      "epoch": 0.807361963190184,
      "grad_norm": 31.34176371809714,
      "learning_rate": 1.1076446280991737e-06,
      "logits/chosen": -0.08349609375,
      "logits/rejected": -0.212890625,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.3063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.412109375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.625,
      "step": 658
    },
    {
      "epoch": 0.8085889570552147,
      "grad_norm": 32.62830203002507,
      "learning_rate": 1.1070247933884298e-06,
      "logits/chosen": -0.0291748046875,
      "logits/rejected": -0.16796875,
      "logps/chosen": -374.0,
      "logps/rejected": -348.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -7.71875,
      "step": 659
    },
    {
      "epoch": 0.8098159509202454,
      "grad_norm": 29.335771689820604,
      "learning_rate": 1.106404958677686e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.220703125,
      "logps/chosen": -412.0,
      "logps/rejected": -374.0,
      "loss": 0.3215,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.053466796875,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.75,
      "step": 660
    },
    {
      "epoch": 0.811042944785276,
      "grad_norm": 39.19555957381655,
      "learning_rate": 1.1057851239669423e-06,
      "logits/chosen": 0.054443359375,
      "logits/rejected": -0.052734375,
      "logps/chosen": -374.0,
      "logps/rejected": -318.0,
      "loss": 0.5135,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.09375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -7.5625,
      "step": 661
    },
    {
      "epoch": 0.8122699386503067,
      "grad_norm": 25.59951308246572,
      "learning_rate": 1.1051652892561982e-06,
      "logits/chosen": 0.05224609375,
      "logits/rejected": -0.06884765625,
      "logps/chosen": -372.0,
      "logps/rejected": -342.0,
      "loss": 0.218,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.3671875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.8125,
      "step": 662
    },
    {
      "epoch": 0.8134969325153374,
      "grad_norm": 29.103568092770104,
      "learning_rate": 1.1045454545454546e-06,
      "logits/chosen": 0.1142578125,
      "logits/rejected": -0.080078125,
      "logps/chosen": -362.0,
      "logps/rejected": -340.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.8125,
      "step": 663
    },
    {
      "epoch": 0.8147239263803681,
      "grad_norm": 24.88151445188299,
      "learning_rate": 1.1039256198347107e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.35546875,
      "logps/chosen": -388.0,
      "logps/rejected": -376.0,
      "loss": 0.213,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.7578125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -8.3125,
      "step": 664
    },
    {
      "epoch": 0.8159509202453987,
      "grad_norm": 34.48345990804285,
      "learning_rate": 1.1033057851239669e-06,
      "logits/chosen": 0.1259765625,
      "logits/rejected": -0.0419921875,
      "logps/chosen": -342.0,
      "logps/rejected": -308.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2470703125,
      "rewards/margins": 7.125,
      "rewards/rejected": -7.375,
      "step": 665
    },
    {
      "epoch": 0.8171779141104294,
      "grad_norm": 35.79692152845337,
      "learning_rate": 1.1026859504132232e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.0927734375,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4703,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.59375,
      "step": 666
    },
    {
      "epoch": 0.8184049079754602,
      "grad_norm": 29.967347606106397,
      "learning_rate": 1.1020661157024794e-06,
      "logits/chosen": -0.010986328125,
      "logits/rejected": -0.2109375,
      "logps/chosen": -376.0,
      "logps/rejected": -344.0,
      "loss": 0.2918,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.7421875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.78125,
      "step": 667
    },
    {
      "epoch": 0.8196319018404908,
      "grad_norm": 30.862796893024488,
      "learning_rate": 1.1014462809917355e-06,
      "logits/chosen": 0.0123291015625,
      "logits/rejected": -0.119140625,
      "logps/chosen": -392.0,
      "logps/rejected": -360.0,
      "loss": 0.3721,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.4375,
      "step": 668
    },
    {
      "epoch": 0.8208588957055215,
      "grad_norm": 35.17919160040694,
      "learning_rate": 1.1008264462809919e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -340.0,
      "logps/rejected": -338.0,
      "loss": 0.3167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6953125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.40625,
      "step": 669
    },
    {
      "epoch": 0.8220858895705522,
      "grad_norm": 25.88170844832394,
      "learning_rate": 1.100206611570248e-06,
      "logits/chosen": 0.17578125,
      "logits/rejected": 0.0250244140625,
      "logps/chosen": -324.0,
      "logps/rejected": -302.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.78125,
      "step": 670
    },
    {
      "epoch": 0.8233128834355828,
      "grad_norm": 30.762328731871737,
      "learning_rate": 1.0995867768595042e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.12109375,
      "logps/chosen": -372.0,
      "logps/rejected": -332.0,
      "loss": 0.2995,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 7.5,
      "rewards/rejected": -5.78125,
      "step": 671
    },
    {
      "epoch": 0.8245398773006135,
      "grad_norm": 30.48600462690235,
      "learning_rate": 1.0989669421487603e-06,
      "logits/chosen": 0.054931640625,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -446.0,
      "logps/rejected": -332.0,
      "loss": 0.3199,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.015625,
      "rewards/margins": 7.71875,
      "rewards/rejected": -5.6875,
      "step": 672
    },
    {
      "epoch": 0.8257668711656442,
      "grad_norm": 32.03977574623459,
      "learning_rate": 1.0983471074380164e-06,
      "logits/chosen": 0.1416015625,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -390.0,
      "logps/rejected": -328.0,
      "loss": 0.3096,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.109375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.125,
      "step": 673
    },
    {
      "epoch": 0.8269938650306748,
      "grad_norm": 32.93059543816188,
      "learning_rate": 1.0977272727272728e-06,
      "logits/chosen": 0.162109375,
      "logits/rejected": 0.00286865234375,
      "logps/chosen": -358.0,
      "logps/rejected": -330.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.53125,
      "step": 674
    },
    {
      "epoch": 0.8282208588957055,
      "grad_norm": 34.971531039669564,
      "learning_rate": 1.097107438016529e-06,
      "logits/chosen": 0.08203125,
      "logits/rejected": -0.130859375,
      "logps/chosen": -426.0,
      "logps/rejected": -368.0,
      "loss": 0.4407,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.3828125,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.0,
      "step": 675
    },
    {
      "epoch": 0.8294478527607362,
      "grad_norm": 34.2224078373622,
      "learning_rate": 1.0964876033057853e-06,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -394.0,
      "logps/rejected": -352.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.375,
      "step": 676
    },
    {
      "epoch": 0.8306748466257668,
      "grad_norm": 22.049121537748125,
      "learning_rate": 1.0958677685950414e-06,
      "logits/chosen": 0.09423828125,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -338.0,
      "logps/rejected": -316.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2099609375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.96875,
      "step": 677
    },
    {
      "epoch": 0.8319018404907975,
      "grad_norm": 31.042587162348006,
      "learning_rate": 1.0952479338842976e-06,
      "logits/chosen": 0.07421875,
      "logits/rejected": -0.1376953125,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4105,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.53125,
      "step": 678
    },
    {
      "epoch": 0.8331288343558282,
      "grad_norm": 32.125565947605324,
      "learning_rate": 1.0946280991735537e-06,
      "logits/chosen": 0.1689453125,
      "logits/rejected": 0.06201171875,
      "logps/chosen": -344.0,
      "logps/rejected": -328.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.0625,
      "step": 679
    },
    {
      "epoch": 0.8343558282208589,
      "grad_norm": 30.54691503767555,
      "learning_rate": 1.0940082644628098e-06,
      "logits/chosen": 0.025390625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -454.0,
      "logps/rejected": -374.0,
      "loss": 0.3146,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -6.875,
      "step": 680
    },
    {
      "epoch": 0.8355828220858895,
      "grad_norm": 25.35697096492706,
      "learning_rate": 1.0933884297520662e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.1796875,
      "logps/chosen": -348.0,
      "logps/rejected": -338.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.72265625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.71875,
      "step": 681
    },
    {
      "epoch": 0.8368098159509203,
      "grad_norm": 29.372252652247152,
      "learning_rate": 1.0927685950413223e-06,
      "logits/chosen": -0.000934600830078125,
      "logits/rejected": -0.115234375,
      "logps/chosen": -352.0,
      "logps/rejected": -334.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5546875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.90625,
      "step": 682
    },
    {
      "epoch": 0.838036809815951,
      "grad_norm": 28.48014927993986,
      "learning_rate": 1.0921487603305785e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.1328125,
      "logps/chosen": -374.0,
      "logps/rejected": -330.0,
      "loss": 0.3192,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.67578125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.375,
      "step": 683
    },
    {
      "epoch": 0.8392638036809816,
      "grad_norm": 30.9956365019602,
      "learning_rate": 1.0915289256198348e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 0.08544921875,
      "logps/chosen": -322.0,
      "logps/rejected": -352.0,
      "loss": 0.3553,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -7.03125,
      "step": 684
    },
    {
      "epoch": 0.8404907975460123,
      "grad_norm": 31.024167516184594,
      "learning_rate": 1.090909090909091e-06,
      "logits/chosen": 0.040283203125,
      "logits/rejected": -0.12255859375,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.3423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 8.625,
      "rewards/rejected": -8.25,
      "step": 685
    },
    {
      "epoch": 0.841717791411043,
      "grad_norm": 25.880225574187453,
      "learning_rate": 1.0902892561983471e-06,
      "logits/chosen": 0.031982421875,
      "logits/rejected": -0.095703125,
      "logps/chosen": -400.0,
      "logps/rejected": -348.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.65625,
      "step": 686
    },
    {
      "epoch": 0.8429447852760736,
      "grad_norm": 28.80339614616542,
      "learning_rate": 1.0896694214876035e-06,
      "logits/chosen": 0.03759765625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -380.0,
      "logps/rejected": -364.0,
      "loss": 0.2973,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.484375,
      "rewards/margins": 7.90625,
      "rewards/rejected": -7.40625,
      "step": 687
    },
    {
      "epoch": 0.8441717791411043,
      "grad_norm": 30.11912577456189,
      "learning_rate": 1.0890495867768594e-06,
      "logits/chosen": -0.000240325927734375,
      "logits/rejected": -0.158203125,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.327,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.5625,
      "rewards/margins": 7.5,
      "rewards/rejected": -8.0625,
      "step": 688
    },
    {
      "epoch": 0.845398773006135,
      "grad_norm": 30.49529821286284,
      "learning_rate": 1.0884297520661158e-06,
      "logits/chosen": 0.027099609375,
      "logits/rejected": -0.058837890625,
      "logps/chosen": -382.0,
      "logps/rejected": -348.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.40625,
      "step": 689
    },
    {
      "epoch": 0.8466257668711656,
      "grad_norm": 41.86773593768432,
      "learning_rate": 1.0878099173553719e-06,
      "logits/chosen": 0.171875,
      "logits/rejected": -0.0174560546875,
      "logps/chosen": -414.0,
      "logps/rejected": -370.0,
      "loss": 0.5772,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.21875,
      "step": 690
    },
    {
      "epoch": 0.8478527607361963,
      "grad_norm": 27.177968327720407,
      "learning_rate": 1.087190082644628e-06,
      "logits/chosen": -0.162109375,
      "logits/rejected": -0.2734375,
      "logps/chosen": -366.0,
      "logps/rejected": -368.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.90625,
      "step": 691
    },
    {
      "epoch": 0.849079754601227,
      "grad_norm": 38.42433716881971,
      "learning_rate": 1.0865702479338844e-06,
      "logits/chosen": 0.0179443359375,
      "logits/rejected": -0.142578125,
      "logps/chosen": -416.0,
      "logps/rejected": -358.0,
      "loss": 0.3292,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.8203125,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.53125,
      "step": 692
    },
    {
      "epoch": 0.8503067484662576,
      "grad_norm": 31.708672912333903,
      "learning_rate": 1.0859504132231405e-06,
      "logits/chosen": -0.00286865234375,
      "logits/rejected": -0.1904296875,
      "logps/chosen": -452.0,
      "logps/rejected": -386.0,
      "loss": 0.3787,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.0625,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.8125,
      "step": 693
    },
    {
      "epoch": 0.8515337423312883,
      "grad_norm": 39.65580173728978,
      "learning_rate": 1.0853305785123967e-06,
      "logits/chosen": 0.024658203125,
      "logits/rejected": -0.162109375,
      "logps/chosen": -384.0,
      "logps/rejected": -352.0,
      "loss": 0.5333,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.671875,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.75,
      "step": 694
    },
    {
      "epoch": 0.852760736196319,
      "grad_norm": 38.14890644660088,
      "learning_rate": 1.084710743801653e-06,
      "logits/chosen": 0.06103515625,
      "logits/rejected": -0.0179443359375,
      "logps/chosen": -338.0,
      "logps/rejected": -322.0,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4609375,
      "rewards/margins": 5.375,
      "rewards/rejected": -5.8125,
      "step": 695
    },
    {
      "epoch": 0.8539877300613496,
      "grad_norm": 32.915799758521196,
      "learning_rate": 1.0840909090909092e-06,
      "logits/chosen": 0.0947265625,
      "logits/rejected": -0.08154296875,
      "logps/chosen": -390.0,
      "logps/rejected": -336.0,
      "loss": 0.3241,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.78125,
      "step": 696
    },
    {
      "epoch": 0.8552147239263803,
      "grad_norm": 35.9913417584187,
      "learning_rate": 1.0834710743801653e-06,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -398.0,
      "logps/rejected": -384.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.9375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.0625,
      "step": 697
    },
    {
      "epoch": 0.8564417177914111,
      "grad_norm": 31.064185654999054,
      "learning_rate": 1.0828512396694214e-06,
      "logits/chosen": 0.08984375,
      "logits/rejected": -0.027587890625,
      "logps/chosen": -404.0,
      "logps/rejected": -366.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.392578125,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.09375,
      "step": 698
    },
    {
      "epoch": 0.8576687116564418,
      "grad_norm": 26.06263909227069,
      "learning_rate": 1.0822314049586776e-06,
      "logits/chosen": 0.0260009765625,
      "logits/rejected": -0.053955078125,
      "logps/chosen": -346.0,
      "logps/rejected": -340.0,
      "loss": 0.2587,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.228515625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.6875,
      "step": 699
    },
    {
      "epoch": 0.8588957055214724,
      "grad_norm": 33.7346004258113,
      "learning_rate": 1.081611570247934e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.06005859375,
      "logps/chosen": -380.0,
      "logps/rejected": -342.0,
      "loss": 0.398,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.0072021484375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.28125,
      "step": 700
    },
    {
      "epoch": 0.8601226993865031,
      "grad_norm": 36.508608167851875,
      "learning_rate": 1.08099173553719e-06,
      "logits/chosen": 0.1630859375,
      "logits/rejected": 0.05810546875,
      "logps/chosen": -390.0,
      "logps/rejected": -342.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.267578125,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.8125,
      "step": 701
    },
    {
      "epoch": 0.8613496932515338,
      "grad_norm": 29.752428717784724,
      "learning_rate": 1.0803719008264462e-06,
      "logits/chosen": -0.005767822265625,
      "logits/rejected": -0.30078125,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2347,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.91015625,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.09375,
      "step": 702
    },
    {
      "epoch": 0.8625766871165644,
      "grad_norm": 28.46276315635888,
      "learning_rate": 1.0797520661157026e-06,
      "logits/chosen": 0.1513671875,
      "logits/rejected": 0.0250244140625,
      "logps/chosen": -424.0,
      "logps/rejected": -358.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1396484375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.375,
      "step": 703
    },
    {
      "epoch": 0.8638036809815951,
      "grad_norm": 28.331719307949253,
      "learning_rate": 1.0791322314049587e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -332.0,
      "logps/rejected": -316.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.07861328125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.875,
      "step": 704
    },
    {
      "epoch": 0.8650306748466258,
      "grad_norm": 33.73975373967079,
      "learning_rate": 1.078512396694215e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -424.0,
      "logps/rejected": -376.0,
      "loss": 0.3238,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.25,
      "step": 705
    },
    {
      "epoch": 0.8662576687116564,
      "grad_norm": 32.06391249106975,
      "learning_rate": 1.077892561983471e-06,
      "logits/chosen": 0.01544189453125,
      "logits/rejected": -0.1796875,
      "logps/chosen": -410.0,
      "logps/rejected": -316.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.6875,
      "step": 706
    },
    {
      "epoch": 0.8674846625766871,
      "grad_norm": 32.50030520028076,
      "learning_rate": 1.0772727272727271e-06,
      "logits/chosen": -0.00185394287109375,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -394.0,
      "logps/rejected": -326.0,
      "loss": 0.3089,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.0947265625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -7.375,
      "step": 707
    },
    {
      "epoch": 0.8687116564417178,
      "grad_norm": 27.35860386441147,
      "learning_rate": 1.0766528925619835e-06,
      "logits/chosen": 0.0245361328125,
      "logits/rejected": -0.193359375,
      "logps/chosen": -380.0,
      "logps/rejected": -328.0,
      "loss": 0.2856,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.65234375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.625,
      "step": 708
    },
    {
      "epoch": 0.8699386503067484,
      "grad_norm": 22.967542887627133,
      "learning_rate": 1.0760330578512396e-06,
      "logits/chosen": 0.11376953125,
      "logits/rejected": -0.031982421875,
      "logps/chosen": -444.0,
      "logps/rejected": -360.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.46875,
      "step": 709
    },
    {
      "epoch": 0.8711656441717791,
      "grad_norm": 30.012370581713046,
      "learning_rate": 1.075413223140496e-06,
      "logits/chosen": 0.1845703125,
      "logits/rejected": 0.06103515625,
      "logps/chosen": -372.0,
      "logps/rejected": -330.0,
      "loss": 0.3445,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1064453125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.375,
      "step": 710
    },
    {
      "epoch": 0.8723926380368098,
      "grad_norm": 26.59602404250697,
      "learning_rate": 1.0747933884297521e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": -0.039306640625,
      "logps/chosen": -390.0,
      "logps/rejected": -332.0,
      "loss": 0.3562,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5703125,
      "rewards/margins": 7.0,
      "rewards/rejected": -6.4375,
      "step": 711
    },
    {
      "epoch": 0.8736196319018404,
      "grad_norm": 34.082753455253695,
      "learning_rate": 1.0741735537190083e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.00482177734375,
      "logps/chosen": -410.0,
      "logps/rejected": -356.0,
      "loss": 0.4269,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.328125,
      "rewards/margins": 7.03125,
      "rewards/rejected": -5.71875,
      "step": 712
    },
    {
      "epoch": 0.8748466257668711,
      "grad_norm": 24.8779582799026,
      "learning_rate": 1.0735537190082646e-06,
      "logits/chosen": 0.05810546875,
      "logits/rejected": -0.1484375,
      "logps/chosen": -386.0,
      "logps/rejected": -336.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.65625,
      "step": 713
    },
    {
      "epoch": 0.8760736196319019,
      "grad_norm": 27.823114674287325,
      "learning_rate": 1.0729338842975208e-06,
      "logits/chosen": 0.181640625,
      "logits/rejected": -0.038818359375,
      "logps/chosen": -370.0,
      "logps/rejected": -328.0,
      "loss": 0.3057,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.0,
      "step": 714
    },
    {
      "epoch": 0.8773006134969326,
      "grad_norm": 29.027125190845624,
      "learning_rate": 1.072314049586777e-06,
      "logits/chosen": -0.06494140625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.3391,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.015625,
      "rewards/margins": 7.75,
      "rewards/rejected": -6.71875,
      "step": 715
    },
    {
      "epoch": 0.8785276073619632,
      "grad_norm": 26.494530473617345,
      "learning_rate": 1.071694214876033e-06,
      "logits/chosen": -0.018798828125,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -384.0,
      "logps/rejected": -324.0,
      "loss": 0.3073,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.65625,
      "step": 716
    },
    {
      "epoch": 0.8797546012269939,
      "grad_norm": 29.02718718589355,
      "learning_rate": 1.0710743801652892e-06,
      "logits/chosen": 0.07470703125,
      "logits/rejected": -0.08056640625,
      "logps/chosen": -344.0,
      "logps/rejected": -284.0,
      "loss": 0.3344,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.6875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.21875,
      "step": 717
    },
    {
      "epoch": 0.8809815950920246,
      "grad_norm": 23.193149611746794,
      "learning_rate": 1.0704545454545455e-06,
      "logits/chosen": -0.034912109375,
      "logits/rejected": -0.1640625,
      "logps/chosen": -402.0,
      "logps/rejected": -356.0,
      "loss": 0.2308,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.25,
      "step": 718
    },
    {
      "epoch": 0.8822085889570552,
      "grad_norm": 36.13316553444024,
      "learning_rate": 1.0698347107438017e-06,
      "logits/chosen": -0.07763671875,
      "logits/rejected": -0.19921875,
      "logps/chosen": -410.0,
      "logps/rejected": -368.0,
      "loss": 0.3568,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.8046875,
      "rewards/margins": 8.0,
      "rewards/rejected": -6.1875,
      "step": 719
    },
    {
      "epoch": 0.8834355828220859,
      "grad_norm": 25.470910809931194,
      "learning_rate": 1.0692148760330578e-06,
      "logits/chosen": -0.07421875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -412.0,
      "logps/rejected": -350.0,
      "loss": 0.2706,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 7.9375,
      "rewards/rejected": -6.03125,
      "step": 720
    },
    {
      "epoch": 0.8846625766871166,
      "grad_norm": 24.45362594207483,
      "learning_rate": 1.0685950413223142e-06,
      "logits/chosen": -0.0810546875,
      "logits/rejected": -0.337890625,
      "logps/chosen": -414.0,
      "logps/rejected": -364.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 1.5625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.25,
      "step": 721
    },
    {
      "epoch": 0.8858895705521472,
      "grad_norm": 26.653798150160988,
      "learning_rate": 1.0679752066115703e-06,
      "logits/chosen": -0.049072265625,
      "logits/rejected": -0.1376953125,
      "logps/chosen": -380.0,
      "logps/rejected": -358.0,
      "loss": 0.2495,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.609375,
      "rewards/margins": 7.875,
      "rewards/rejected": -6.28125,
      "step": 722
    },
    {
      "epoch": 0.8871165644171779,
      "grad_norm": 29.259615066962372,
      "learning_rate": 1.0673553719008265e-06,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.3287,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.5,
      "step": 723
    },
    {
      "epoch": 0.8883435582822086,
      "grad_norm": 30.45569089414315,
      "learning_rate": 1.0667355371900826e-06,
      "logits/chosen": -0.0308837890625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.3219,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -6.5,
      "step": 724
    },
    {
      "epoch": 0.8895705521472392,
      "grad_norm": 30.8228410953136,
      "learning_rate": 1.0661157024793387e-06,
      "logits/chosen": 0.12255859375,
      "logits/rejected": -0.05419921875,
      "logps/chosen": -404.0,
      "logps/rejected": -340.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.375,
      "step": 725
    },
    {
      "epoch": 0.8907975460122699,
      "grad_norm": 32.44931458712784,
      "learning_rate": 1.065495867768595e-06,
      "logits/chosen": -0.030029296875,
      "logits/rejected": -0.154296875,
      "logps/chosen": -340.0,
      "logps/rejected": -302.0,
      "loss": 0.2852,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5703125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.96875,
      "step": 726
    },
    {
      "epoch": 0.8920245398773006,
      "grad_norm": 30.00382649809932,
      "learning_rate": 1.0648760330578512e-06,
      "logits/chosen": -0.0013427734375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.2567,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.84765625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.71875,
      "step": 727
    },
    {
      "epoch": 0.8932515337423312,
      "grad_norm": 32.084882438300085,
      "learning_rate": 1.0642561983471074e-06,
      "logits/chosen": -0.0244140625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -406.0,
      "logps/rejected": -384.0,
      "loss": 0.3182,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.234375,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.625,
      "step": 728
    },
    {
      "epoch": 0.894478527607362,
      "grad_norm": 24.427532348622456,
      "learning_rate": 1.0636363636363637e-06,
      "logits/chosen": -0.040771484375,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -396.0,
      "logps/rejected": -372.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.9765625,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.90625,
      "step": 729
    },
    {
      "epoch": 0.8957055214723927,
      "grad_norm": 24.535854490724105,
      "learning_rate": 1.0630165289256199e-06,
      "logits/chosen": -0.103515625,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.2127,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 1.515625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.3125,
      "step": 730
    },
    {
      "epoch": 0.8969325153374234,
      "grad_norm": 34.14852950381922,
      "learning_rate": 1.062396694214876e-06,
      "logits/chosen": 0.1171875,
      "logits/rejected": 0.002349853515625,
      "logps/chosen": -438.0,
      "logps/rejected": -366.0,
      "loss": 0.3465,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.5625,
      "step": 731
    },
    {
      "epoch": 0.898159509202454,
      "grad_norm": 34.434393814707754,
      "learning_rate": 1.0617768595041324e-06,
      "logits/chosen": -0.058837890625,
      "logits/rejected": -0.2099609375,
      "logps/chosen": -380.0,
      "logps/rejected": -372.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.6875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.03125,
      "step": 732
    },
    {
      "epoch": 0.8993865030674847,
      "grad_norm": 28.68332287996042,
      "learning_rate": 1.0611570247933883e-06,
      "logits/chosen": 0.0673828125,
      "logits/rejected": -0.083984375,
      "logps/chosen": -366.0,
      "logps/rejected": -362.0,
      "loss": 0.2815,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1015625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.5625,
      "step": 733
    },
    {
      "epoch": 0.9006134969325154,
      "grad_norm": 32.670441266137765,
      "learning_rate": 1.0605371900826447e-06,
      "logits/chosen": 0.06884765625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -384.0,
      "logps/rejected": -354.0,
      "loss": 0.3319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.90625,
      "step": 734
    },
    {
      "epoch": 0.901840490797546,
      "grad_norm": 28.766622871975752,
      "learning_rate": 1.0599173553719008e-06,
      "logits/chosen": -0.037841796875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -434.0,
      "logps/rejected": -390.0,
      "loss": 0.2447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2490234375,
      "rewards/margins": 8.875,
      "rewards/rejected": -8.625,
      "step": 735
    },
    {
      "epoch": 0.9030674846625767,
      "grad_norm": 31.10013526298491,
      "learning_rate": 1.059297520661157e-06,
      "logits/chosen": 0.1533203125,
      "logits/rejected": 0.029541015625,
      "logps/chosen": -362.0,
      "logps/rejected": -340.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.5625,
      "step": 736
    },
    {
      "epoch": 0.9042944785276074,
      "grad_norm": 38.03816635846729,
      "learning_rate": 1.0586776859504133e-06,
      "logits/chosen": 0.04541015625,
      "logits/rejected": -0.051513671875,
      "logps/chosen": -360.0,
      "logps/rejected": -348.0,
      "loss": 0.4641,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.478515625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.5625,
      "step": 737
    },
    {
      "epoch": 0.905521472392638,
      "grad_norm": 32.716309617319716,
      "learning_rate": 1.0580578512396694e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -450.0,
      "logps/rejected": -364.0,
      "loss": 0.3437,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.60546875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.59375,
      "step": 738
    },
    {
      "epoch": 0.9067484662576687,
      "grad_norm": 29.627927045089628,
      "learning_rate": 1.0574380165289258e-06,
      "logits/chosen": 0.060302734375,
      "logits/rejected": -0.0654296875,
      "logps/chosen": -346.0,
      "logps/rejected": -340.0,
      "loss": 0.2827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06640625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -7.25,
      "step": 739
    },
    {
      "epoch": 0.9079754601226994,
      "grad_norm": 29.046321525927297,
      "learning_rate": 1.056818181818182e-06,
      "logits/chosen": -0.0308837890625,
      "logits/rejected": -0.18359375,
      "logps/chosen": -344.0,
      "logps/rejected": -326.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.02392578125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.53125,
      "step": 740
    },
    {
      "epoch": 0.90920245398773,
      "grad_norm": 29.826427625052336,
      "learning_rate": 1.056198347107438e-06,
      "logits/chosen": -0.00982666015625,
      "logits/rejected": -0.162109375,
      "logps/chosen": -356.0,
      "logps/rejected": -328.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.8359375,
      "rewards/margins": 8.75,
      "rewards/rejected": -7.875,
      "step": 741
    },
    {
      "epoch": 0.9104294478527607,
      "grad_norm": 28.054610141633944,
      "learning_rate": 1.0555785123966942e-06,
      "logits/chosen": -0.033935546875,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -368.0,
      "logps/rejected": -372.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.6640625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -7.90625,
      "step": 742
    },
    {
      "epoch": 0.9116564417177914,
      "grad_norm": 35.487753263657986,
      "learning_rate": 1.0549586776859503e-06,
      "logits/chosen": 0.01055908203125,
      "logits/rejected": -0.040771484375,
      "logps/chosen": -424.0,
      "logps/rejected": -392.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.2265625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.0,
      "step": 743
    },
    {
      "epoch": 0.912883435582822,
      "grad_norm": 28.496366117513464,
      "learning_rate": 1.0543388429752067e-06,
      "logits/chosen": 0.035400390625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.2775,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 8.6875,
      "rewards/rejected": -7.71875,
      "step": 744
    },
    {
      "epoch": 0.9141104294478528,
      "grad_norm": 30.005979043165016,
      "learning_rate": 1.0537190082644628e-06,
      "logits/chosen": 0.08642578125,
      "logits/rejected": -0.0003509521484375,
      "logps/chosen": -346.0,
      "logps/rejected": -318.0,
      "loss": 0.3381,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.1044921875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.75,
      "step": 745
    },
    {
      "epoch": 0.9153374233128835,
      "grad_norm": 36.365949373279655,
      "learning_rate": 1.053099173553719e-06,
      "logits/chosen": 0.09033203125,
      "logits/rejected": -0.07958984375,
      "logps/chosen": -408.0,
      "logps/rejected": -356.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -7.0,
      "step": 746
    },
    {
      "epoch": 0.9165644171779141,
      "grad_norm": 30.46892734256714,
      "learning_rate": 1.0524793388429753e-06,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.21875,
      "logps/chosen": -342.0,
      "logps/rejected": -344.0,
      "loss": 0.2905,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.0234375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.75,
      "step": 747
    },
    {
      "epoch": 0.9177914110429448,
      "grad_norm": 37.226894144134455,
      "learning_rate": 1.0518595041322315e-06,
      "logits/chosen": 0.125,
      "logits/rejected": 0.01458740234375,
      "logps/chosen": -380.0,
      "logps/rejected": -354.0,
      "loss": 0.4992,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.78125,
      "step": 748
    },
    {
      "epoch": 0.9190184049079755,
      "grad_norm": 34.24194812875213,
      "learning_rate": 1.0512396694214876e-06,
      "logits/chosen": -0.04541015625,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -424.0,
      "logps/rejected": -358.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.34375,
      "step": 749
    },
    {
      "epoch": 0.9202453987730062,
      "grad_norm": 35.848596312284045,
      "learning_rate": 1.050619834710744e-06,
      "logits/chosen": 0.01190185546875,
      "logits/rejected": -0.119140625,
      "logps/chosen": -412.0,
      "logps/rejected": -348.0,
      "loss": 0.4472,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.09326171875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.28125,
      "step": 750
    },
    {
      "epoch": 0.9214723926380368,
      "grad_norm": 36.49684940562663,
      "learning_rate": 1.05e-06,
      "logits/chosen": -0.0064697265625,
      "logits/rejected": -0.169921875,
      "logps/chosen": -428.0,
      "logps/rejected": -366.0,
      "loss": 0.3626,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.8125,
      "step": 751
    },
    {
      "epoch": 0.9226993865030675,
      "grad_norm": 31.9902592360704,
      "learning_rate": 1.0493801652892563e-06,
      "logits/chosen": -0.028564453125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -424.0,
      "logps/rejected": -368.0,
      "loss": 0.3427,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.640625,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.71875,
      "step": 752
    },
    {
      "epoch": 0.9239263803680982,
      "grad_norm": 30.210419690529548,
      "learning_rate": 1.0487603305785124e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": 0.01361083984375,
      "logps/chosen": -408.0,
      "logps/rejected": -362.0,
      "loss": 0.3032,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.25,
      "step": 753
    },
    {
      "epoch": 0.9251533742331288,
      "grad_norm": 33.70924536763799,
      "learning_rate": 1.0481404958677685e-06,
      "logits/chosen": 0.06640625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -394.0,
      "logps/rejected": -348.0,
      "loss": 0.4405,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.21875,
      "step": 754
    },
    {
      "epoch": 0.9263803680981595,
      "grad_norm": 28.923057351049156,
      "learning_rate": 1.0475206611570249e-06,
      "logits/chosen": 0.01611328125,
      "logits/rejected": -0.1064453125,
      "logps/chosen": -388.0,
      "logps/rejected": -366.0,
      "loss": 0.313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.291015625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.5,
      "step": 755
    },
    {
      "epoch": 0.9276073619631902,
      "grad_norm": 26.956563960777117,
      "learning_rate": 1.046900826446281e-06,
      "logits/chosen": 0.115234375,
      "logits/rejected": 0.02978515625,
      "logps/chosen": -342.0,
      "logps/rejected": -326.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.640625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.46875,
      "step": 756
    },
    {
      "epoch": 0.9288343558282208,
      "grad_norm": 25.54044780724446,
      "learning_rate": 1.0462809917355372e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": -0.0595703125,
      "logps/chosen": -404.0,
      "logps/rejected": -344.0,
      "loss": 0.27,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7578125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.3125,
      "step": 757
    },
    {
      "epoch": 0.9300613496932515,
      "grad_norm": 35.658646153584634,
      "learning_rate": 1.0456611570247935e-06,
      "logits/chosen": 0.08740234375,
      "logits/rejected": -0.080078125,
      "logps/chosen": -376.0,
      "logps/rejected": -320.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.55859375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.28125,
      "step": 758
    },
    {
      "epoch": 0.9312883435582822,
      "grad_norm": 29.003566311314795,
      "learning_rate": 1.0450413223140497e-06,
      "logits/chosen": 0.1123046875,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -396.0,
      "logps/rejected": -340.0,
      "loss": 0.2888,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16015625,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.78125,
      "step": 759
    },
    {
      "epoch": 0.9325153374233128,
      "grad_norm": 25.36330419002366,
      "learning_rate": 1.0444214876033058e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.35546875,
      "logps/chosen": -442.0,
      "logps/rejected": -366.0,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 8.625,
      "rewards/rejected": -7.4375,
      "step": 760
    },
    {
      "epoch": 0.9337423312883436,
      "grad_norm": 31.72216949313869,
      "learning_rate": 1.043801652892562e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.0277099609375,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.3984375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.8125,
      "step": 761
    },
    {
      "epoch": 0.9349693251533743,
      "grad_norm": 31.01314079949462,
      "learning_rate": 1.043181818181818e-06,
      "logits/chosen": 0.1591796875,
      "logits/rejected": -0.020751953125,
      "logps/chosen": -428.0,
      "logps/rejected": -364.0,
      "loss": 0.3149,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.2373046875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.28125,
      "step": 762
    },
    {
      "epoch": 0.9361963190184049,
      "grad_norm": 28.119069224585317,
      "learning_rate": 1.0425619834710744e-06,
      "logits/chosen": 0.0986328125,
      "logits/rejected": 0.0164794921875,
      "logps/chosen": -400.0,
      "logps/rejected": -364.0,
      "loss": 0.3398,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.125,
      "step": 763
    },
    {
      "epoch": 0.9374233128834356,
      "grad_norm": 26.42805050753053,
      "learning_rate": 1.0419421487603306e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -330.0,
      "logps/rejected": -316.0,
      "loss": 0.2955,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.494140625,
      "rewards/margins": 7.0,
      "rewards/rejected": -7.46875,
      "step": 764
    },
    {
      "epoch": 0.9386503067484663,
      "grad_norm": 35.659703021872964,
      "learning_rate": 1.041322314049587e-06,
      "logits/chosen": 0.1298828125,
      "logits/rejected": 0.005828857421875,
      "logps/chosen": -404.0,
      "logps/rejected": -320.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.2041015625,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.34375,
      "step": 765
    },
    {
      "epoch": 0.939877300613497,
      "grad_norm": 27.867245312963597,
      "learning_rate": 1.040702479338843e-06,
      "logits/chosen": 0.04345703125,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -446.0,
      "logps/rejected": -340.0,
      "loss": 0.3188,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.515625,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.0,
      "step": 766
    },
    {
      "epoch": 0.9411042944785276,
      "grad_norm": 33.0423901915957,
      "learning_rate": 1.0400826446280992e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.10498046875,
      "logps/chosen": -386.0,
      "logps/rejected": -338.0,
      "loss": 0.355,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1689453125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.375,
      "step": 767
    },
    {
      "epoch": 0.9423312883435583,
      "grad_norm": 31.83881144027911,
      "learning_rate": 1.0394628099173554e-06,
      "logits/chosen": 0.003570556640625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -410.0,
      "logps/rejected": -384.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.96875,
      "step": 768
    },
    {
      "epoch": 0.943558282208589,
      "grad_norm": 24.722102647279602,
      "learning_rate": 1.0388429752066115e-06,
      "logits/chosen": 0.00079345703125,
      "logits/rejected": -0.138671875,
      "logps/chosen": -378.0,
      "logps/rejected": -344.0,
      "loss": 0.2777,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.59375,
      "step": 769
    },
    {
      "epoch": 0.9447852760736196,
      "grad_norm": 33.3608331180557,
      "learning_rate": 1.0382231404958679e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.265625,
      "logps/chosen": -416.0,
      "logps/rejected": -372.0,
      "loss": 0.321,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.59375,
      "step": 770
    },
    {
      "epoch": 0.9460122699386503,
      "grad_norm": 36.53871156772584,
      "learning_rate": 1.037603305785124e-06,
      "logits/chosen": -0.00537109375,
      "logits/rejected": -0.01507568359375,
      "logps/chosen": -430.0,
      "logps/rejected": -360.0,
      "loss": 0.5217,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.875,
      "step": 771
    },
    {
      "epoch": 0.947239263803681,
      "grad_norm": 30.200058873190507,
      "learning_rate": 1.0369834710743801e-06,
      "logits/chosen": 0.1328125,
      "logits/rejected": -0.0869140625,
      "logps/chosen": -424.0,
      "logps/rejected": -362.0,
      "loss": 0.3536,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.38671875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.90625,
      "step": 772
    },
    {
      "epoch": 0.9484662576687116,
      "grad_norm": 31.95788929467944,
      "learning_rate": 1.0363636363636365e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": -0.037841796875,
      "logps/chosen": -374.0,
      "logps/rejected": -324.0,
      "loss": 0.3406,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.65234375,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.15625,
      "step": 773
    },
    {
      "epoch": 0.9496932515337423,
      "grad_norm": 34.11224513726983,
      "learning_rate": 1.0357438016528926e-06,
      "logits/chosen": 0.1396484375,
      "logits/rejected": -0.0037078857421875,
      "logps/chosen": -362.0,
      "logps/rejected": -326.0,
      "loss": 0.4295,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -1.359375,
      "rewards/margins": 5.1875,
      "rewards/rejected": -6.5625,
      "step": 774
    },
    {
      "epoch": 0.950920245398773,
      "grad_norm": 31.83399815065193,
      "learning_rate": 1.0351239669421488e-06,
      "logits/chosen": 0.2197265625,
      "logits/rejected": 0.04296875,
      "logps/chosen": -418.0,
      "logps/rejected": -362.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.32421875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.53125,
      "step": 775
    },
    {
      "epoch": 0.9521472392638037,
      "grad_norm": 29.278356141500073,
      "learning_rate": 1.0345041322314051e-06,
      "logits/chosen": -0.01409912109375,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -414.0,
      "logps/rejected": -366.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0888671875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.75,
      "step": 776
    },
    {
      "epoch": 0.9533742331288344,
      "grad_norm": 33.3868265617977,
      "learning_rate": 1.033884297520661e-06,
      "logits/chosen": 0.0267333984375,
      "logits/rejected": -0.1015625,
      "logps/chosen": -408.0,
      "logps/rejected": -346.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.3125,
      "step": 777
    },
    {
      "epoch": 0.9546012269938651,
      "grad_norm": 30.60810394473529,
      "learning_rate": 1.0332644628099174e-06,
      "logits/chosen": 0.0703125,
      "logits/rejected": -0.130859375,
      "logps/chosen": -400.0,
      "logps/rejected": -328.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.74609375,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.75,
      "step": 778
    },
    {
      "epoch": 0.9558282208588957,
      "grad_norm": 31.819603506171298,
      "learning_rate": 1.0326446280991735e-06,
      "logits/chosen": 0.00994873046875,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -442.0,
      "logps/rejected": -362.0,
      "loss": 0.331,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.16796875,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.40625,
      "step": 779
    },
    {
      "epoch": 0.9570552147239264,
      "grad_norm": 33.50608635895235,
      "learning_rate": 1.0320247933884297e-06,
      "logits/chosen": 0.13671875,
      "logits/rejected": -0.037353515625,
      "logps/chosen": -386.0,
      "logps/rejected": -360.0,
      "loss": 0.4421,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -6.875,
      "step": 780
    },
    {
      "epoch": 0.9582822085889571,
      "grad_norm": 27.175413708110298,
      "learning_rate": 1.031404958677686e-06,
      "logits/chosen": -0.0087890625,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.2289,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 8.75,
      "rewards/rejected": -7.8125,
      "step": 781
    },
    {
      "epoch": 0.9595092024539877,
      "grad_norm": 27.961517528382746,
      "learning_rate": 1.0307851239669422e-06,
      "logits/chosen": 0.0341796875,
      "logits/rejected": -0.055419921875,
      "logps/chosen": -370.0,
      "logps/rejected": -354.0,
      "loss": 0.2727,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.0152587890625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.1875,
      "step": 782
    },
    {
      "epoch": 0.9607361963190184,
      "grad_norm": 36.53707624363368,
      "learning_rate": 1.0301652892561983e-06,
      "logits/chosen": 0.0115966796875,
      "logits/rejected": -0.12353515625,
      "logps/chosen": -420.0,
      "logps/rejected": -370.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.5,
      "step": 783
    },
    {
      "epoch": 0.9619631901840491,
      "grad_norm": 33.01812506650061,
      "learning_rate": 1.0295454545454547e-06,
      "logits/chosen": 0.09716796875,
      "logits/rejected": 0.056640625,
      "logps/chosen": -386.0,
      "logps/rejected": -354.0,
      "loss": 0.3887,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.69140625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -7.125,
      "step": 784
    },
    {
      "epoch": 0.9631901840490797,
      "grad_norm": 25.64548904648115,
      "learning_rate": 1.0289256198347108e-06,
      "logits/chosen": 0.126953125,
      "logits/rejected": -0.0400390625,
      "logps/chosen": -384.0,
      "logps/rejected": -324.0,
      "loss": 0.2131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0047607421875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -7.375,
      "step": 785
    },
    {
      "epoch": 0.9644171779141104,
      "grad_norm": 24.085836304747,
      "learning_rate": 1.028305785123967e-06,
      "logits/chosen": 0.11376953125,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -372.0,
      "logps/rejected": -334.0,
      "loss": 0.2197,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.8125,
      "step": 786
    },
    {
      "epoch": 0.9656441717791411,
      "grad_norm": 32.97798807698007,
      "learning_rate": 1.027685950413223e-06,
      "logits/chosen": 0.0269775390625,
      "logits/rejected": -0.08740234375,
      "logps/chosen": -412.0,
      "logps/rejected": -348.0,
      "loss": 0.3407,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.9375,
      "step": 787
    },
    {
      "epoch": 0.9668711656441717,
      "grad_norm": 24.19044914351049,
      "learning_rate": 1.0270661157024792e-06,
      "logits/chosen": -0.00860595703125,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -390.0,
      "logps/rejected": -366.0,
      "loss": 0.2123,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.486328125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.6875,
      "step": 788
    },
    {
      "epoch": 0.9680981595092024,
      "grad_norm": 30.061433994827276,
      "learning_rate": 1.0264462809917356e-06,
      "logits/chosen": 0.12060546875,
      "logits/rejected": -0.06396484375,
      "logps/chosen": -384.0,
      "logps/rejected": -332.0,
      "loss": 0.383,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.52734375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.25,
      "step": 789
    },
    {
      "epoch": 0.9693251533742331,
      "grad_norm": 27.089223909733843,
      "learning_rate": 1.0258264462809917e-06,
      "logits/chosen": -0.028564453125,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -388.0,
      "logps/rejected": -330.0,
      "loss": 0.1804,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -7.46875,
      "step": 790
    },
    {
      "epoch": 0.9705521472392638,
      "grad_norm": 34.35477577844613,
      "learning_rate": 1.0252066115702479e-06,
      "logits/chosen": -0.0303955078125,
      "logits/rejected": -0.23046875,
      "logps/chosen": -398.0,
      "logps/rejected": -346.0,
      "loss": 0.3642,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.349609375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.90625,
      "step": 791
    },
    {
      "epoch": 0.9717791411042945,
      "grad_norm": 33.05641026053793,
      "learning_rate": 1.0245867768595042e-06,
      "logits/chosen": 0.0006256103515625,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -438.0,
      "logps/rejected": -374.0,
      "loss": 0.406,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.380859375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.4375,
      "step": 792
    },
    {
      "epoch": 0.9730061349693252,
      "grad_norm": 32.69405943160363,
      "learning_rate": 1.0239669421487604e-06,
      "logits/chosen": 0.0284423828125,
      "logits/rejected": -0.09765625,
      "logps/chosen": -392.0,
      "logps/rejected": -362.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.10546875,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.84375,
      "step": 793
    },
    {
      "epoch": 0.9742331288343559,
      "grad_norm": 33.74613547302303,
      "learning_rate": 1.0233471074380167e-06,
      "logits/chosen": 0.0830078125,
      "logits/rejected": -0.043701171875,
      "logps/chosen": -358.0,
      "logps/rejected": -336.0,
      "loss": 0.4877,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 6.59375,
      "rewards/rejected": -7.1875,
      "step": 794
    },
    {
      "epoch": 0.9754601226993865,
      "grad_norm": 22.619679414305867,
      "learning_rate": 1.0227272727272727e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.296875,
      "logps/chosen": -392.0,
      "logps/rejected": -372.0,
      "loss": 0.281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 9.25,
      "rewards/rejected": -8.4375,
      "step": 795
    },
    {
      "epoch": 0.9766871165644172,
      "grad_norm": 35.85933901249144,
      "learning_rate": 1.0221074380165288e-06,
      "logits/chosen": -0.0260009765625,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -410.0,
      "logps/rejected": -338.0,
      "loss": 0.3447,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 7.25,
      "rewards/rejected": -8.0625,
      "step": 796
    },
    {
      "epoch": 0.9779141104294479,
      "grad_norm": 24.484140263550707,
      "learning_rate": 1.0214876033057852e-06,
      "logits/chosen": 0.059814453125,
      "logits/rejected": -0.11962890625,
      "logps/chosen": -404.0,
      "logps/rejected": -324.0,
      "loss": 0.243,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.0966796875,
      "rewards/margins": 7.15625,
      "rewards/rejected": -7.25,
      "step": 797
    },
    {
      "epoch": 0.9791411042944785,
      "grad_norm": 35.333603529709684,
      "learning_rate": 1.0208677685950413e-06,
      "logits/chosen": 0.00506591796875,
      "logits/rejected": -0.173828125,
      "logps/chosen": -430.0,
      "logps/rejected": -378.0,
      "loss": 0.3546,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.1455078125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.625,
      "step": 798
    },
    {
      "epoch": 0.9803680981595092,
      "grad_norm": 32.17706869798208,
      "learning_rate": 1.0202479338842976e-06,
      "logits/chosen": -0.05029296875,
      "logits/rejected": -0.201171875,
      "logps/chosen": -400.0,
      "logps/rejected": -360.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.28125,
      "step": 799
    },
    {
      "epoch": 0.9815950920245399,
      "grad_norm": 32.44337864899576,
      "learning_rate": 1.0196280991735538e-06,
      "logits/chosen": -0.0224609375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -386.0,
      "logps/rejected": -344.0,
      "loss": 0.3141,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -7.375,
      "step": 800
    },
    {
      "epoch": 0.9828220858895705,
      "grad_norm": 37.380385495976945,
      "learning_rate": 1.01900826446281e-06,
      "logits/chosen": -0.024658203125,
      "logits/rejected": -0.17578125,
      "logps/chosen": -436.0,
      "logps/rejected": -382.0,
      "loss": 0.4546,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.0001010894775390625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.6875,
      "step": 801
    },
    {
      "epoch": 0.9840490797546012,
      "grad_norm": 35.755684948235334,
      "learning_rate": 1.0183884297520663e-06,
      "logits/chosen": -0.07421875,
      "logits/rejected": -0.21484375,
      "logps/chosen": -386.0,
      "logps/rejected": -348.0,
      "loss": 0.472,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1552734375,
      "rewards/margins": 7.0,
      "rewards/rejected": -7.15625,
      "step": 802
    },
    {
      "epoch": 0.9852760736196319,
      "grad_norm": 29.08898198089266,
      "learning_rate": 1.0177685950413224e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -398.0,
      "logps/rejected": -348.0,
      "loss": 0.3346,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 7.75,
      "rewards/rejected": -7.21875,
      "step": 803
    },
    {
      "epoch": 0.9865030674846625,
      "grad_norm": 30.37947277261302,
      "learning_rate": 1.0171487603305786e-06,
      "logits/chosen": 0.04150390625,
      "logits/rejected": -0.06689453125,
      "logps/chosen": -358.0,
      "logps/rejected": -346.0,
      "loss": 0.3638,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.392578125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.34375,
      "step": 804
    },
    {
      "epoch": 0.9877300613496932,
      "grad_norm": 34.04488703971709,
      "learning_rate": 1.0165289256198347e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -434.0,
      "logps/rejected": -394.0,
      "loss": 0.4569,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.375,
      "step": 805
    },
    {
      "epoch": 0.9889570552147239,
      "grad_norm": 35.48502129676359,
      "learning_rate": 1.0159090909090908e-06,
      "logits/chosen": 0.16015625,
      "logits/rejected": 0.03564453125,
      "logps/chosen": -412.0,
      "logps/rejected": -384.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.84375,
      "step": 806
    },
    {
      "epoch": 0.9901840490797545,
      "grad_norm": 33.61353577378301,
      "learning_rate": 1.0152892561983472e-06,
      "logits/chosen": 0.061767578125,
      "logits/rejected": -0.0908203125,
      "logps/chosen": -400.0,
      "logps/rejected": -350.0,
      "loss": 0.3144,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.484375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.75,
      "step": 807
    },
    {
      "epoch": 0.9914110429447853,
      "grad_norm": 31.091169744856,
      "learning_rate": 1.0146694214876033e-06,
      "logits/chosen": -0.083984375,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -364.0,
      "logps/rejected": -352.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.640625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -8.1875,
      "step": 808
    },
    {
      "epoch": 0.992638036809816,
      "grad_norm": 34.39338760048118,
      "learning_rate": 1.0140495867768595e-06,
      "logits/chosen": -0.00872802734375,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -438.0,
      "logps/rejected": -360.0,
      "loss": 0.319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -7.71875,
      "step": 809
    },
    {
      "epoch": 0.9938650306748467,
      "grad_norm": 26.950235054794184,
      "learning_rate": 1.0134297520661158e-06,
      "logits/chosen": -0.11572265625,
      "logits/rejected": -0.28515625,
      "logps/chosen": -446.0,
      "logps/rejected": -384.0,
      "loss": 0.2346,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.44140625,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.5,
      "step": 810
    },
    {
      "epoch": 0.9950920245398773,
      "grad_norm": 26.578554179921486,
      "learning_rate": 1.012809917355372e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.21484375,
      "logps/chosen": -394.0,
      "logps/rejected": -380.0,
      "loss": 0.3194,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -7.03125,
      "step": 811
    },
    {
      "epoch": 0.996319018404908,
      "grad_norm": 34.32887556980461,
      "learning_rate": 1.0121900826446281e-06,
      "logits/chosen": -0.05908203125,
      "logits/rejected": -0.125,
      "logps/chosen": -364.0,
      "logps/rejected": -340.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09619140625,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.71875,
      "step": 812
    },
    {
      "epoch": 0.9975460122699387,
      "grad_norm": 25.82868308838759,
      "learning_rate": 1.0115702479338843e-06,
      "logits/chosen": 0.01611328125,
      "logits/rejected": -0.1123046875,
      "logps/chosen": -342.0,
      "logps/rejected": -336.0,
      "loss": 0.2067,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.375,
      "step": 813
    },
    {
      "epoch": 0.9987730061349693,
      "grad_norm": 35.210336167923685,
      "learning_rate": 1.0109504132231404e-06,
      "logits/chosen": 0.054443359375,
      "logits/rejected": -0.0206298828125,
      "logps/chosen": -388.0,
      "logps/rejected": -370.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.80859375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.90625,
      "step": 814
    },
    {
      "epoch": 1.0,
      "grad_norm": 31.93988486773496,
      "learning_rate": 1.0103305785123968e-06,
      "logits/chosen": 0.1279296875,
      "logits/rejected": -0.061279296875,
      "logps/chosen": -388.0,
      "logps/rejected": -364.0,
      "loss": 0.2533,
      "rewards/accuracies": 0.8991597294807434,
      "rewards/chosen": -0.095703125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.625,
      "step": 815
    },
    {
      "epoch": 1.0012269938650307,
      "grad_norm": 5.842576996186414,
      "learning_rate": 1.009710743801653e-06,
      "logits/chosen": 0.0537109375,
      "logits/rejected": -0.08154296875,
      "logps/chosen": -376.0,
      "logps/rejected": -332.0,
      "loss": 0.034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 9.5,
      "rewards/rejected": -8.1875,
      "step": 816
    },
    {
      "epoch": 1.0024539877300613,
      "grad_norm": 8.73132250150435,
      "learning_rate": 1.009090909090909e-06,
      "logits/chosen": 0.03857421875,
      "logits/rejected": -0.107421875,
      "logps/chosen": -362.0,
      "logps/rejected": -334.0,
      "loss": 0.0486,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 8.8125,
      "rewards/rejected": -8.125,
      "step": 817
    },
    {
      "epoch": 1.003680981595092,
      "grad_norm": 4.472623669255691,
      "learning_rate": 1.0084710743801654e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": -0.1728515625,
      "logps/chosen": -364.0,
      "logps/rejected": -362.0,
      "loss": 0.0318,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -8.0625,
      "step": 818
    },
    {
      "epoch": 1.0049079754601227,
      "grad_norm": 10.555989313773281,
      "learning_rate": 1.0078512396694215e-06,
      "logits/chosen": 0.0576171875,
      "logits/rejected": -0.12109375,
      "logps/chosen": -390.0,
      "logps/rejected": -366.0,
      "loss": 0.043,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 9.875,
      "rewards/rejected": -8.4375,
      "step": 819
    },
    {
      "epoch": 1.0061349693251533,
      "grad_norm": 7.844244829439074,
      "learning_rate": 1.0072314049586777e-06,
      "logits/chosen": 0.09375,
      "logits/rejected": -0.02880859375,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.0346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6796875,
      "rewards/margins": 9.25,
      "rewards/rejected": -7.5625,
      "step": 820
    },
    {
      "epoch": 1.007361963190184,
      "grad_norm": 13.093642235831343,
      "learning_rate": 1.006611570247934e-06,
      "logits/chosen": -0.0054931640625,
      "logits/rejected": -0.126953125,
      "logps/chosen": -350.0,
      "logps/rejected": -342.0,
      "loss": 0.0919,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.578125,
      "rewards/margins": 9.5,
      "rewards/rejected": -7.90625,
      "step": 821
    },
    {
      "epoch": 1.0085889570552147,
      "grad_norm": 11.258353605754225,
      "learning_rate": 1.00599173553719e-06,
      "logits/chosen": 0.00653076171875,
      "logits/rejected": -0.126953125,
      "logps/chosen": -404.0,
      "logps/rejected": -370.0,
      "loss": 0.0581,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 1.6484375,
      "rewards/margins": 9.0,
      "rewards/rejected": -7.375,
      "step": 822
    },
    {
      "epoch": 1.0098159509202453,
      "grad_norm": 7.2063058089923135,
      "learning_rate": 1.0053719008264463e-06,
      "logits/chosen": 0.03466796875,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -364.0,
      "logps/rejected": -350.0,
      "loss": 0.0354,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.84375,
      "rewards/margins": 9.4375,
      "rewards/rejected": -7.5625,
      "step": 823
    },
    {
      "epoch": 1.011042944785276,
      "grad_norm": 10.387457112253443,
      "learning_rate": 1.0047520661157024e-06,
      "logits/chosen": -0.04833984375,
      "logits/rejected": -0.1904296875,
      "logps/chosen": -378.0,
      "logps/rejected": -362.0,
      "loss": 0.0277,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.921875,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.0,
      "step": 824
    },
    {
      "epoch": 1.0122699386503067,
      "grad_norm": 6.9895041600763665,
      "learning_rate": 1.0041322314049586e-06,
      "logits/chosen": 0.140625,
      "logits/rejected": -0.130859375,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.0298,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 9.625,
      "rewards/rejected": -8.0625,
      "step": 825
    },
    {
      "epoch": 1.0134969325153373,
      "grad_norm": 12.841731144722432,
      "learning_rate": 1.003512396694215e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.197265625,
      "logps/chosen": -372.0,
      "logps/rejected": -354.0,
      "loss": 0.0693,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 8.6875,
      "rewards/rejected": -7.5,
      "step": 826
    },
    {
      "epoch": 1.014723926380368,
      "grad_norm": 6.4682689307001455,
      "learning_rate": 1.002892561983471e-06,
      "logits/chosen": 0.01708984375,
      "logits/rejected": -0.091796875,
      "logps/chosen": -344.0,
      "logps/rejected": -338.0,
      "loss": 0.0373,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.65625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.75,
      "step": 827
    },
    {
      "epoch": 1.0159509202453987,
      "grad_norm": 4.914698955857516,
      "learning_rate": 1.0022727272727274e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": -0.036865234375,
      "logps/chosen": -382.0,
      "logps/rejected": -330.0,
      "loss": 0.0321,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.15625,
      "rewards/margins": 8.9375,
      "rewards/rejected": -7.75,
      "step": 828
    },
    {
      "epoch": 1.0171779141104293,
      "grad_norm": 13.706631134085173,
      "learning_rate": 1.0016528925619836e-06,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -376.0,
      "logps/rejected": -386.0,
      "loss": 0.0804,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 9.875,
      "rewards/rejected": -8.375,
      "step": 829
    },
    {
      "epoch": 1.01840490797546,
      "grad_norm": 4.7042664986817755,
      "learning_rate": 1.0010330578512397e-06,
      "logits/chosen": -0.059814453125,
      "logits/rejected": -0.25390625,
      "logps/chosen": -410.0,
      "logps/rejected": -384.0,
      "loss": 0.0274,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.03125,
      "rewards/margins": 11.125,
      "rewards/rejected": -9.125,
      "step": 830
    },
    {
      "epoch": 1.019631901840491,
      "grad_norm": 10.294544241161669,
      "learning_rate": 1.0004132231404959e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.0810546875,
      "logps/chosen": -404.0,
      "logps/rejected": -404.0,
      "loss": 0.0445,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.7109375,
      "rewards/margins": 11.1875,
      "rewards/rejected": -9.4375,
      "step": 831
    },
    {
      "epoch": 1.0208588957055216,
      "grad_norm": 11.737112567032785,
      "learning_rate": 9.99793388429752e-07,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -406.0,
      "logps/rejected": -388.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.15625,
      "rewards/margins": 10.0,
      "rewards/rejected": -8.875,
      "step": 832
    },
    {
      "epoch": 1.0220858895705522,
      "grad_norm": 22.2024732224313,
      "learning_rate": 9.991735537190084e-07,
      "logits/chosen": -0.111328125,
      "logits/rejected": -0.26953125,
      "logps/chosen": -450.0,
      "logps/rejected": -410.0,
      "loss": 0.1041,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.09375,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.4375,
      "step": 833
    },
    {
      "epoch": 1.023312883435583,
      "grad_norm": 10.90677573357752,
      "learning_rate": 9.985537190082645e-07,
      "logits/chosen": 0.024658203125,
      "logits/rejected": 0.00970458984375,
      "logps/chosen": -356.0,
      "logps/rejected": -366.0,
      "loss": 0.0368,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1435546875,
      "rewards/margins": 9.0,
      "rewards/rejected": -8.875,
      "step": 834
    },
    {
      "epoch": 1.0245398773006136,
      "grad_norm": 12.900773925818388,
      "learning_rate": 9.979338842975206e-07,
      "logits/chosen": 0.0712890625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -414.0,
      "logps/rejected": -404.0,
      "loss": 0.0634,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.859375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.3125,
      "step": 835
    },
    {
      "epoch": 1.0257668711656442,
      "grad_norm": 16.38159113775479,
      "learning_rate": 9.97314049586777e-07,
      "logits/chosen": -0.0419921875,
      "logits/rejected": -0.123046875,
      "logps/chosen": -376.0,
      "logps/rejected": -378.0,
      "loss": 0.0545,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.703125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -9.75,
      "step": 836
    },
    {
      "epoch": 1.026993865030675,
      "grad_norm": 5.759934115077334,
      "learning_rate": 9.966942148760331e-07,
      "logits/chosen": 0.12353515625,
      "logits/rejected": -0.13671875,
      "logps/chosen": -444.0,
      "logps/rejected": -408.0,
      "loss": 0.0291,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.15625,
      "rewards/margins": 10.5625,
      "rewards/rejected": -9.4375,
      "step": 837
    },
    {
      "epoch": 1.0282208588957056,
      "grad_norm": 10.475656005242545,
      "learning_rate": 9.960743801652893e-07,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.2265625,
      "logps/chosen": -394.0,
      "logps/rejected": -386.0,
      "loss": 0.0452,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.30078125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.125,
      "step": 838
    },
    {
      "epoch": 1.0294478527607362,
      "grad_norm": 16.273519813105544,
      "learning_rate": 9.954545454545456e-07,
      "logits/chosen": 0.0142822265625,
      "logits/rejected": -0.1328125,
      "logps/chosen": -398.0,
      "logps/rejected": -388.0,
      "loss": 0.052,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.640625,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.625,
      "step": 839
    },
    {
      "epoch": 1.030674846625767,
      "grad_norm": 15.221579329063106,
      "learning_rate": 9.948347107438016e-07,
      "logits/chosen": -0.0167236328125,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -446.0,
      "logps/rejected": -422.0,
      "loss": 0.0552,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.91015625,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.0,
      "step": 840
    },
    {
      "epoch": 1.0319018404907976,
      "grad_norm": 8.194954853669916,
      "learning_rate": 9.94214876033058e-07,
      "logits/chosen": 0.054443359375,
      "logits/rejected": -0.09716796875,
      "logps/chosen": -388.0,
      "logps/rejected": -396.0,
      "loss": 0.042,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.0072021484375,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.125,
      "step": 841
    },
    {
      "epoch": 1.0331288343558283,
      "grad_norm": 6.243363966929884,
      "learning_rate": 9.93595041322314e-07,
      "logits/chosen": -0.03955078125,
      "logits/rejected": -0.16015625,
      "logps/chosen": -456.0,
      "logps/rejected": -364.0,
      "loss": 0.0377,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.39453125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -9.625,
      "step": 842
    },
    {
      "epoch": 1.034355828220859,
      "grad_norm": 13.025481965324344,
      "learning_rate": 9.929752066115702e-07,
      "logits/chosen": -0.03955078125,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -460.0,
      "logps/rejected": -416.0,
      "loss": 0.0486,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.5390625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.125,
      "step": 843
    },
    {
      "epoch": 1.0355828220858896,
      "grad_norm": 9.843030269655861,
      "learning_rate": 9.923553719008265e-07,
      "logits/chosen": 0.0703125,
      "logits/rejected": -0.15625,
      "logps/chosen": -434.0,
      "logps/rejected": -380.0,
      "loss": 0.0284,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.1953125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.25,
      "step": 844
    },
    {
      "epoch": 1.0368098159509203,
      "grad_norm": 6.820305865616003,
      "learning_rate": 9.917355371900827e-07,
      "logits/chosen": -0.00494384765625,
      "logits/rejected": -0.11767578125,
      "logps/chosen": -374.0,
      "logps/rejected": -352.0,
      "loss": 0.0243,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41015625,
      "rewards/margins": 9.1875,
      "rewards/rejected": -8.75,
      "step": 845
    },
    {
      "epoch": 1.038036809815951,
      "grad_norm": 10.389123582255607,
      "learning_rate": 9.911157024793388e-07,
      "logits/chosen": 0.1025390625,
      "logits/rejected": -0.11181640625,
      "logps/chosen": -422.0,
      "logps/rejected": -356.0,
      "loss": 0.0361,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.109375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.3125,
      "step": 846
    },
    {
      "epoch": 1.0392638036809816,
      "grad_norm": 6.3892793464793165,
      "learning_rate": 9.904958677685952e-07,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.0830078125,
      "logps/chosen": -374.0,
      "logps/rejected": -366.0,
      "loss": 0.0319,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -9.375,
      "step": 847
    },
    {
      "epoch": 1.0404907975460123,
      "grad_norm": 6.1623878145786435,
      "learning_rate": 9.898760330578511e-07,
      "logits/chosen": -0.03955078125,
      "logits/rejected": -0.1357421875,
      "logps/chosen": -354.0,
      "logps/rejected": -362.0,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.205078125,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.4375,
      "step": 848
    },
    {
      "epoch": 1.041717791411043,
      "grad_norm": 9.98252857798321,
      "learning_rate": 9.892561983471075e-07,
      "logits/chosen": 0.01361083984375,
      "logits/rejected": -0.05859375,
      "logps/chosen": -384.0,
      "logps/rejected": -418.0,
      "loss": 0.044,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.06396484375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -10.125,
      "step": 849
    },
    {
      "epoch": 1.0429447852760736,
      "grad_norm": 6.152234316016775,
      "learning_rate": 9.886363636363636e-07,
      "logits/chosen": -0.0224609375,
      "logits/rejected": -0.26171875,
      "logps/chosen": -374.0,
      "logps/rejected": -368.0,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.125,
      "step": 850
    },
    {
      "epoch": 1.0441717791411043,
      "grad_norm": 5.157350169674604,
      "learning_rate": 9.880165289256197e-07,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.275390625,
      "logps/chosen": -382.0,
      "logps/rejected": -394.0,
      "loss": 0.016,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.6875,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.3125,
      "step": 851
    },
    {
      "epoch": 1.045398773006135,
      "grad_norm": 4.950192991710521,
      "learning_rate": 9.87396694214876e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.287109375,
      "logps/chosen": -432.0,
      "logps/rejected": -418.0,
      "loss": 0.0193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.90234375,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.0,
      "step": 852
    },
    {
      "epoch": 1.0466257668711656,
      "grad_norm": 6.8291284978183375,
      "learning_rate": 9.867768595041322e-07,
      "logits/chosen": -0.05615234375,
      "logits/rejected": -0.15625,
      "logps/chosen": -416.0,
      "logps/rejected": -372.0,
      "loss": 0.0293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.71875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.4375,
      "step": 853
    },
    {
      "epoch": 1.0478527607361963,
      "grad_norm": 21.523576031133718,
      "learning_rate": 9.861570247933884e-07,
      "logits/chosen": 0.037109375,
      "logits/rejected": -0.09814453125,
      "logps/chosen": -396.0,
      "logps/rejected": -388.0,
      "loss": 0.128,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.2265625,
      "rewards/margins": 9.25,
      "rewards/rejected": -9.0625,
      "step": 854
    },
    {
      "epoch": 1.049079754601227,
      "grad_norm": 7.615435271592111,
      "learning_rate": 9.855371900826447e-07,
      "logits/chosen": 0.0947265625,
      "logits/rejected": -0.045654296875,
      "logps/chosen": -398.0,
      "logps/rejected": -412.0,
      "loss": 0.0403,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.3125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -9.5,
      "step": 855
    },
    {
      "epoch": 1.0503067484662576,
      "grad_norm": 13.390214785568759,
      "learning_rate": 9.849173553719009e-07,
      "logits/chosen": -0.020751953125,
      "logits/rejected": -0.2021484375,
      "logps/chosen": -408.0,
      "logps/rejected": -364.0,
      "loss": 0.0522,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.375,
      "step": 856
    },
    {
      "epoch": 1.0515337423312883,
      "grad_norm": 7.721662927396619,
      "learning_rate": 9.84297520661157e-07,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.31640625,
      "logps/chosen": -404.0,
      "logps/rejected": -382.0,
      "loss": 0.0336,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.98828125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -9.5,
      "step": 857
    },
    {
      "epoch": 1.052760736196319,
      "grad_norm": 5.024633893121141,
      "learning_rate": 9.836776859504132e-07,
      "logits/chosen": -0.008056640625,
      "logits/rejected": -0.06884765625,
      "logps/chosen": -322.0,
      "logps/rejected": -328.0,
      "loss": 0.0242,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2109375,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.125,
      "step": 858
    },
    {
      "epoch": 1.0539877300613496,
      "grad_norm": 11.331768445922425,
      "learning_rate": 9.830578512396693e-07,
      "logits/chosen": -0.05029296875,
      "logits/rejected": -0.267578125,
      "logps/chosen": -400.0,
      "logps/rejected": -352.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.9375,
      "step": 859
    },
    {
      "epoch": 1.0552147239263803,
      "grad_norm": 11.938945819408568,
      "learning_rate": 9.824380165289257e-07,
      "logits/chosen": 0.042724609375,
      "logits/rejected": -0.16796875,
      "logps/chosen": -416.0,
      "logps/rejected": -384.0,
      "loss": 0.0571,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.267578125,
      "rewards/margins": 9.125,
      "rewards/rejected": -9.375,
      "step": 860
    },
    {
      "epoch": 1.056441717791411,
      "grad_norm": 10.622500605691952,
      "learning_rate": 9.818181818181818e-07,
      "logits/chosen": -0.00323486328125,
      "logits/rejected": -0.1953125,
      "logps/chosen": -404.0,
      "logps/rejected": -358.0,
      "loss": 0.0358,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.65625,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.5625,
      "step": 861
    },
    {
      "epoch": 1.0576687116564418,
      "grad_norm": 13.987502566864515,
      "learning_rate": 9.811983471074381e-07,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.2021484375,
      "logps/chosen": -386.0,
      "logps/rejected": -378.0,
      "loss": 0.053,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.5625,
      "step": 862
    },
    {
      "epoch": 1.0588957055214725,
      "grad_norm": 5.222070055196097,
      "learning_rate": 9.805785123966943e-07,
      "logits/chosen": -0.12890625,
      "logits/rejected": -0.345703125,
      "logps/chosen": -428.0,
      "logps/rejected": -404.0,
      "loss": 0.0159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.640625,
      "rewards/margins": 11.875,
      "rewards/rejected": -10.25,
      "step": 863
    },
    {
      "epoch": 1.0601226993865032,
      "grad_norm": 4.4447902099136645,
      "learning_rate": 9.799586776859504e-07,
      "logits/chosen": -0.051513671875,
      "logits/rejected": -0.248046875,
      "logps/chosen": -382.0,
      "logps/rejected": -390.0,
      "loss": 0.022,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.193359375,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.75,
      "step": 864
    },
    {
      "epoch": 1.0613496932515338,
      "grad_norm": 6.7260760377615,
      "learning_rate": 9.793388429752068e-07,
      "logits/chosen": -0.2177734375,
      "logits/rejected": -0.30859375,
      "logps/chosen": -432.0,
      "logps/rejected": -370.0,
      "loss": 0.0262,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1982421875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.5,
      "step": 865
    },
    {
      "epoch": 1.0625766871165645,
      "grad_norm": 7.412871849821934,
      "learning_rate": 9.787190082644627e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.3515625,
      "logps/chosen": -356.0,
      "logps/rejected": -364.0,
      "loss": 0.0325,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.455078125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.375,
      "step": 866
    },
    {
      "epoch": 1.0638036809815952,
      "grad_norm": 12.621207505635246,
      "learning_rate": 9.78099173553719e-07,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -434.0,
      "logps/rejected": -378.0,
      "loss": 0.0623,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.1123046875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.8125,
      "step": 867
    },
    {
      "epoch": 1.0650306748466258,
      "grad_norm": 6.401402652027187,
      "learning_rate": 9.774793388429752e-07,
      "logits/chosen": 0.1396484375,
      "logits/rejected": -0.06396484375,
      "logps/chosen": -358.0,
      "logps/rejected": -326.0,
      "loss": 0.0388,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.042236328125,
      "rewards/margins": 9.1875,
      "rewards/rejected": -9.125,
      "step": 868
    },
    {
      "epoch": 1.0662576687116565,
      "grad_norm": 8.548172524646416,
      "learning_rate": 9.768595041322313e-07,
      "logits/chosen": -0.10205078125,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -410.0,
      "logps/rejected": -394.0,
      "loss": 0.0415,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.5,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.3125,
      "step": 869
    },
    {
      "epoch": 1.0674846625766872,
      "grad_norm": 10.984226790709622,
      "learning_rate": 9.762396694214877e-07,
      "logits/chosen": 0.06396484375,
      "logits/rejected": -0.11474609375,
      "logps/chosen": -390.0,
      "logps/rejected": -374.0,
      "loss": 0.0438,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.0,
      "step": 870
    },
    {
      "epoch": 1.0687116564417178,
      "grad_norm": 17.83709695663358,
      "learning_rate": 9.756198347107438e-07,
      "logits/chosen": 0.0732421875,
      "logits/rejected": -0.09130859375,
      "logps/chosen": -410.0,
      "logps/rejected": -364.0,
      "loss": 0.0866,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1845703125,
      "rewards/margins": 9.375,
      "rewards/rejected": -9.25,
      "step": 871
    },
    {
      "epoch": 1.0699386503067485,
      "grad_norm": 6.167620411597598,
      "learning_rate": 9.75e-07,
      "logits/chosen": 0.03466796875,
      "logits/rejected": -0.0859375,
      "logps/chosen": -418.0,
      "logps/rejected": -338.0,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.53125,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.625,
      "step": 872
    },
    {
      "epoch": 1.0711656441717792,
      "grad_norm": 7.7202347168828265,
      "learning_rate": 9.743801652892563e-07,
      "logits/chosen": -0.056640625,
      "logits/rejected": -0.21484375,
      "logps/chosen": -380.0,
      "logps/rejected": -394.0,
      "loss": 0.0402,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.8125,
      "step": 873
    },
    {
      "epoch": 1.0723926380368098,
      "grad_norm": 20.769443880639034,
      "learning_rate": 9.737603305785125e-07,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -436.0,
      "logps/rejected": -412.0,
      "loss": 0.0993,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.1875,
      "step": 874
    },
    {
      "epoch": 1.0736196319018405,
      "grad_norm": 4.947185378698119,
      "learning_rate": 9.731404958677686e-07,
      "logits/chosen": 0.03564453125,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -354.0,
      "logps/rejected": -388.0,
      "loss": 0.0241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.474609375,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.9375,
      "step": 875
    },
    {
      "epoch": 1.0748466257668712,
      "grad_norm": 5.995360821265753,
      "learning_rate": 9.725206611570248e-07,
      "logits/chosen": -0.09814453125,
      "logits/rejected": -0.279296875,
      "logps/chosen": -420.0,
      "logps/rejected": -418.0,
      "loss": 0.0269,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.9375,
      "step": 876
    },
    {
      "epoch": 1.0760736196319018,
      "grad_norm": 2.795180251678899,
      "learning_rate": 9.71900826446281e-07,
      "logits/chosen": 0.0986328125,
      "logits/rejected": -0.050048828125,
      "logps/chosen": -408.0,
      "logps/rejected": -368.0,
      "loss": 0.0117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0380859375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.9375,
      "step": 877
    },
    {
      "epoch": 1.0773006134969325,
      "grad_norm": 11.795724961832926,
      "learning_rate": 9.712809917355373e-07,
      "logits/chosen": -0.08056640625,
      "logits/rejected": -0.287109375,
      "logps/chosen": -384.0,
      "logps/rejected": -380.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.21484375,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.25,
      "step": 878
    },
    {
      "epoch": 1.0785276073619632,
      "grad_norm": 4.020220366948393,
      "learning_rate": 9.706611570247934e-07,
      "logits/chosen": 0.040283203125,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -414.0,
      "logps/rejected": -362.0,
      "loss": 0.0148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.09375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.0,
      "step": 879
    },
    {
      "epoch": 1.0797546012269938,
      "grad_norm": 7.945458923967855,
      "learning_rate": 9.700413223140495e-07,
      "logits/chosen": 0.037109375,
      "logits/rejected": -0.205078125,
      "logps/chosen": -354.0,
      "logps/rejected": -354.0,
      "loss": 0.0369,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.421875,
      "rewards/margins": 10.625,
      "rewards/rejected": -11.0625,
      "step": 880
    },
    {
      "epoch": 1.0809815950920245,
      "grad_norm": 7.655504936651129,
      "learning_rate": 9.694214876033059e-07,
      "logits/chosen": 0.08447265625,
      "logits/rejected": -0.055908203125,
      "logps/chosen": -350.0,
      "logps/rejected": -320.0,
      "loss": 0.0397,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 8.8125,
      "rewards/rejected": -8.5625,
      "step": 881
    },
    {
      "epoch": 1.0822085889570552,
      "grad_norm": 17.14199925290297,
      "learning_rate": 9.68801652892562e-07,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.205078125,
      "logps/chosen": -402.0,
      "logps/rejected": -398.0,
      "loss": 0.092,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.296875,
      "rewards/margins": 10.625,
      "rewards/rejected": -9.375,
      "step": 882
    },
    {
      "epoch": 1.0834355828220859,
      "grad_norm": 6.074505313460856,
      "learning_rate": 9.681818181818184e-07,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.232421875,
      "logps/chosen": -388.0,
      "logps/rejected": -364.0,
      "loss": 0.0179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4375,
      "rewards/margins": 9.9375,
      "rewards/rejected": -8.5,
      "step": 883
    },
    {
      "epoch": 1.0846625766871165,
      "grad_norm": 5.184873219214592,
      "learning_rate": 9.675619834710743e-07,
      "logits/chosen": -0.1015625,
      "logits/rejected": -0.263671875,
      "logps/chosen": -428.0,
      "logps/rejected": -410.0,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8515625,
      "rewards/margins": 11.625,
      "rewards/rejected": -9.75,
      "step": 884
    },
    {
      "epoch": 1.0858895705521472,
      "grad_norm": 5.141932376248221,
      "learning_rate": 9.669421487603305e-07,
      "logits/chosen": -0.0250244140625,
      "logits/rejected": -0.16015625,
      "logps/chosen": -392.0,
      "logps/rejected": -354.0,
      "loss": 0.0325,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.5,
      "rewards/margins": 10.25,
      "rewards/rejected": -8.75,
      "step": 885
    },
    {
      "epoch": 1.0871165644171779,
      "grad_norm": 10.51661040179474,
      "learning_rate": 9.663223140495868e-07,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.126953125,
      "logps/chosen": -384.0,
      "logps/rejected": -384.0,
      "loss": 0.0416,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 9.3125,
      "rewards/rejected": -8.5625,
      "step": 886
    },
    {
      "epoch": 1.0883435582822085,
      "grad_norm": 10.214979385844872,
      "learning_rate": 9.65702479338843e-07,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.294921875,
      "logps/chosen": -402.0,
      "logps/rejected": -390.0,
      "loss": 0.0423,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.125,
      "step": 887
    },
    {
      "epoch": 1.0895705521472392,
      "grad_norm": 12.886011610183632,
      "learning_rate": 9.650826446280993e-07,
      "logits/chosen": -0.08544921875,
      "logits/rejected": -0.357421875,
      "logps/chosen": -400.0,
      "logps/rejected": -380.0,
      "loss": 0.0493,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.4375,
      "step": 888
    },
    {
      "epoch": 1.0907975460122699,
      "grad_norm": 21.84348705309875,
      "learning_rate": 9.644628099173554e-07,
      "logits/chosen": -0.044677734375,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -454.0,
      "logps/rejected": -420.0,
      "loss": 0.0864,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 1.3828125,
      "rewards/margins": 10.875,
      "rewards/rejected": -9.5,
      "step": 889
    },
    {
      "epoch": 1.0920245398773005,
      "grad_norm": 7.008527769291815,
      "learning_rate": 9.638429752066116e-07,
      "logits/chosen": -0.0947265625,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -366.0,
      "logps/rejected": -358.0,
      "loss": 0.0303,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.140625,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.125,
      "step": 890
    },
    {
      "epoch": 1.0932515337423312,
      "grad_norm": 9.38763327259464,
      "learning_rate": 9.63223140495868e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.28125,
      "logps/chosen": -368.0,
      "logps/rejected": -386.0,
      "loss": 0.0416,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.435546875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.125,
      "step": 891
    },
    {
      "epoch": 1.0944785276073619,
      "grad_norm": 7.190745784431904,
      "learning_rate": 9.62603305785124e-07,
      "logits/chosen": -0.04248046875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -358.0,
      "logps/rejected": -364.0,
      "loss": 0.0387,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.2099609375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.375,
      "step": 892
    },
    {
      "epoch": 1.0957055214723925,
      "grad_norm": 10.096733031161396,
      "learning_rate": 9.619834710743802e-07,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.09326171875,
      "logps/chosen": -320.0,
      "logps/rejected": -332.0,
      "loss": 0.0549,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -9.3125,
      "step": 893
    },
    {
      "epoch": 1.0969325153374232,
      "grad_norm": 15.853564017698188,
      "learning_rate": 9.613636363636364e-07,
      "logits/chosen": -0.06689453125,
      "logits/rejected": -0.28125,
      "logps/chosen": -442.0,
      "logps/rejected": -416.0,
      "loss": 0.0575,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.3125,
      "step": 894
    },
    {
      "epoch": 1.098159509202454,
      "grad_norm": 10.254886625086469,
      "learning_rate": 9.607438016528925e-07,
      "logits/chosen": -0.045166015625,
      "logits/rejected": -0.20703125,
      "logps/chosen": -388.0,
      "logps/rejected": -376.0,
      "loss": 0.0352,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.515625,
      "rewards/margins": 11.0,
      "rewards/rejected": -9.4375,
      "step": 895
    },
    {
      "epoch": 1.0993865030674848,
      "grad_norm": 7.1661826290113755,
      "learning_rate": 9.601239669421489e-07,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.248046875,
      "logps/chosen": -370.0,
      "logps/rejected": -374.0,
      "loss": 0.0272,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.4375,
      "step": 896
    },
    {
      "epoch": 1.1006134969325154,
      "grad_norm": 13.118106354556259,
      "learning_rate": 9.59504132231405e-07,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.357421875,
      "logps/chosen": -442.0,
      "logps/rejected": -392.0,
      "loss": 0.0388,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.90234375,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.375,
      "step": 897
    },
    {
      "epoch": 1.101840490797546,
      "grad_norm": 7.692840365219393,
      "learning_rate": 9.588842975206611e-07,
      "logits/chosen": -0.12353515625,
      "logits/rejected": -0.33203125,
      "logps/chosen": -420.0,
      "logps/rejected": -388.0,
      "loss": 0.0326,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.375,
      "step": 898
    },
    {
      "epoch": 1.1030674846625768,
      "grad_norm": 4.182192778148996,
      "learning_rate": 9.582644628099175e-07,
      "logits/chosen": -0.053466796875,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -340.0,
      "logps/rejected": -348.0,
      "loss": 0.0244,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.3125,
      "step": 899
    },
    {
      "epoch": 1.1042944785276074,
      "grad_norm": 17.13756577176759,
      "learning_rate": 9.576446280991736e-07,
      "logits/chosen": 0.0125732421875,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -404.0,
      "logps/rejected": -398.0,
      "loss": 0.0925,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.447265625,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.75,
      "step": 900
    },
    {
      "epoch": 1.105521472392638,
      "grad_norm": 6.771958107284894,
      "learning_rate": 9.570247933884298e-07,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.36328125,
      "logps/chosen": -490.0,
      "logps/rejected": -428.0,
      "loss": 0.0284,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.890625,
      "rewards/margins": 13.125,
      "rewards/rejected": -11.25,
      "step": 901
    },
    {
      "epoch": 1.1067484662576688,
      "grad_norm": 7.061170750215255,
      "learning_rate": 9.56404958677686e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -376.0,
      "logps/rejected": -384.0,
      "loss": 0.0349,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.5390625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.5625,
      "step": 902
    },
    {
      "epoch": 1.1079754601226994,
      "grad_norm": 8.245692452787445,
      "learning_rate": 9.55785123966942e-07,
      "logits/chosen": -0.08642578125,
      "logits/rejected": -0.30078125,
      "logps/chosen": -364.0,
      "logps/rejected": -370.0,
      "loss": 0.0341,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.4375,
      "step": 903
    },
    {
      "epoch": 1.10920245398773,
      "grad_norm": 7.285502821165709,
      "learning_rate": 9.551652892561984e-07,
      "logits/chosen": -0.09228515625,
      "logits/rejected": -0.22265625,
      "logps/chosen": -378.0,
      "logps/rejected": -416.0,
      "loss": 0.0233,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.109375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -10.6875,
      "step": 904
    },
    {
      "epoch": 1.1104294478527608,
      "grad_norm": 11.766334881942175,
      "learning_rate": 9.545454545454546e-07,
      "logits/chosen": -0.26171875,
      "logits/rejected": -0.458984375,
      "logps/chosen": -392.0,
      "logps/rejected": -392.0,
      "loss": 0.0414,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.453125,
      "rewards/margins": 12.0625,
      "rewards/rejected": -10.5625,
      "step": 905
    },
    {
      "epoch": 1.1116564417177914,
      "grad_norm": 11.885836553849193,
      "learning_rate": 9.539256198347107e-07,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.26171875,
      "logps/chosen": -342.0,
      "logps/rejected": -382.0,
      "loss": 0.0611,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.337890625,
      "rewards/margins": 9.5,
      "rewards/rejected": -9.875,
      "step": 906
    },
    {
      "epoch": 1.112883435582822,
      "grad_norm": 8.496822948479098,
      "learning_rate": 9.533057851239669e-07,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.388671875,
      "logps/chosen": -386.0,
      "logps/rejected": -366.0,
      "loss": 0.0392,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.4921875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.1875,
      "step": 907
    },
    {
      "epoch": 1.1141104294478528,
      "grad_norm": 4.416563202380208,
      "learning_rate": 9.526859504132232e-07,
      "logits/chosen": -0.16796875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -392.0,
      "logps/rejected": -380.0,
      "loss": 0.0153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.0,
      "step": 908
    },
    {
      "epoch": 1.1153374233128834,
      "grad_norm": 12.914705090980133,
      "learning_rate": 9.520661157024794e-07,
      "logits/chosen": -0.014892578125,
      "logits/rejected": -0.1123046875,
      "logps/chosen": -418.0,
      "logps/rejected": -524.0,
      "loss": 0.0577,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7265625,
      "rewards/margins": 19.375,
      "rewards/rejected": -18.625,
      "step": 909
    },
    {
      "epoch": 1.116564417177914,
      "grad_norm": 11.193369005763312,
      "learning_rate": 9.514462809917357e-07,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.3359375,
      "logps/chosen": -392.0,
      "logps/rejected": -400.0,
      "loss": 0.0381,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.9296875,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.6875,
      "step": 910
    },
    {
      "epoch": 1.1177914110429448,
      "grad_norm": 10.08348423519109,
      "learning_rate": 9.508264462809917e-07,
      "logits/chosen": -0.109375,
      "logits/rejected": -0.3515625,
      "logps/chosen": -472.0,
      "logps/rejected": -428.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 13.375,
      "rewards/rejected": -11.6875,
      "step": 911
    },
    {
      "epoch": 1.1190184049079754,
      "grad_norm": 14.853944201654002,
      "learning_rate": 9.502066115702479e-07,
      "logits/chosen": 0.01153564453125,
      "logits/rejected": -0.0849609375,
      "logps/chosen": -362.0,
      "logps/rejected": -384.0,
      "loss": 0.0499,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.07080078125,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.1875,
      "step": 912
    },
    {
      "epoch": 1.120245398773006,
      "grad_norm": 7.41210069844449,
      "learning_rate": 9.495867768595041e-07,
      "logits/chosen": -0.1845703125,
      "logits/rejected": -0.28125,
      "logps/chosen": -328.0,
      "logps/rejected": -376.0,
      "loss": 0.0393,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.0,
      "step": 913
    },
    {
      "epoch": 1.1214723926380368,
      "grad_norm": 17.23126040997008,
      "learning_rate": 9.489669421487604e-07,
      "logits/chosen": 0.0013275146484375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -412.0,
      "logps/rejected": -376.0,
      "loss": 0.0818,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.037109375,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.125,
      "step": 914
    },
    {
      "epoch": 1.1226993865030674,
      "grad_norm": 7.728386443202459,
      "learning_rate": 9.483471074380166e-07,
      "logits/chosen": -0.034912109375,
      "logits/rejected": -0.291015625,
      "logps/chosen": -408.0,
      "logps/rejected": -402.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.8125,
      "step": 915
    },
    {
      "epoch": 1.123926380368098,
      "grad_norm": 8.320575830299521,
      "learning_rate": 9.477272727272727e-07,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.34765625,
      "logps/chosen": -394.0,
      "logps/rejected": -376.0,
      "loss": 0.0332,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.51171875,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.875,
      "step": 916
    },
    {
      "epoch": 1.1251533742331288,
      "grad_norm": 10.220447898199359,
      "learning_rate": 9.47107438016529e-07,
      "logits/chosen": -0.1650390625,
      "logits/rejected": -0.392578125,
      "logps/chosen": -428.0,
      "logps/rejected": -430.0,
      "loss": 0.0592,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.291015625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.625,
      "step": 917
    },
    {
      "epoch": 1.1263803680981594,
      "grad_norm": 16.327456489125716,
      "learning_rate": 9.464876033057852e-07,
      "logits/chosen": -0.040771484375,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -388.0,
      "logps/rejected": -384.0,
      "loss": 0.0625,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.375,
      "step": 918
    },
    {
      "epoch": 1.1276073619631901,
      "grad_norm": 9.650345793046794,
      "learning_rate": 9.458677685950414e-07,
      "logits/chosen": -0.2412109375,
      "logits/rejected": -0.294921875,
      "logps/chosen": -384.0,
      "logps/rejected": -408.0,
      "loss": 0.0513,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.44140625,
      "rewards/margins": 10.625,
      "rewards/rejected": -11.0625,
      "step": 919
    },
    {
      "epoch": 1.1288343558282208,
      "grad_norm": 5.645777430480824,
      "learning_rate": 9.452479338842975e-07,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.287109375,
      "logps/chosen": -410.0,
      "logps/rejected": -396.0,
      "loss": 0.0183,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0281982421875,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.0,
      "step": 920
    },
    {
      "epoch": 1.1300613496932514,
      "grad_norm": 6.67262482291788,
      "learning_rate": 9.446280991735537e-07,
      "logits/chosen": -0.08935546875,
      "logits/rejected": -0.2421875,
      "logps/chosen": -398.0,
      "logps/rejected": -396.0,
      "loss": 0.0236,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.078125,
      "rewards/margins": 11.6875,
      "rewards/rejected": -10.5625,
      "step": 921
    },
    {
      "epoch": 1.1312883435582821,
      "grad_norm": 6.984229781332186,
      "learning_rate": 9.440082644628099e-07,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.15234375,
      "logps/chosen": -392.0,
      "logps/rejected": -402.0,
      "loss": 0.0334,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.98046875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -9.9375,
      "step": 922
    },
    {
      "epoch": 1.132515337423313,
      "grad_norm": 12.662651710075986,
      "learning_rate": 9.433884297520662e-07,
      "logits/chosen": 0.005828857421875,
      "logits/rejected": -0.08349609375,
      "logps/chosen": -364.0,
      "logps/rejected": -378.0,
      "loss": 0.0303,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -9.625,
      "step": 923
    },
    {
      "epoch": 1.1337423312883437,
      "grad_norm": 9.002647218488805,
      "learning_rate": 9.427685950413223e-07,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.2265625,
      "logps/chosen": -384.0,
      "logps/rejected": -368.0,
      "loss": 0.0351,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.91015625,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.375,
      "step": 924
    },
    {
      "epoch": 1.1349693251533743,
      "grad_norm": 12.813399120951164,
      "learning_rate": 9.421487603305785e-07,
      "logits/chosen": -0.203125,
      "logits/rejected": -0.404296875,
      "logps/chosen": -378.0,
      "logps/rejected": -364.0,
      "loss": 0.0329,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.0306396484375,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.8125,
      "step": 925
    },
    {
      "epoch": 1.136196319018405,
      "grad_norm": 13.681237077440016,
      "learning_rate": 9.415289256198348e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.384765625,
      "logps/chosen": -426.0,
      "logps/rejected": -410.0,
      "loss": 0.0828,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.9921875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.1875,
      "step": 926
    },
    {
      "epoch": 1.1374233128834357,
      "grad_norm": 14.007812307598066,
      "learning_rate": 9.40909090909091e-07,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -426.0,
      "logps/rejected": -406.0,
      "loss": 0.0454,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.921875,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.1875,
      "step": 927
    },
    {
      "epoch": 1.1386503067484663,
      "grad_norm": 5.804375472545515,
      "learning_rate": 9.402892561983472e-07,
      "logits/chosen": -0.06982421875,
      "logits/rejected": -0.2734375,
      "logps/chosen": -416.0,
      "logps/rejected": -398.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.67578125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.125,
      "step": 928
    },
    {
      "epoch": 1.139877300613497,
      "grad_norm": 7.0706705773655205,
      "learning_rate": 9.396694214876032e-07,
      "logits/chosen": -0.038330078125,
      "logits/rejected": -0.212890625,
      "logps/chosen": -430.0,
      "logps/rejected": -348.0,
      "loss": 0.0216,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.90625,
      "rewards/margins": 10.75,
      "rewards/rejected": -9.8125,
      "step": 929
    },
    {
      "epoch": 1.1411042944785277,
      "grad_norm": 20.03419906761046,
      "learning_rate": 9.390495867768595e-07,
      "logits/chosen": -0.0673828125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -408.0,
      "logps/rejected": -392.0,
      "loss": 0.0678,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.546875,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.9375,
      "step": 930
    },
    {
      "epoch": 1.1423312883435583,
      "grad_norm": 8.79614969540993,
      "learning_rate": 9.384297520661157e-07,
      "logits/chosen": -0.0712890625,
      "logits/rejected": -0.154296875,
      "logps/chosen": -412.0,
      "logps/rejected": -376.0,
      "loss": 0.0392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.041748046875,
      "rewards/margins": 9.5,
      "rewards/rejected": -9.5625,
      "step": 931
    },
    {
      "epoch": 1.143558282208589,
      "grad_norm": 5.533709744700053,
      "learning_rate": 9.37809917355372e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -380.0,
      "logps/rejected": -380.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.045166015625,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.0625,
      "step": 932
    },
    {
      "epoch": 1.1447852760736197,
      "grad_norm": 17.392392128691636,
      "learning_rate": 9.371900826446281e-07,
      "logits/chosen": -0.04052734375,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -372.0,
      "logps/rejected": -400.0,
      "loss": 0.0582,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.875,
      "step": 933
    },
    {
      "epoch": 1.1460122699386504,
      "grad_norm": 11.220639802991899,
      "learning_rate": 9.365702479338843e-07,
      "logits/chosen": -0.076171875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -386.0,
      "logps/rejected": -366.0,
      "loss": 0.0466,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.3125,
      "step": 934
    },
    {
      "epoch": 1.147239263803681,
      "grad_norm": 3.127170097265917,
      "learning_rate": 9.359504132231406e-07,
      "logits/chosen": -0.125,
      "logits/rejected": -0.26171875,
      "logps/chosen": -416.0,
      "logps/rejected": -408.0,
      "loss": 0.0156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.703125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.0,
      "step": 935
    },
    {
      "epoch": 1.1484662576687117,
      "grad_norm": 8.797510711759905,
      "learning_rate": 9.353305785123968e-07,
      "logits/chosen": 0.04833984375,
      "logits/rejected": -0.1953125,
      "logps/chosen": -366.0,
      "logps/rejected": -382.0,
      "loss": 0.0399,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.0927734375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.6875,
      "step": 936
    },
    {
      "epoch": 1.1496932515337424,
      "grad_norm": 8.508479799139819,
      "learning_rate": 9.347107438016529e-07,
      "logits/chosen": -0.032958984375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -434.0,
      "logps/rejected": -406.0,
      "loss": 0.0222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.8125,
      "step": 937
    },
    {
      "epoch": 1.150920245398773,
      "grad_norm": 11.734844982011332,
      "learning_rate": 9.34090909090909e-07,
      "logits/chosen": -0.01031494140625,
      "logits/rejected": -0.058837890625,
      "logps/chosen": -374.0,
      "logps/rejected": -378.0,
      "loss": 0.0576,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.294921875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.375,
      "step": 938
    },
    {
      "epoch": 1.1521472392638037,
      "grad_norm": 6.686498950384566,
      "learning_rate": 9.334710743801653e-07,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.318359375,
      "logps/chosen": -450.0,
      "logps/rejected": -420.0,
      "loss": 0.022,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0703125,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.0,
      "step": 939
    },
    {
      "epoch": 1.1533742331288344,
      "grad_norm": 9.333043593563687,
      "learning_rate": 9.328512396694215e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.361328125,
      "logps/chosen": -420.0,
      "logps/rejected": -388.0,
      "loss": 0.0298,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0234375,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.125,
      "step": 940
    },
    {
      "epoch": 1.154601226993865,
      "grad_norm": 13.328010603903623,
      "learning_rate": 9.322314049586778e-07,
      "logits/chosen": -0.0150146484375,
      "logits/rejected": -0.220703125,
      "logps/chosen": -418.0,
      "logps/rejected": -390.0,
      "loss": 0.058,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.25,
      "rewards/margins": 10.9375,
      "rewards/rejected": -9.6875,
      "step": 941
    },
    {
      "epoch": 1.1558282208588957,
      "grad_norm": 10.748405478390469,
      "learning_rate": 9.316115702479339e-07,
      "logits/chosen": -0.2431640625,
      "logits/rejected": -0.380859375,
      "logps/chosen": -418.0,
      "logps/rejected": -406.0,
      "loss": 0.0373,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.0625,
      "step": 942
    },
    {
      "epoch": 1.1570552147239264,
      "grad_norm": 6.66797822267693,
      "learning_rate": 9.309917355371901e-07,
      "logits/chosen": 0.0140380859375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -354.0,
      "logps/rejected": -350.0,
      "loss": 0.021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.359375,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.875,
      "step": 943
    },
    {
      "epoch": 1.158282208588957,
      "grad_norm": 6.910993717404971,
      "learning_rate": 9.303719008264464e-07,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -404.0,
      "logps/rejected": -412.0,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.4375,
      "step": 944
    },
    {
      "epoch": 1.1595092024539877,
      "grad_norm": 8.94698116889764,
      "learning_rate": 9.297520661157025e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.33984375,
      "logps/chosen": -416.0,
      "logps/rejected": -380.0,
      "loss": 0.0401,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.390625,
      "rewards/margins": 10.75,
      "rewards/rejected": -9.375,
      "step": 945
    },
    {
      "epoch": 1.1607361963190184,
      "grad_norm": 5.757411937782045,
      "learning_rate": 9.291322314049587e-07,
      "logits/chosen": -0.0016937255859375,
      "logits/rejected": -0.07177734375,
      "logps/chosen": -380.0,
      "logps/rejected": -400.0,
      "loss": 0.0224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.291015625,
      "rewards/margins": 9.8125,
      "rewards/rejected": -9.5625,
      "step": 946
    },
    {
      "epoch": 1.161963190184049,
      "grad_norm": 10.946703026968246,
      "learning_rate": 9.285123966942148e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -346.0,
      "logps/rejected": -368.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 10.125,
      "rewards/rejected": -9.375,
      "step": 947
    },
    {
      "epoch": 1.1631901840490797,
      "grad_norm": 13.673425770565624,
      "learning_rate": 9.278925619834711e-07,
      "logits/chosen": -0.1962890625,
      "logits/rejected": -0.427734375,
      "logps/chosen": -428.0,
      "logps/rejected": -380.0,
      "loss": 0.0544,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.5,
      "rewards/margins": 10.6875,
      "rewards/rejected": -9.1875,
      "step": 948
    },
    {
      "epoch": 1.1644171779141104,
      "grad_norm": 10.764489540205084,
      "learning_rate": 9.272727272727273e-07,
      "logits/chosen": 0.0032958984375,
      "logits/rejected": -0.265625,
      "logps/chosen": -414.0,
      "logps/rejected": -410.0,
      "loss": 0.0627,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.46875,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.125,
      "step": 949
    },
    {
      "epoch": 1.165644171779141,
      "grad_norm": 6.163510999316528,
      "learning_rate": 9.266528925619834e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.23828125,
      "logps/chosen": -338.0,
      "logps/rejected": -336.0,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.265625,
      "rewards/margins": 10.5,
      "rewards/rejected": -10.25,
      "step": 950
    },
    {
      "epoch": 1.1668711656441717,
      "grad_norm": 7.49094764434787,
      "learning_rate": 9.260330578512397e-07,
      "logits/chosen": -0.021240234375,
      "logits/rejected": -0.146484375,
      "logps/chosen": -350.0,
      "logps/rejected": -354.0,
      "loss": 0.0353,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.6875,
      "step": 951
    },
    {
      "epoch": 1.1680981595092024,
      "grad_norm": 11.812912807851047,
      "learning_rate": 9.254132231404959e-07,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.494140625,
      "logps/chosen": -424.0,
      "logps/rejected": -386.0,
      "loss": 0.0565,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.09375,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.0625,
      "step": 952
    },
    {
      "epoch": 1.169325153374233,
      "grad_norm": 9.899530945112486,
      "learning_rate": 9.247933884297522e-07,
      "logits/chosen": -0.07080078125,
      "logits/rejected": -0.271484375,
      "logps/chosen": -388.0,
      "logps/rejected": -360.0,
      "loss": 0.038,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.96484375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -9.5625,
      "step": 953
    },
    {
      "epoch": 1.1705521472392637,
      "grad_norm": 14.874148700852642,
      "learning_rate": 9.241735537190083e-07,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -322.0,
      "logps/rejected": -320.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.439453125,
      "rewards/margins": 9.8125,
      "rewards/rejected": -10.25,
      "step": 954
    },
    {
      "epoch": 1.1717791411042944,
      "grad_norm": 14.69776784288544,
      "learning_rate": 9.235537190082644e-07,
      "logits/chosen": -0.005157470703125,
      "logits/rejected": -0.0771484375,
      "logps/chosen": -380.0,
      "logps/rejected": -378.0,
      "loss": 0.0538,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.015625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -9.8125,
      "step": 955
    },
    {
      "epoch": 1.173006134969325,
      "grad_norm": 8.526200653171099,
      "learning_rate": 9.229338842975206e-07,
      "logits/chosen": -0.10009765625,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -316.0,
      "logps/rejected": -344.0,
      "loss": 0.0306,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 9.75,
      "rewards/rejected": -9.4375,
      "step": 956
    },
    {
      "epoch": 1.1742331288343557,
      "grad_norm": 7.973457405245253,
      "learning_rate": 9.223140495867769e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.357421875,
      "logps/chosen": -390.0,
      "logps/rejected": -404.0,
      "loss": 0.0368,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 10.875,
      "rewards/rejected": -9.8125,
      "step": 957
    },
    {
      "epoch": 1.1754601226993864,
      "grad_norm": 14.106346468292243,
      "learning_rate": 9.216942148760331e-07,
      "logits/chosen": -0.055908203125,
      "logits/rejected": -0.15625,
      "logps/chosen": -364.0,
      "logps/rejected": -404.0,
      "loss": 0.0479,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.404296875,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.625,
      "step": 958
    },
    {
      "epoch": 1.1766871165644173,
      "grad_norm": 11.510149735049389,
      "learning_rate": 9.210743801652892e-07,
      "logits/chosen": -0.095703125,
      "logits/rejected": -0.267578125,
      "logps/chosen": -396.0,
      "logps/rejected": -388.0,
      "loss": 0.0431,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.1875,
      "step": 959
    },
    {
      "epoch": 1.177914110429448,
      "grad_norm": 3.8552096147370842,
      "learning_rate": 9.204545454545455e-07,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.330078125,
      "logps/chosen": -346.0,
      "logps/rejected": -350.0,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.546875,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.75,
      "step": 960
    },
    {
      "epoch": 1.1791411042944786,
      "grad_norm": 13.533375661298507,
      "learning_rate": 9.198347107438017e-07,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.0407,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.85546875,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.625,
      "step": 961
    },
    {
      "epoch": 1.1803680981595093,
      "grad_norm": 9.63092706255587,
      "learning_rate": 9.192148760330579e-07,
      "logits/chosen": 0.00921630859375,
      "logits/rejected": -0.16015625,
      "logps/chosen": -440.0,
      "logps/rejected": -430.0,
      "loss": 0.0318,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.375,
      "step": 962
    },
    {
      "epoch": 1.18159509202454,
      "grad_norm": 13.10088160617771,
      "learning_rate": 9.185950413223141e-07,
      "logits/chosen": -0.11474609375,
      "logits/rejected": -0.306640625,
      "logps/chosen": -378.0,
      "logps/rejected": -352.0,
      "loss": 0.0397,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.61328125,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.25,
      "step": 963
    },
    {
      "epoch": 1.1828220858895706,
      "grad_norm": 7.358279878141561,
      "learning_rate": 9.179752066115702e-07,
      "logits/chosen": -0.049560546875,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -380.0,
      "logps/rejected": -368.0,
      "loss": 0.0233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.6875,
      "step": 964
    },
    {
      "epoch": 1.1840490797546013,
      "grad_norm": 10.963522347339602,
      "learning_rate": 9.173553719008264e-07,
      "logits/chosen": -0.12158203125,
      "logits/rejected": -0.255859375,
      "logps/chosen": -366.0,
      "logps/rejected": -386.0,
      "loss": 0.0329,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.287109375,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.0625,
      "step": 965
    },
    {
      "epoch": 1.185276073619632,
      "grad_norm": 11.155993123724087,
      "learning_rate": 9.167355371900827e-07,
      "logits/chosen": -0.1875,
      "logits/rejected": -0.322265625,
      "logps/chosen": -396.0,
      "logps/rejected": -360.0,
      "loss": 0.0412,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.00421142578125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.625,
      "step": 966
    },
    {
      "epoch": 1.1865030674846626,
      "grad_norm": 7.534460905603958,
      "learning_rate": 9.161157024793388e-07,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.28515625,
      "logps/chosen": -412.0,
      "logps/rejected": -378.0,
      "loss": 0.027,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.95703125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.5,
      "step": 967
    },
    {
      "epoch": 1.1877300613496933,
      "grad_norm": 17.6032434079415,
      "learning_rate": 9.15495867768595e-07,
      "logits/chosen": -0.041015625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -424.0,
      "logps/rejected": -396.0,
      "loss": 0.0462,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.875,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.375,
      "step": 968
    },
    {
      "epoch": 1.188957055214724,
      "grad_norm": 15.694096678580106,
      "learning_rate": 9.148760330578513e-07,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.3046875,
      "logps/chosen": -388.0,
      "logps/rejected": -422.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.390625,
      "rewards/margins": 10.0,
      "rewards/rejected": -11.375,
      "step": 969
    },
    {
      "epoch": 1.1901840490797546,
      "grad_norm": 11.921476909261088,
      "learning_rate": 9.142561983471075e-07,
      "logits/chosen": -0.10595703125,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -372.0,
      "logps/rejected": -502.0,
      "loss": 0.0453,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.01531982421875,
      "rewards/margins": 23.125,
      "rewards/rejected": -23.125,
      "step": 970
    },
    {
      "epoch": 1.1914110429447853,
      "grad_norm": 13.62660815818298,
      "learning_rate": 9.136363636363637e-07,
      "logits/chosen": -0.0091552734375,
      "logits/rejected": -0.25,
      "logps/chosen": -358.0,
      "logps/rejected": -356.0,
      "loss": 0.0572,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.09375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.625,
      "step": 971
    },
    {
      "epoch": 1.192638036809816,
      "grad_norm": 7.237399332254699,
      "learning_rate": 9.130165289256199e-07,
      "logits/chosen": -0.057373046875,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -342.0,
      "logps/rejected": -312.0,
      "loss": 0.0381,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.3515625,
      "rewards/margins": 9.5625,
      "rewards/rejected": -9.875,
      "step": 972
    },
    {
      "epoch": 1.1938650306748466,
      "grad_norm": 9.089933594933969,
      "learning_rate": 9.12396694214876e-07,
      "logits/chosen": -0.1533203125,
      "logits/rejected": -0.279296875,
      "logps/chosen": -412.0,
      "logps/rejected": -432.0,
      "loss": 0.029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.875,
      "step": 973
    },
    {
      "epoch": 1.1950920245398773,
      "grad_norm": 3.7486390717851217,
      "learning_rate": 9.117768595041322e-07,
      "logits/chosen": -0.2275390625,
      "logits/rejected": -0.41796875,
      "logps/chosen": -398.0,
      "logps/rejected": -406.0,
      "loss": 0.0146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.09375,
      "rewards/margins": 11.9375,
      "rewards/rejected": -10.8125,
      "step": 974
    },
    {
      "epoch": 1.196319018404908,
      "grad_norm": 17.692363179531846,
      "learning_rate": 9.111570247933885e-07,
      "logits/chosen": 0.045654296875,
      "logits/rejected": -0.07861328125,
      "logps/chosen": -360.0,
      "logps/rejected": -332.0,
      "loss": 0.0651,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.2265625,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.6875,
      "step": 975
    },
    {
      "epoch": 1.1975460122699386,
      "grad_norm": 13.042080391360054,
      "learning_rate": 9.105371900826446e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.392578125,
      "logps/chosen": -422.0,
      "logps/rejected": -390.0,
      "loss": 0.0488,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -9.4375,
      "step": 976
    },
    {
      "epoch": 1.1987730061349693,
      "grad_norm": 17.283449726274785,
      "learning_rate": 9.099173553719009e-07,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -388.0,
      "logps/rejected": -374.0,
      "loss": 0.083,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.91796875,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.5625,
      "step": 977
    },
    {
      "epoch": 1.2,
      "grad_norm": 10.327665992150298,
      "learning_rate": 9.092975206611571e-07,
      "logits/chosen": -0.11865234375,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -364.0,
      "logps/rejected": -344.0,
      "loss": 0.0616,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.046630859375,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.125,
      "step": 978
    },
    {
      "epoch": 1.2012269938650306,
      "grad_norm": 6.639005543404221,
      "learning_rate": 9.086776859504132e-07,
      "logits/chosen": -0.00439453125,
      "logits/rejected": -0.126953125,
      "logps/chosen": -410.0,
      "logps/rejected": -384.0,
      "loss": 0.0294,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.76953125,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.4375,
      "step": 979
    },
    {
      "epoch": 1.2024539877300613,
      "grad_norm": 3.7048596375075307,
      "learning_rate": 9.080578512396695e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.35546875,
      "logps/chosen": -362.0,
      "logps/rejected": -374.0,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6484375,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.0,
      "step": 980
    },
    {
      "epoch": 1.203680981595092,
      "grad_norm": 9.842884355659873,
      "learning_rate": 9.074380165289257e-07,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.1796875,
      "logps/chosen": -386.0,
      "logps/rejected": -386.0,
      "loss": 0.0228,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.0625,
      "step": 981
    },
    {
      "epoch": 1.2049079754601226,
      "grad_norm": 8.509249883161877,
      "learning_rate": 9.068181818181818e-07,
      "logits/chosen": -0.09130859375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -408.0,
      "logps/rejected": -408.0,
      "loss": 0.0327,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.734375,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.0,
      "step": 982
    },
    {
      "epoch": 1.2061349693251533,
      "grad_norm": 17.32331392424542,
      "learning_rate": 9.06198347107438e-07,
      "logits/chosen": -0.09814453125,
      "logits/rejected": -0.28125,
      "logps/chosen": -382.0,
      "logps/rejected": -346.0,
      "loss": 0.0823,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.671875,
      "rewards/margins": 9.625,
      "rewards/rejected": -9.0,
      "step": 983
    },
    {
      "epoch": 1.207361963190184,
      "grad_norm": 6.308837245908633,
      "learning_rate": 9.055785123966942e-07,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.310546875,
      "logps/chosen": -362.0,
      "logps/rejected": -346.0,
      "loss": 0.0285,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.447265625,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.875,
      "step": 984
    },
    {
      "epoch": 1.2085889570552146,
      "grad_norm": 11.159178783645137,
      "learning_rate": 9.049586776859504e-07,
      "logits/chosen": -0.04443359375,
      "logits/rejected": -0.212890625,
      "logps/chosen": -350.0,
      "logps/rejected": -352.0,
      "loss": 0.0427,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.5625,
      "step": 985
    },
    {
      "epoch": 1.2098159509202455,
      "grad_norm": 9.051373556481083,
      "learning_rate": 9.043388429752067e-07,
      "logits/chosen": 0.0625,
      "logits/rejected": -0.08935546875,
      "logps/chosen": -390.0,
      "logps/rejected": -356.0,
      "loss": 0.0347,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 10.0,
      "rewards/rejected": -8.9375,
      "step": 986
    },
    {
      "epoch": 1.2110429447852762,
      "grad_norm": 6.198302604249075,
      "learning_rate": 9.037190082644629e-07,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.390625,
      "logps/chosen": -458.0,
      "logps/rejected": -406.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.28125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -9.875,
      "step": 987
    },
    {
      "epoch": 1.2122699386503069,
      "grad_norm": 4.102469498417344,
      "learning_rate": 9.03099173553719e-07,
      "logits/chosen": -0.047607421875,
      "logits/rejected": -0.26171875,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.0143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.240234375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.3125,
      "step": 988
    },
    {
      "epoch": 1.2134969325153375,
      "grad_norm": 8.46030603702519,
      "learning_rate": 9.024793388429753e-07,
      "logits/chosen": -0.0023345947265625,
      "logits/rejected": -0.16796875,
      "logps/chosen": -354.0,
      "logps/rejected": -342.0,
      "loss": 0.0404,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.21484375,
      "rewards/margins": 9.4375,
      "rewards/rejected": -9.6875,
      "step": 989
    },
    {
      "epoch": 1.2147239263803682,
      "grad_norm": 10.249236383125274,
      "learning_rate": 9.018595041322315e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.326171875,
      "logps/chosen": -402.0,
      "logps/rejected": -396.0,
      "loss": 0.0387,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.58984375,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.875,
      "step": 990
    },
    {
      "epoch": 1.2159509202453989,
      "grad_norm": 20.10350249363253,
      "learning_rate": 9.012396694214876e-07,
      "logits/chosen": -0.09130859375,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -416.0,
      "logps/rejected": -408.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.263671875,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.25,
      "step": 991
    },
    {
      "epoch": 1.2171779141104295,
      "grad_norm": 15.353242393368603,
      "learning_rate": 9.006198347107438e-07,
      "logits/chosen": -0.11572265625,
      "logits/rejected": -0.302734375,
      "logps/chosen": -388.0,
      "logps/rejected": -392.0,
      "loss": 0.067,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.11962890625,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.75,
      "step": 992
    },
    {
      "epoch": 1.2184049079754602,
      "grad_norm": 14.594366015899684,
      "learning_rate": 9e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.314453125,
      "logps/chosen": -376.0,
      "logps/rejected": -376.0,
      "loss": 0.0584,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7265625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.0625,
      "step": 993
    },
    {
      "epoch": 1.2196319018404909,
      "grad_norm": 17.819835095986992,
      "learning_rate": 8.993801652892562e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.341796875,
      "logps/chosen": -428.0,
      "logps/rejected": -408.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.251953125,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.8125,
      "step": 994
    },
    {
      "epoch": 1.2208588957055215,
      "grad_norm": 15.26610473864711,
      "learning_rate": 8.987603305785125e-07,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -362.0,
      "logps/rejected": -398.0,
      "loss": 0.046,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.52734375,
      "rewards/margins": 10.3125,
      "rewards/rejected": -10.8125,
      "step": 995
    },
    {
      "epoch": 1.2220858895705522,
      "grad_norm": 19.263245504901857,
      "learning_rate": 8.981404958677686e-07,
      "logits/chosen": -0.023193359375,
      "logits/rejected": -0.287109375,
      "logps/chosen": -388.0,
      "logps/rejected": -372.0,
      "loss": 0.0498,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.2021484375,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.375,
      "step": 996
    },
    {
      "epoch": 1.2233128834355829,
      "grad_norm": 12.681921452742241,
      "learning_rate": 8.975206611570248e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.37109375,
      "logps/chosen": -420.0,
      "logps/rejected": -376.0,
      "loss": 0.0371,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.171875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -9.8125,
      "step": 997
    },
    {
      "epoch": 1.2245398773006135,
      "grad_norm": 6.006248666573442,
      "learning_rate": 8.969008264462811e-07,
      "logits/chosen": -0.017578125,
      "logits/rejected": -0.21484375,
      "logps/chosen": -446.0,
      "logps/rejected": -414.0,
      "loss": 0.0184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.171875,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.0625,
      "step": 998
    },
    {
      "epoch": 1.2257668711656442,
      "grad_norm": 11.143450935670302,
      "learning_rate": 8.962809917355373e-07,
      "logits/chosen": -0.07177734375,
      "logits/rejected": -0.240234375,
      "logps/chosen": -410.0,
      "logps/rejected": -386.0,
      "loss": 0.0398,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.21875,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.0625,
      "step": 999
    },
    {
      "epoch": 1.2269938650306749,
      "grad_norm": 8.965376937967298,
      "learning_rate": 8.956611570247934e-07,
      "logits/chosen": -0.01385498046875,
      "logits/rejected": -0.1015625,
      "logps/chosen": -386.0,
      "logps/rejected": -386.0,
      "loss": 0.0469,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0234375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -9.75,
      "step": 1000
    },
    {
      "epoch": 1.2282208588957055,
      "grad_norm": 11.244494297541378,
      "learning_rate": 8.950413223140495e-07,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -410.0,
      "logps/rejected": -378.0,
      "loss": 0.0447,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 11.75,
      "rewards/rejected": -10.3125,
      "step": 1001
    },
    {
      "epoch": 1.2294478527607362,
      "grad_norm": 16.003709589565055,
      "learning_rate": 8.944214876033058e-07,
      "logits/chosen": -0.059326171875,
      "logits/rejected": -0.09375,
      "logps/chosen": -378.0,
      "logps/rejected": -410.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -9.9375,
      "step": 1002
    },
    {
      "epoch": 1.2306748466257669,
      "grad_norm": 12.870892763097368,
      "learning_rate": 8.93801652892562e-07,
      "logits/chosen": -0.1552734375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -362.0,
      "logps/rejected": -376.0,
      "loss": 0.0662,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.48828125,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.125,
      "step": 1003
    },
    {
      "epoch": 1.2319018404907975,
      "grad_norm": 5.2378730711968435,
      "learning_rate": 8.931818181818183e-07,
      "logits/chosen": -0.03076171875,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.0257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.55859375,
      "rewards/margins": 9.875,
      "rewards/rejected": -9.3125,
      "step": 1004
    },
    {
      "epoch": 1.2331288343558282,
      "grad_norm": 6.60847796992945,
      "learning_rate": 8.925619834710744e-07,
      "logits/chosen": -0.0322265625,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -396.0,
      "logps/rejected": -360.0,
      "loss": 0.031,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.125,
      "step": 1005
    },
    {
      "epoch": 1.2343558282208589,
      "grad_norm": 7.331811572998349,
      "learning_rate": 8.919421487603306e-07,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.26171875,
      "logps/chosen": -368.0,
      "logps/rejected": -360.0,
      "loss": 0.03,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.375,
      "step": 1006
    },
    {
      "epoch": 1.2355828220858895,
      "grad_norm": 12.728485046758394,
      "learning_rate": 8.913223140495869e-07,
      "logits/chosen": -0.0595703125,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -404.0,
      "logps/rejected": -404.0,
      "loss": 0.0543,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.25,
      "step": 1007
    },
    {
      "epoch": 1.2368098159509202,
      "grad_norm": 4.4503501042063816,
      "learning_rate": 8.90702479338843e-07,
      "logits/chosen": -0.0751953125,
      "logits/rejected": -0.1904296875,
      "logps/chosen": -358.0,
      "logps/rejected": -356.0,
      "loss": 0.0251,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1806640625,
      "rewards/margins": 10.125,
      "rewards/rejected": -9.9375,
      "step": 1008
    },
    {
      "epoch": 1.2380368098159509,
      "grad_norm": 12.812283743763317,
      "learning_rate": 8.900826446280992e-07,
      "logits/chosen": 0.0169677734375,
      "logits/rejected": -0.185546875,
      "logps/chosen": -426.0,
      "logps/rejected": -380.0,
      "loss": 0.0431,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.40625,
      "rewards/margins": 11.1875,
      "rewards/rejected": -9.8125,
      "step": 1009
    },
    {
      "epoch": 1.2392638036809815,
      "grad_norm": 6.899754042678334,
      "learning_rate": 8.894628099173553e-07,
      "logits/chosen": -0.0181884765625,
      "logits/rejected": -0.10205078125,
      "logps/chosen": -354.0,
      "logps/rejected": -374.0,
      "loss": 0.03,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10595703125,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.0625,
      "step": 1010
    },
    {
      "epoch": 1.2404907975460122,
      "grad_norm": 6.300076629714054,
      "learning_rate": 8.888429752066116e-07,
      "logits/chosen": -0.0289306640625,
      "logits/rejected": -0.21875,
      "logps/chosen": -412.0,
      "logps/rejected": -390.0,
      "loss": 0.0272,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.25,
      "step": 1011
    },
    {
      "epoch": 1.2417177914110429,
      "grad_norm": 15.956703887603753,
      "learning_rate": 8.882231404958678e-07,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.25390625,
      "logps/chosen": -384.0,
      "logps/rejected": -372.0,
      "loss": 0.0733,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.75,
      "step": 1012
    },
    {
      "epoch": 1.2429447852760735,
      "grad_norm": 7.738683663773001,
      "learning_rate": 8.87603305785124e-07,
      "logits/chosen": 0.01318359375,
      "logits/rejected": -0.049560546875,
      "logps/chosen": -398.0,
      "logps/rejected": -380.0,
      "loss": 0.0336,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.11328125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -9.1875,
      "step": 1013
    },
    {
      "epoch": 1.2441717791411042,
      "grad_norm": 8.908202927372313,
      "learning_rate": 8.869834710743802e-07,
      "logits/chosen": -0.09765625,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -386.0,
      "logps/rejected": -406.0,
      "loss": 0.0403,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.00567626953125,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.0,
      "step": 1014
    },
    {
      "epoch": 1.2453987730061349,
      "grad_norm": 8.437111565369221,
      "learning_rate": 8.863636363636364e-07,
      "logits/chosen": -0.06689453125,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -370.0,
      "logps/rejected": -388.0,
      "loss": 0.036,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.21875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -10.4375,
      "step": 1015
    },
    {
      "epoch": 1.2466257668711656,
      "grad_norm": 19.806315774782945,
      "learning_rate": 8.857438016528927e-07,
      "logits/chosen": -0.0859375,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -364.0,
      "logps/rejected": -348.0,
      "loss": 0.0999,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 9.4375,
      "rewards/rejected": -9.5,
      "step": 1016
    },
    {
      "epoch": 1.2478527607361962,
      "grad_norm": 1.9621085105816412,
      "learning_rate": 8.851239669421488e-07,
      "logits/chosen": -0.08740234375,
      "logits/rejected": -0.31640625,
      "logps/chosen": -404.0,
      "logps/rejected": -362.0,
      "loss": 0.0089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.234375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -10.5625,
      "step": 1017
    },
    {
      "epoch": 1.2490797546012269,
      "grad_norm": 7.315995365487366,
      "learning_rate": 8.845041322314049e-07,
      "logits/chosen": -0.11865234375,
      "logits/rejected": -0.1962890625,
      "logps/chosen": -408.0,
      "logps/rejected": -416.0,
      "loss": 0.0283,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.11572265625,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.25,
      "step": 1018
    },
    {
      "epoch": 1.2503067484662576,
      "grad_norm": 6.452661280214959,
      "learning_rate": 8.838842975206611e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.302734375,
      "logps/chosen": -376.0,
      "logps/rejected": -378.0,
      "loss": 0.0318,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.296875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.875,
      "step": 1019
    },
    {
      "epoch": 1.2515337423312882,
      "grad_norm": 4.8866762625917,
      "learning_rate": 8.832644628099174e-07,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.193359375,
      "logps/chosen": -414.0,
      "logps/rejected": -402.0,
      "loss": 0.0215,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.33203125,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.4375,
      "step": 1020
    },
    {
      "epoch": 1.252760736196319,
      "grad_norm": 7.044566658635158,
      "learning_rate": 8.826446280991736e-07,
      "logits/chosen": -0.05908203125,
      "logits/rejected": -0.13671875,
      "logps/chosen": -370.0,
      "logps/rejected": -378.0,
      "loss": 0.0313,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.171875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.25,
      "step": 1021
    },
    {
      "epoch": 1.2539877300613496,
      "grad_norm": 2.074684049190498,
      "learning_rate": 8.820247933884298e-07,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.30859375,
      "logps/chosen": -420.0,
      "logps/rejected": -396.0,
      "loss": 0.0134,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.9375,
      "step": 1022
    },
    {
      "epoch": 1.2552147239263804,
      "grad_norm": 11.38544236701507,
      "learning_rate": 8.81404958677686e-07,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.314453125,
      "logps/chosen": -402.0,
      "logps/rejected": -398.0,
      "loss": 0.0566,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -10.875,
      "step": 1023
    },
    {
      "epoch": 1.2564417177914111,
      "grad_norm": 7.64965729920436,
      "learning_rate": 8.807851239669422e-07,
      "logits/chosen": -0.028564453125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -410.0,
      "logps/rejected": -420.0,
      "loss": 0.0254,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.5,
      "step": 1024
    },
    {
      "epoch": 1.2576687116564418,
      "grad_norm": 23.522395476453028,
      "learning_rate": 8.801652892561984e-07,
      "logits/chosen": -0.03466796875,
      "logits/rejected": -0.1962890625,
      "logps/chosen": -414.0,
      "logps/rejected": -418.0,
      "loss": 0.1276,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.3125,
      "step": 1025
    },
    {
      "epoch": 1.2588957055214725,
      "grad_norm": 4.975922218453701,
      "learning_rate": 8.795454545454545e-07,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.30078125,
      "logps/chosen": -370.0,
      "logps/rejected": -376.0,
      "loss": 0.0313,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.625,
      "step": 1026
    },
    {
      "epoch": 1.2601226993865031,
      "grad_norm": 4.952502473285682,
      "learning_rate": 8.789256198347107e-07,
      "logits/chosen": -0.17578125,
      "logits/rejected": -0.41015625,
      "logps/chosen": -448.0,
      "logps/rejected": -412.0,
      "loss": 0.0181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.75,
      "step": 1027
    },
    {
      "epoch": 1.2613496932515338,
      "grad_norm": 7.5494432139799015,
      "learning_rate": 8.783057851239669e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.3125,
      "logps/chosen": -396.0,
      "logps/rejected": -396.0,
      "loss": 0.0222,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.0625,
      "step": 1028
    },
    {
      "epoch": 1.2625766871165645,
      "grad_norm": 8.079926823970226,
      "learning_rate": 8.776859504132232e-07,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -390.0,
      "logps/rejected": -384.0,
      "loss": 0.0315,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.455078125,
      "rewards/margins": 10.5,
      "rewards/rejected": -10.0,
      "step": 1029
    },
    {
      "epoch": 1.2638036809815951,
      "grad_norm": 6.946808552773218,
      "learning_rate": 8.770661157024793e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": -0.38671875,
      "logps/chosen": -414.0,
      "logps/rejected": -410.0,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 12.125,
      "rewards/rejected": -10.9375,
      "step": 1030
    },
    {
      "epoch": 1.2650306748466258,
      "grad_norm": 13.376232318087313,
      "learning_rate": 8.764462809917356e-07,
      "logits/chosen": -0.0703125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -382.0,
      "logps/rejected": -360.0,
      "loss": 0.0746,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.55859375,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.875,
      "step": 1031
    },
    {
      "epoch": 1.2662576687116565,
      "grad_norm": 6.898643809079758,
      "learning_rate": 8.758264462809918e-07,
      "logits/chosen": 0.042724609375,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -376.0,
      "logps/rejected": -358.0,
      "loss": 0.0263,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -9.8125,
      "step": 1032
    },
    {
      "epoch": 1.2674846625766871,
      "grad_norm": 14.180165703858819,
      "learning_rate": 8.75206611570248e-07,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.33984375,
      "logps/chosen": -438.0,
      "logps/rejected": -392.0,
      "loss": 0.0718,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.004547119140625,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.0625,
      "step": 1033
    },
    {
      "epoch": 1.2687116564417178,
      "grad_norm": 5.086342064289492,
      "learning_rate": 8.745867768595042e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.38671875,
      "logps/chosen": -464.0,
      "logps/rejected": -420.0,
      "loss": 0.0192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.25,
      "rewards/margins": 12.1875,
      "rewards/rejected": -10.9375,
      "step": 1034
    },
    {
      "epoch": 1.2699386503067485,
      "grad_norm": 14.981826471029674,
      "learning_rate": 8.739669421487602e-07,
      "logits/chosen": -0.0712890625,
      "logits/rejected": -0.279296875,
      "logps/chosen": -408.0,
      "logps/rejected": -388.0,
      "loss": 0.0436,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.8046875,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.5625,
      "step": 1035
    },
    {
      "epoch": 1.2711656441717791,
      "grad_norm": 11.555096518453992,
      "learning_rate": 8.733471074380165e-07,
      "logits/chosen": -0.0693359375,
      "logits/rejected": -0.20703125,
      "logps/chosen": -380.0,
      "logps/rejected": -368.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 10.875,
      "rewards/rejected": -9.6875,
      "step": 1036
    },
    {
      "epoch": 1.2723926380368098,
      "grad_norm": 4.796864490403106,
      "learning_rate": 8.727272727272727e-07,
      "logits/chosen": -0.27734375,
      "logits/rejected": -0.41015625,
      "logps/chosen": -416.0,
      "logps/rejected": -432.0,
      "loss": 0.0167,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.984375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.5625,
      "step": 1037
    },
    {
      "epoch": 1.2736196319018405,
      "grad_norm": 7.4606854944626555,
      "learning_rate": 8.72107438016529e-07,
      "logits/chosen": -0.1103515625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -382.0,
      "logps/rejected": -392.0,
      "loss": 0.0268,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.60546875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.0625,
      "step": 1038
    },
    {
      "epoch": 1.2748466257668711,
      "grad_norm": 11.522250663977559,
      "learning_rate": 8.714876033057851e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -382.0,
      "logps/rejected": -414.0,
      "loss": 0.0475,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.0751953125,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.3125,
      "step": 1039
    },
    {
      "epoch": 1.2760736196319018,
      "grad_norm": 13.412392481033551,
      "learning_rate": 8.708677685950414e-07,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.470703125,
      "logps/chosen": -426.0,
      "logps/rejected": -366.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.125,
      "step": 1040
    },
    {
      "epoch": 1.2773006134969325,
      "grad_norm": 4.076327393298983,
      "learning_rate": 8.702479338842976e-07,
      "logits/chosen": -0.1357421875,
      "logits/rejected": -0.2734375,
      "logps/chosen": -382.0,
      "logps/rejected": -378.0,
      "loss": 0.013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.279296875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.0625,
      "step": 1041
    },
    {
      "epoch": 1.2785276073619631,
      "grad_norm": 12.198994819526218,
      "learning_rate": 8.696280991735537e-07,
      "logits/chosen": -0.3046875,
      "logits/rejected": -0.5,
      "logps/chosen": -452.0,
      "logps/rejected": -434.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.0,
      "step": 1042
    },
    {
      "epoch": 1.2797546012269938,
      "grad_norm": 15.876129225419682,
      "learning_rate": 8.6900826446281e-07,
      "logits/chosen": -0.10400390625,
      "logits/rejected": -0.287109375,
      "logps/chosen": -380.0,
      "logps/rejected": -386.0,
      "loss": 0.0536,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.63671875,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.8125,
      "step": 1043
    },
    {
      "epoch": 1.2809815950920245,
      "grad_norm": 5.177920548454582,
      "learning_rate": 8.68388429752066e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.421875,
      "logps/chosen": -432.0,
      "logps/rejected": -402.0,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.0625,
      "step": 1044
    },
    {
      "epoch": 1.2822085889570551,
      "grad_norm": 8.9147491979472,
      "learning_rate": 8.677685950413223e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.52734375,
      "logps/chosen": -430.0,
      "logps/rejected": -374.0,
      "loss": 0.0292,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.734375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.0,
      "step": 1045
    },
    {
      "epoch": 1.2834355828220858,
      "grad_norm": 6.882145843965057,
      "learning_rate": 8.671487603305785e-07,
      "logits/chosen": -0.017578125,
      "logits/rejected": -0.1689453125,
      "logps/chosen": -396.0,
      "logps/rejected": -404.0,
      "loss": 0.0288,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.0260009765625,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.6875,
      "step": 1046
    },
    {
      "epoch": 1.2846625766871167,
      "grad_norm": 12.443133314198835,
      "learning_rate": 8.665289256198347e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.330078125,
      "logps/chosen": -404.0,
      "logps/rejected": -404.0,
      "loss": 0.0533,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.5625,
      "step": 1047
    },
    {
      "epoch": 1.2858895705521474,
      "grad_norm": 4.187308341376978,
      "learning_rate": 8.659090909090909e-07,
      "logits/chosen": -0.0703125,
      "logits/rejected": -0.265625,
      "logps/chosen": -342.0,
      "logps/rejected": -348.0,
      "loss": 0.019,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.8125,
      "step": 1048
    },
    {
      "epoch": 1.287116564417178,
      "grad_norm": 12.339404257771985,
      "learning_rate": 8.652892561983472e-07,
      "logits/chosen": -0.10693359375,
      "logits/rejected": -0.326171875,
      "logps/chosen": -376.0,
      "logps/rejected": -374.0,
      "loss": 0.0446,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.8125,
      "step": 1049
    },
    {
      "epoch": 1.2883435582822087,
      "grad_norm": 7.946831033807572,
      "learning_rate": 8.646694214876034e-07,
      "logits/chosen": -0.2451171875,
      "logits/rejected": -0.384765625,
      "logps/chosen": -398.0,
      "logps/rejected": -406.0,
      "loss": 0.0372,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.25390625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.3125,
      "step": 1050
    },
    {
      "epoch": 1.2895705521472394,
      "grad_norm": 12.451016602757491,
      "learning_rate": 8.640495867768595e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": -0.466796875,
      "logps/chosen": -420.0,
      "logps/rejected": -424.0,
      "loss": 0.037,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.0,
      "step": 1051
    },
    {
      "epoch": 1.29079754601227,
      "grad_norm": 14.98941632692573,
      "learning_rate": 8.634297520661158e-07,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -432.0,
      "logps/rejected": -414.0,
      "loss": 0.0593,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.12353515625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.8125,
      "step": 1052
    },
    {
      "epoch": 1.2920245398773007,
      "grad_norm": 17.525115785642342,
      "learning_rate": 8.628099173553718e-07,
      "logits/chosen": -0.06640625,
      "logits/rejected": -0.287109375,
      "logps/chosen": -366.0,
      "logps/rejected": -366.0,
      "loss": 0.069,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26171875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.0625,
      "step": 1053
    },
    {
      "epoch": 1.2932515337423314,
      "grad_norm": 11.652127715235839,
      "learning_rate": 8.621900826446281e-07,
      "logits/chosen": -0.11767578125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -402.0,
      "logps/rejected": -394.0,
      "loss": 0.0503,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.6875,
      "step": 1054
    },
    {
      "epoch": 1.294478527607362,
      "grad_norm": 5.171446811615976,
      "learning_rate": 8.615702479338843e-07,
      "logits/chosen": -0.11962890625,
      "logits/rejected": -0.33203125,
      "logps/chosen": -386.0,
      "logps/rejected": -386.0,
      "loss": 0.0178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0439453125,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.75,
      "step": 1055
    },
    {
      "epoch": 1.2957055214723927,
      "grad_norm": 3.3720116007928245,
      "learning_rate": 8.609504132231405e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.34375,
      "logps/chosen": -368.0,
      "logps/rejected": -346.0,
      "loss": 0.0109,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0181884765625,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.875,
      "step": 1056
    },
    {
      "epoch": 1.2969325153374234,
      "grad_norm": 12.84642536946459,
      "learning_rate": 8.603305785123967e-07,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -404.0,
      "logps/rejected": -390.0,
      "loss": 0.0402,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.9375,
      "step": 1057
    },
    {
      "epoch": 1.298159509202454,
      "grad_norm": 9.137246685227527,
      "learning_rate": 8.59710743801653e-07,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.365234375,
      "logps/chosen": -390.0,
      "logps/rejected": -388.0,
      "loss": 0.0341,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.84765625,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.5,
      "step": 1058
    },
    {
      "epoch": 1.2993865030674847,
      "grad_norm": 8.50866186346429,
      "learning_rate": 8.590909090909092e-07,
      "logits/chosen": -0.1845703125,
      "logits/rejected": -0.38671875,
      "logps/chosen": -416.0,
      "logps/rejected": -384.0,
      "loss": 0.0325,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.341796875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.6875,
      "step": 1059
    },
    {
      "epoch": 1.3006134969325154,
      "grad_norm": 8.4621363068171,
      "learning_rate": 8.584710743801653e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.2421875,
      "logps/chosen": -338.0,
      "logps/rejected": -350.0,
      "loss": 0.0316,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.392578125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.875,
      "step": 1060
    },
    {
      "epoch": 1.301840490797546,
      "grad_norm": 12.798702675578042,
      "learning_rate": 8.578512396694216e-07,
      "logits/chosen": -0.059326171875,
      "logits/rejected": -0.1630859375,
      "logps/chosen": -342.0,
      "logps/rejected": -372.0,
      "loss": 0.0443,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.052734375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.6875,
      "step": 1061
    },
    {
      "epoch": 1.3030674846625767,
      "grad_norm": 10.362126702943744,
      "learning_rate": 8.572314049586776e-07,
      "logits/chosen": -0.1015625,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -422.0,
      "logps/rejected": -416.0,
      "loss": 0.0494,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.369140625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.6875,
      "step": 1062
    },
    {
      "epoch": 1.3042944785276074,
      "grad_norm": 13.766378342401254,
      "learning_rate": 8.566115702479339e-07,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.296875,
      "logps/chosen": -370.0,
      "logps/rejected": -366.0,
      "loss": 0.0643,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.228515625,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.9375,
      "step": 1063
    },
    {
      "epoch": 1.305521472392638,
      "grad_norm": 8.950574541603398,
      "learning_rate": 8.559917355371901e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.3046875,
      "logps/chosen": -386.0,
      "logps/rejected": -392.0,
      "loss": 0.034,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.057861328125,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.4375,
      "step": 1064
    },
    {
      "epoch": 1.3067484662576687,
      "grad_norm": 8.164498598328498,
      "learning_rate": 8.553719008264463e-07,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.4296875,
      "logps/chosen": -448.0,
      "logps/rejected": -408.0,
      "loss": 0.0283,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2734375,
      "rewards/margins": 13.3125,
      "rewards/rejected": -12.0,
      "step": 1065
    },
    {
      "epoch": 1.3079754601226994,
      "grad_norm": 9.939864149496472,
      "learning_rate": 8.547520661157025e-07,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.435546875,
      "logps/chosen": -464.0,
      "logps/rejected": -430.0,
      "loss": 0.0316,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.328125,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.1875,
      "step": 1066
    },
    {
      "epoch": 1.30920245398773,
      "grad_norm": 5.026276230946228,
      "learning_rate": 8.541322314049588e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.27734375,
      "logps/chosen": -376.0,
      "logps/rejected": -386.0,
      "loss": 0.0294,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.359375,
      "rewards/margins": 9.75,
      "rewards/rejected": -10.125,
      "step": 1067
    },
    {
      "epoch": 1.3104294478527607,
      "grad_norm": 11.031737625122464,
      "learning_rate": 8.535123966942149e-07,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.32421875,
      "logps/chosen": -388.0,
      "logps/rejected": -366.0,
      "loss": 0.0537,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.283203125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.0625,
      "step": 1068
    },
    {
      "epoch": 1.3116564417177914,
      "grad_norm": 8.905100714168142,
      "learning_rate": 8.528925619834711e-07,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.3046875,
      "logps/chosen": -406.0,
      "logps/rejected": -402.0,
      "loss": 0.0429,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.48828125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.1875,
      "step": 1069
    },
    {
      "epoch": 1.312883435582822,
      "grad_norm": 11.752525584916722,
      "learning_rate": 8.522727272727274e-07,
      "logits/chosen": -0.19140625,
      "logits/rejected": -0.259765625,
      "logps/chosen": -386.0,
      "logps/rejected": -406.0,
      "loss": 0.0348,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.96484375,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.1875,
      "step": 1070
    },
    {
      "epoch": 1.3141104294478527,
      "grad_norm": 12.851082658918303,
      "learning_rate": 8.516528925619834e-07,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.408203125,
      "logps/chosen": -392.0,
      "logps/rejected": -388.0,
      "loss": 0.0656,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.4375,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.8125,
      "step": 1071
    },
    {
      "epoch": 1.3153374233128834,
      "grad_norm": 9.860743385521058,
      "learning_rate": 8.510330578512397e-07,
      "logits/chosen": -0.002685546875,
      "logits/rejected": -0.197265625,
      "logps/chosen": -408.0,
      "logps/rejected": -398.0,
      "loss": 0.0258,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.4375,
      "step": 1072
    },
    {
      "epoch": 1.316564417177914,
      "grad_norm": 8.94951495923183,
      "learning_rate": 8.504132231404958e-07,
      "logits/chosen": -0.1982421875,
      "logits/rejected": -0.30859375,
      "logps/chosen": -330.0,
      "logps/rejected": -372.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.0400390625,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.1875,
      "step": 1073
    },
    {
      "epoch": 1.3177914110429447,
      "grad_norm": 5.703540754764619,
      "learning_rate": 8.497933884297521e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": -0.3203125,
      "logps/chosen": -390.0,
      "logps/rejected": -388.0,
      "loss": 0.0261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.291015625,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.125,
      "step": 1074
    },
    {
      "epoch": 1.3190184049079754,
      "grad_norm": 9.876644774460352,
      "learning_rate": 8.491735537190083e-07,
      "logits/chosen": -0.060791015625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -398.0,
      "logps/rejected": -396.0,
      "loss": 0.0274,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.3046875,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.75,
      "step": 1075
    },
    {
      "epoch": 1.320245398773006,
      "grad_norm": 4.967869121366148,
      "learning_rate": 8.485537190082646e-07,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.123046875,
      "logps/chosen": -400.0,
      "logps/rejected": -398.0,
      "loss": 0.0212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.1875,
      "step": 1076
    },
    {
      "epoch": 1.3214723926380367,
      "grad_norm": 9.322961880086396,
      "learning_rate": 8.479338842975207e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": -0.3359375,
      "logps/chosen": -302.0,
      "logps/rejected": -312.0,
      "loss": 0.0452,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.3984375,
      "rewards/margins": 9.875,
      "rewards/rejected": -10.25,
      "step": 1077
    },
    {
      "epoch": 1.3226993865030674,
      "grad_norm": 5.738661119438272,
      "learning_rate": 8.473140495867769e-07,
      "logits/chosen": -0.049072265625,
      "logits/rejected": -0.158203125,
      "logps/chosen": -386.0,
      "logps/rejected": -396.0,
      "loss": 0.0334,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 9.3125,
      "rewards/rejected": -9.8125,
      "step": 1078
    },
    {
      "epoch": 1.323926380368098,
      "grad_norm": 6.333892508394478,
      "learning_rate": 8.466942148760332e-07,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -384.0,
      "logps/rejected": -388.0,
      "loss": 0.0328,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.16796875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.5,
      "step": 1079
    },
    {
      "epoch": 1.3251533742331287,
      "grad_norm": 13.790501614554328,
      "learning_rate": 8.460743801652892e-07,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.263671875,
      "logps/chosen": -366.0,
      "logps/rejected": -358.0,
      "loss": 0.047,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.0322265625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.0625,
      "step": 1080
    },
    {
      "epoch": 1.3263803680981594,
      "grad_norm": 7.653539387358885,
      "learning_rate": 8.454545454545455e-07,
      "logits/chosen": -0.1005859375,
      "logits/rejected": -0.259765625,
      "logps/chosen": -392.0,
      "logps/rejected": -396.0,
      "loss": 0.0364,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1640625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.75,
      "step": 1081
    },
    {
      "epoch": 1.32760736196319,
      "grad_norm": 10.410957750314225,
      "learning_rate": 8.448347107438016e-07,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.435546875,
      "logps/chosen": -412.0,
      "logps/rejected": -366.0,
      "loss": 0.0374,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2119140625,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.125,
      "step": 1082
    },
    {
      "epoch": 1.3288343558282207,
      "grad_norm": 5.66314951524206,
      "learning_rate": 8.442148760330579e-07,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.283203125,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.142578125,
      "rewards/margins": 9.875,
      "rewards/rejected": -10.0,
      "step": 1083
    },
    {
      "epoch": 1.3300613496932514,
      "grad_norm": 11.42793523481754,
      "learning_rate": 8.435950413223141e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.16015625,
      "logps/chosen": -354.0,
      "logps/rejected": -362.0,
      "loss": 0.0449,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1640625,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.5,
      "step": 1084
    },
    {
      "epoch": 1.331288343558282,
      "grad_norm": 6.950168336723294,
      "learning_rate": 8.429752066115703e-07,
      "logits/chosen": -0.033203125,
      "logits/rejected": -0.181640625,
      "logps/chosen": -386.0,
      "logps/rejected": -376.0,
      "loss": 0.0221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.125,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.25,
      "step": 1085
    },
    {
      "epoch": 1.332515337423313,
      "grad_norm": 7.903285776855893,
      "learning_rate": 8.423553719008265e-07,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.228515625,
      "logps/chosen": -316.0,
      "logps/rejected": -344.0,
      "loss": 0.0384,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.375,
      "step": 1086
    },
    {
      "epoch": 1.3337423312883436,
      "grad_norm": 14.294247050939283,
      "learning_rate": 8.417355371900827e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -322.0,
      "logps/rejected": -376.0,
      "loss": 0.0381,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.251953125,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.625,
      "step": 1087
    },
    {
      "epoch": 1.3349693251533743,
      "grad_norm": 12.270443455514881,
      "learning_rate": 8.41115702479339e-07,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.365234375,
      "logps/chosen": -446.0,
      "logps/rejected": -378.0,
      "loss": 0.0352,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.65234375,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.625,
      "step": 1088
    },
    {
      "epoch": 1.336196319018405,
      "grad_norm": 9.38631227726379,
      "learning_rate": 8.40495867768595e-07,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.34375,
      "logps/chosen": -390.0,
      "logps/rejected": -360.0,
      "loss": 0.039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 9.875,
      "rewards/rejected": -10.5,
      "step": 1089
    },
    {
      "epoch": 1.3374233128834356,
      "grad_norm": 13.010546372728042,
      "learning_rate": 8.398760330578512e-07,
      "logits/chosen": -0.1357421875,
      "logits/rejected": -0.33984375,
      "logps/chosen": -374.0,
      "logps/rejected": -352.0,
      "loss": 0.0449,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.1728515625,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.25,
      "step": 1090
    },
    {
      "epoch": 1.3386503067484663,
      "grad_norm": 17.2682088486596,
      "learning_rate": 8.392561983471074e-07,
      "logits/chosen": 0.013427734375,
      "logits/rejected": -0.1552734375,
      "logps/chosen": -424.0,
      "logps/rejected": -368.0,
      "loss": 0.0874,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.1650390625,
      "rewards/margins": 10.125,
      "rewards/rejected": -9.9375,
      "step": 1091
    },
    {
      "epoch": 1.339877300613497,
      "grad_norm": 6.844436331855588,
      "learning_rate": 8.386363636363637e-07,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.31640625,
      "logps/chosen": -414.0,
      "logps/rejected": -398.0,
      "loss": 0.0256,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.484375,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.25,
      "step": 1092
    },
    {
      "epoch": 1.3411042944785276,
      "grad_norm": 5.767510463883874,
      "learning_rate": 8.380165289256199e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.34375,
      "logps/chosen": -378.0,
      "logps/rejected": -372.0,
      "loss": 0.0207,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.328125,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.9375,
      "step": 1093
    },
    {
      "epoch": 1.3423312883435583,
      "grad_norm": 8.158470602909508,
      "learning_rate": 8.37396694214876e-07,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -426.0,
      "logps/rejected": -420.0,
      "loss": 0.0291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.546875,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.3125,
      "step": 1094
    },
    {
      "epoch": 1.343558282208589,
      "grad_norm": 5.508962768344179,
      "learning_rate": 8.367768595041323e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -446.0,
      "logps/rejected": -430.0,
      "loss": 0.0202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.890625,
      "rewards/margins": 12.4375,
      "rewards/rejected": -11.5,
      "step": 1095
    },
    {
      "epoch": 1.3447852760736196,
      "grad_norm": 18.396109473702488,
      "learning_rate": 8.361570247933885e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.279296875,
      "logps/chosen": -382.0,
      "logps/rejected": -374.0,
      "loss": 0.0744,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.091796875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -9.9375,
      "step": 1096
    },
    {
      "epoch": 1.3460122699386503,
      "grad_norm": 4.9008028557309355,
      "learning_rate": 8.355371900826447e-07,
      "logits/chosen": -0.1015625,
      "logits/rejected": -0.279296875,
      "logps/chosen": -432.0,
      "logps/rejected": -380.0,
      "loss": 0.0189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.0625,
      "step": 1097
    },
    {
      "epoch": 1.347239263803681,
      "grad_norm": 17.180940504647815,
      "learning_rate": 8.349173553719008e-07,
      "logits/chosen": -0.0299072265625,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -364.0,
      "logps/rejected": -384.0,
      "loss": 0.0604,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1845703125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.625,
      "step": 1098
    },
    {
      "epoch": 1.3484662576687116,
      "grad_norm": 15.090380221292753,
      "learning_rate": 8.34297520661157e-07,
      "logits/chosen": -0.09619140625,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -354.0,
      "logps/rejected": -344.0,
      "loss": 0.0831,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.130859375,
      "rewards/margins": 9.8125,
      "rewards/rejected": -9.9375,
      "step": 1099
    },
    {
      "epoch": 1.3496932515337423,
      "grad_norm": 8.957606377667311,
      "learning_rate": 8.336776859504132e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.37890625,
      "logps/chosen": -406.0,
      "logps/rejected": -374.0,
      "loss": 0.0292,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 10.875,
      "rewards/rejected": -9.9375,
      "step": 1100
    },
    {
      "epoch": 1.350920245398773,
      "grad_norm": 12.12291867951929,
      "learning_rate": 8.330578512396695e-07,
      "logits/chosen": 0.0400390625,
      "logits/rejected": -0.1171875,
      "logps/chosen": -370.0,
      "logps/rejected": -348.0,
      "loss": 0.0654,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.4921875,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.25,
      "step": 1101
    },
    {
      "epoch": 1.3521472392638036,
      "grad_norm": 18.332708975287847,
      "learning_rate": 8.324380165289256e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.431640625,
      "logps/chosen": -370.0,
      "logps/rejected": -352.0,
      "loss": 0.0716,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.671875,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.0625,
      "step": 1102
    },
    {
      "epoch": 1.3533742331288343,
      "grad_norm": 10.7430697279738,
      "learning_rate": 8.318181818181819e-07,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.275390625,
      "logps/chosen": -416.0,
      "logps/rejected": -400.0,
      "loss": 0.0442,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.3984375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.4375,
      "step": 1103
    },
    {
      "epoch": 1.354601226993865,
      "grad_norm": 12.033995818293517,
      "learning_rate": 8.311983471074381e-07,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2265625,
      "logps/chosen": -364.0,
      "logps/rejected": -374.0,
      "loss": 0.0536,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.9375,
      "step": 1104
    },
    {
      "epoch": 1.3558282208588956,
      "grad_norm": 4.495393110896528,
      "learning_rate": 8.305785123966943e-07,
      "logits/chosen": -0.2177734375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -408.0,
      "logps/rejected": -414.0,
      "loss": 0.0158,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.09375,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.25,
      "step": 1105
    },
    {
      "epoch": 1.3570552147239263,
      "grad_norm": 10.27676045940915,
      "learning_rate": 8.299586776859504e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.263671875,
      "logps/chosen": -360.0,
      "logps/rejected": -374.0,
      "loss": 0.0323,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.177734375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.5,
      "step": 1106
    },
    {
      "epoch": 1.358282208588957,
      "grad_norm": 15.415657480989857,
      "learning_rate": 8.293388429752065e-07,
      "logits/chosen": 0.03759765625,
      "logits/rejected": -0.07861328125,
      "logps/chosen": -402.0,
      "logps/rejected": -374.0,
      "loss": 0.0477,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.76953125,
      "rewards/margins": 10.3125,
      "rewards/rejected": -9.5625,
      "step": 1107
    },
    {
      "epoch": 1.3595092024539877,
      "grad_norm": 5.235763636929729,
      "learning_rate": 8.287190082644628e-07,
      "logits/chosen": -0.1552734375,
      "logits/rejected": -0.3359375,
      "logps/chosen": -440.0,
      "logps/rejected": -402.0,
      "loss": 0.0197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1484375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.0,
      "step": 1108
    },
    {
      "epoch": 1.3607361963190185,
      "grad_norm": 10.622377911792848,
      "learning_rate": 8.28099173553719e-07,
      "logits/chosen": -0.0361328125,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -400.0,
      "logps/rejected": -382.0,
      "loss": 0.0409,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6328125,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.875,
      "step": 1109
    },
    {
      "epoch": 1.3619631901840492,
      "grad_norm": 15.219013654594981,
      "learning_rate": 8.274793388429753e-07,
      "logits/chosen": -0.11572265625,
      "logits/rejected": -0.330078125,
      "logps/chosen": -398.0,
      "logps/rejected": -392.0,
      "loss": 0.0485,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.482421875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.9375,
      "step": 1110
    },
    {
      "epoch": 1.3631901840490799,
      "grad_norm": 5.052963784815555,
      "learning_rate": 8.268595041322314e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -432.0,
      "logps/rejected": -380.0,
      "loss": 0.0166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6640625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.625,
      "step": 1111
    },
    {
      "epoch": 1.3644171779141105,
      "grad_norm": 13.661073432102611,
      "learning_rate": 8.262396694214877e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.23046875,
      "logps/chosen": -418.0,
      "logps/rejected": -406.0,
      "loss": 0.0562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.298828125,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.75,
      "step": 1112
    },
    {
      "epoch": 1.3656441717791412,
      "grad_norm": 11.164562514831367,
      "learning_rate": 8.256198347107439e-07,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.16015625,
      "logps/chosen": -356.0,
      "logps/rejected": -360.0,
      "loss": 0.0353,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -9.9375,
      "step": 1113
    },
    {
      "epoch": 1.3668711656441719,
      "grad_norm": 12.65975505597321,
      "learning_rate": 8.25e-07,
      "logits/chosen": -0.25390625,
      "logits/rejected": -0.3671875,
      "logps/chosen": -344.0,
      "logps/rejected": -358.0,
      "loss": 0.0419,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.45703125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.5,
      "step": 1114
    },
    {
      "epoch": 1.3680981595092025,
      "grad_norm": 9.32147436119456,
      "learning_rate": 8.243801652892562e-07,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.25,
      "logps/chosen": -376.0,
      "logps/rejected": -368.0,
      "loss": 0.0374,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.388671875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.8125,
      "step": 1115
    },
    {
      "epoch": 1.3693251533742332,
      "grad_norm": 7.035037640219457,
      "learning_rate": 8.237603305785123e-07,
      "logits/chosen": -0.06982421875,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -430.0,
      "logps/rejected": -402.0,
      "loss": 0.0362,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.326171875,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.6875,
      "step": 1116
    },
    {
      "epoch": 1.3705521472392639,
      "grad_norm": 7.072591615437579,
      "learning_rate": 8.231404958677686e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.47265625,
      "logps/chosen": -434.0,
      "logps/rejected": -400.0,
      "loss": 0.028,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.671875,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.1875,
      "step": 1117
    },
    {
      "epoch": 1.3717791411042946,
      "grad_norm": 15.059088219830123,
      "learning_rate": 8.225206611570248e-07,
      "logits/chosen": -0.056884765625,
      "logits/rejected": -0.2294921875,
      "logps/chosen": -380.0,
      "logps/rejected": -382.0,
      "loss": 0.0791,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.40234375,
      "rewards/margins": 10.5,
      "rewards/rejected": -10.875,
      "step": 1118
    },
    {
      "epoch": 1.3730061349693252,
      "grad_norm": 7.182852167794888,
      "learning_rate": 8.21900826446281e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.328125,
      "logps/chosen": -424.0,
      "logps/rejected": -420.0,
      "loss": 0.0365,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.0,
      "step": 1119
    },
    {
      "epoch": 1.3742331288343559,
      "grad_norm": 6.6804769663365855,
      "learning_rate": 8.212809917355372e-07,
      "logits/chosen": -0.1103515625,
      "logits/rejected": -0.400390625,
      "logps/chosen": -430.0,
      "logps/rejected": -398.0,
      "loss": 0.0239,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.58984375,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.4375,
      "step": 1120
    },
    {
      "epoch": 1.3754601226993866,
      "grad_norm": 16.092490568103823,
      "learning_rate": 8.206611570247935e-07,
      "logits/chosen": -0.216796875,
      "logits/rejected": -0.375,
      "logps/chosen": -444.0,
      "logps/rejected": -406.0,
      "loss": 0.0841,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1826171875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.125,
      "step": 1121
    },
    {
      "epoch": 1.3766871165644172,
      "grad_norm": 45.62488030196555,
      "learning_rate": 8.200413223140497e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.361328125,
      "logps/chosen": -372.0,
      "logps/rejected": -374.0,
      "loss": 0.1206,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -1.2421875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -11.375,
      "step": 1122
    },
    {
      "epoch": 1.377914110429448,
      "grad_norm": 7.987049229922856,
      "learning_rate": 8.194214876033058e-07,
      "logits/chosen": -0.23046875,
      "logits/rejected": -0.380859375,
      "logps/chosen": -446.0,
      "logps/rejected": -384.0,
      "loss": 0.0291,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.59765625,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.625,
      "step": 1123
    },
    {
      "epoch": 1.3791411042944786,
      "grad_norm": 10.84925435010219,
      "learning_rate": 8.188016528925619e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.33203125,
      "logps/chosen": -398.0,
      "logps/rejected": -372.0,
      "loss": 0.045,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 11.4375,
      "rewards/rejected": -12.0,
      "step": 1124
    },
    {
      "epoch": 1.3803680981595092,
      "grad_norm": 12.438670127459194,
      "learning_rate": 8.181818181818181e-07,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.3671875,
      "logps/chosen": -388.0,
      "logps/rejected": -382.0,
      "loss": 0.0514,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.70703125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.4375,
      "step": 1125
    },
    {
      "epoch": 1.38159509202454,
      "grad_norm": 7.702209911740994,
      "learning_rate": 8.175619834710744e-07,
      "logits/chosen": -0.21875,
      "logits/rejected": -0.345703125,
      "logps/chosen": -378.0,
      "logps/rejected": -372.0,
      "loss": 0.0244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.9375,
      "step": 1126
    },
    {
      "epoch": 1.3828220858895706,
      "grad_norm": 9.98545953456932,
      "learning_rate": 8.169421487603306e-07,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.392578125,
      "logps/chosen": -452.0,
      "logps/rejected": -438.0,
      "loss": 0.0506,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.515625,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.5625,
      "step": 1127
    },
    {
      "epoch": 1.3840490797546012,
      "grad_norm": 14.966530393829544,
      "learning_rate": 8.163223140495868e-07,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.294921875,
      "logps/chosen": -398.0,
      "logps/rejected": -416.0,
      "loss": 0.0573,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.83203125,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.875,
      "step": 1128
    },
    {
      "epoch": 1.385276073619632,
      "grad_norm": 6.960083817877731,
      "learning_rate": 8.15702479338843e-07,
      "logits/chosen": -0.0244140625,
      "logits/rejected": -0.166015625,
      "logps/chosen": -400.0,
      "logps/rejected": -372.0,
      "loss": 0.022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29296875,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.9375,
      "step": 1129
    },
    {
      "epoch": 1.3865030674846626,
      "grad_norm": 15.732715609813184,
      "learning_rate": 8.150826446280993e-07,
      "logits/chosen": 0.0286865234375,
      "logits/rejected": -0.1923828125,
      "logps/chosen": -438.0,
      "logps/rejected": -408.0,
      "loss": 0.0498,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.361328125,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.9375,
      "step": 1130
    },
    {
      "epoch": 1.3877300613496932,
      "grad_norm": 5.221583127777315,
      "learning_rate": 8.144628099173554e-07,
      "logits/chosen": -0.232421875,
      "logits/rejected": -0.5234375,
      "logps/chosen": -434.0,
      "logps/rejected": -400.0,
      "loss": 0.0197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04541015625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.9375,
      "step": 1131
    },
    {
      "epoch": 1.388957055214724,
      "grad_norm": 12.813444795393385,
      "learning_rate": 8.138429752066116e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.36328125,
      "logps/chosen": -494.0,
      "logps/rejected": -408.0,
      "loss": 0.0313,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.890625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.375,
      "step": 1132
    },
    {
      "epoch": 1.3901840490797546,
      "grad_norm": 15.660321100678845,
      "learning_rate": 8.132231404958677e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": -0.34765625,
      "logps/chosen": -410.0,
      "logps/rejected": -410.0,
      "loss": 0.0504,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.72265625,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.125,
      "step": 1133
    },
    {
      "epoch": 1.3914110429447852,
      "grad_norm": 7.9073668160942745,
      "learning_rate": 8.126033057851239e-07,
      "logits/chosen": -0.1962890625,
      "logits/rejected": -0.404296875,
      "logps/chosen": -324.0,
      "logps/rejected": -340.0,
      "loss": 0.0393,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.9375,
      "step": 1134
    },
    {
      "epoch": 1.392638036809816,
      "grad_norm": 7.004247584369281,
      "learning_rate": 8.119834710743802e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.3359375,
      "logps/chosen": -410.0,
      "logps/rejected": -374.0,
      "loss": 0.0317,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.69140625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.875,
      "step": 1135
    },
    {
      "epoch": 1.3938650306748466,
      "grad_norm": 13.652544230864027,
      "learning_rate": 8.113636363636363e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": -0.4375,
      "logps/chosen": -466.0,
      "logps/rejected": -408.0,
      "loss": 0.0464,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 12.125,
      "rewards/rejected": -10.9375,
      "step": 1136
    },
    {
      "epoch": 1.3950920245398772,
      "grad_norm": 13.746874338763654,
      "learning_rate": 8.107438016528926e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.48046875,
      "logps/chosen": -466.0,
      "logps/rejected": -406.0,
      "loss": 0.0681,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.345703125,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.5625,
      "step": 1137
    },
    {
      "epoch": 1.396319018404908,
      "grad_norm": 13.137416168579717,
      "learning_rate": 8.101239669421488e-07,
      "logits/chosen": -0.05126953125,
      "logits/rejected": -0.25,
      "logps/chosen": -386.0,
      "logps/rejected": -356.0,
      "loss": 0.0423,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.051513671875,
      "rewards/margins": 9.875,
      "rewards/rejected": -9.8125,
      "step": 1138
    },
    {
      "epoch": 1.3975460122699386,
      "grad_norm": 14.313209901423209,
      "learning_rate": 8.095041322314051e-07,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.404296875,
      "logps/chosen": -360.0,
      "logps/rejected": -368.0,
      "loss": 0.0629,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2392578125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.9375,
      "step": 1139
    },
    {
      "epoch": 1.3987730061349692,
      "grad_norm": 12.684020520423767,
      "learning_rate": 8.088842975206612e-07,
      "logits/chosen": -0.10498046875,
      "logits/rejected": -0.232421875,
      "logps/chosen": -352.0,
      "logps/rejected": -360.0,
      "loss": 0.0614,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.2294921875,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.8125,
      "step": 1140
    },
    {
      "epoch": 1.4,
      "grad_norm": 8.366317687764122,
      "learning_rate": 8.082644628099174e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.396484375,
      "logps/chosen": -354.0,
      "logps/rejected": -356.0,
      "loss": 0.0322,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.154296875,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.25,
      "step": 1141
    },
    {
      "epoch": 1.4012269938650306,
      "grad_norm": 9.78962851865542,
      "learning_rate": 8.076446280991735e-07,
      "logits/chosen": -0.057861328125,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -346.0,
      "logps/rejected": -382.0,
      "loss": 0.0316,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.34765625,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.625,
      "step": 1142
    },
    {
      "epoch": 1.4024539877300612,
      "grad_norm": 15.82156683988277,
      "learning_rate": 8.070247933884297e-07,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.248046875,
      "logps/chosen": -396.0,
      "logps/rejected": -416.0,
      "loss": 0.0618,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.25,
      "step": 1143
    },
    {
      "epoch": 1.403680981595092,
      "grad_norm": 17.350320312048925,
      "learning_rate": 8.06404958677686e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": -0.38671875,
      "logps/chosen": -396.0,
      "logps/rejected": -388.0,
      "loss": 0.0738,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.169921875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.125,
      "step": 1144
    },
    {
      "epoch": 1.4049079754601226,
      "grad_norm": 13.43693939499879,
      "learning_rate": 8.057851239669421e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -456.0,
      "logps/rejected": -434.0,
      "loss": 0.0376,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.86328125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -10.9375,
      "step": 1145
    },
    {
      "epoch": 1.4061349693251532,
      "grad_norm": 10.84486528809094,
      "learning_rate": 8.051652892561984e-07,
      "logits/chosen": -0.046875,
      "logits/rejected": -0.16796875,
      "logps/chosen": -376.0,
      "logps/rejected": -400.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.05322265625,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.6875,
      "step": 1146
    },
    {
      "epoch": 1.407361963190184,
      "grad_norm": 7.584140894053648,
      "learning_rate": 8.045454545454546e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.361328125,
      "logps/chosen": -394.0,
      "logps/rejected": -386.0,
      "loss": 0.0283,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.5,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.625,
      "step": 1147
    },
    {
      "epoch": 1.4085889570552146,
      "grad_norm": 9.922196775746215,
      "learning_rate": 8.039256198347108e-07,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -370.0,
      "logps/rejected": -378.0,
      "loss": 0.0426,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.052978515625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.25,
      "step": 1148
    },
    {
      "epoch": 1.4098159509202455,
      "grad_norm": 6.148272524003038,
      "learning_rate": 8.03305785123967e-07,
      "logits/chosen": -0.140625,
      "logits/rejected": -0.30859375,
      "logps/chosen": -384.0,
      "logps/rejected": -398.0,
      "loss": 0.0231,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.35546875,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.5,
      "step": 1149
    },
    {
      "epoch": 1.4110429447852761,
      "grad_norm": 6.406336886570724,
      "learning_rate": 8.026859504132232e-07,
      "logits/chosen": -0.06689453125,
      "logits/rejected": -0.240234375,
      "logps/chosen": -390.0,
      "logps/rejected": -386.0,
      "loss": 0.0284,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.0020904541015625,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.125,
      "step": 1150
    },
    {
      "epoch": 1.4122699386503068,
      "grad_norm": 5.500976110615787,
      "learning_rate": 8.020661157024793e-07,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.330078125,
      "logps/chosen": -388.0,
      "logps/rejected": -406.0,
      "loss": 0.022,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.138671875,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.75,
      "step": 1151
    },
    {
      "epoch": 1.4134969325153375,
      "grad_norm": 15.46899721034435,
      "learning_rate": 8.014462809917355e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.318359375,
      "logps/chosen": -420.0,
      "logps/rejected": -436.0,
      "loss": 0.0474,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.5859375,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.6875,
      "step": 1152
    },
    {
      "epoch": 1.4147239263803681,
      "grad_norm": 29.448301297633176,
      "learning_rate": 8.008264462809917e-07,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.26171875,
      "logps/chosen": -406.0,
      "logps/rejected": -406.0,
      "loss": 0.0778,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.349609375,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.0,
      "step": 1153
    },
    {
      "epoch": 1.4159509202453988,
      "grad_norm": 8.55335933719368,
      "learning_rate": 8.002066115702479e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.373046875,
      "logps/chosen": -446.0,
      "logps/rejected": -424.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.62109375,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.0625,
      "step": 1154
    },
    {
      "epoch": 1.4171779141104295,
      "grad_norm": 11.91106417252758,
      "learning_rate": 7.995867768595042e-07,
      "logits/chosen": -0.1962890625,
      "logits/rejected": -0.310546875,
      "logps/chosen": -378.0,
      "logps/rejected": -386.0,
      "loss": 0.0437,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.146484375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.9375,
      "step": 1155
    },
    {
      "epoch": 1.4184049079754601,
      "grad_norm": 14.345078375136291,
      "learning_rate": 7.989669421487604e-07,
      "logits/chosen": -0.1103515625,
      "logits/rejected": -0.369140625,
      "logps/chosen": -440.0,
      "logps/rejected": -396.0,
      "loss": 0.0455,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7421875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.625,
      "step": 1156
    },
    {
      "epoch": 1.4196319018404908,
      "grad_norm": 13.702503887945511,
      "learning_rate": 7.983471074380166e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.28125,
      "logps/chosen": -450.0,
      "logps/rejected": -424.0,
      "loss": 0.0482,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.93359375,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.0,
      "step": 1157
    },
    {
      "epoch": 1.4208588957055215,
      "grad_norm": 8.654555323915172,
      "learning_rate": 7.977272727272728e-07,
      "logits/chosen": -0.1650390625,
      "logits/rejected": -0.306640625,
      "logps/chosen": -346.0,
      "logps/rejected": -354.0,
      "loss": 0.0362,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.498046875,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.875,
      "step": 1158
    },
    {
      "epoch": 1.4220858895705522,
      "grad_norm": 14.010487033573805,
      "learning_rate": 7.97107438016529e-07,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.1875,
      "logps/chosen": -354.0,
      "logps/rejected": -388.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.10986328125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.8125,
      "step": 1159
    },
    {
      "epoch": 1.4233128834355828,
      "grad_norm": 16.133153822756856,
      "learning_rate": 7.964876033057851e-07,
      "logits/chosen": -0.2373046875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -414.0,
      "logps/rejected": -398.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.23828125,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.875,
      "step": 1160
    },
    {
      "epoch": 1.4245398773006135,
      "grad_norm": 9.461066515172439,
      "learning_rate": 7.958677685950413e-07,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.1962890625,
      "logps/chosen": -378.0,
      "logps/rejected": -414.0,
      "loss": 0.07,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.07421875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.625,
      "step": 1161
    },
    {
      "epoch": 1.4257668711656442,
      "grad_norm": 13.085163631657917,
      "learning_rate": 7.952479338842975e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.314453125,
      "logps/chosen": -394.0,
      "logps/rejected": -382.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.059814453125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.75,
      "step": 1162
    },
    {
      "epoch": 1.4269938650306748,
      "grad_norm": 9.56097265388881,
      "learning_rate": 7.946280991735537e-07,
      "logits/chosen": -0.279296875,
      "logits/rejected": -0.52734375,
      "logps/chosen": -402.0,
      "logps/rejected": -378.0,
      "loss": 0.0236,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.341796875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.25,
      "step": 1163
    },
    {
      "epoch": 1.4282208588957055,
      "grad_norm": 16.722388625925955,
      "learning_rate": 7.9400826446281e-07,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.126953125,
      "logps/chosen": -336.0,
      "logps/rejected": -368.0,
      "loss": 0.0682,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.017822265625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.5,
      "step": 1164
    },
    {
      "epoch": 1.4294478527607362,
      "grad_norm": 7.348322611328401,
      "learning_rate": 7.933884297520661e-07,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -388.0,
      "logps/rejected": -370.0,
      "loss": 0.0227,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.08154296875,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.625,
      "step": 1165
    },
    {
      "epoch": 1.4306748466257668,
      "grad_norm": 5.34988661629868,
      "learning_rate": 7.927685950413224e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.369140625,
      "logps/chosen": -410.0,
      "logps/rejected": -386.0,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2353515625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.3125,
      "step": 1166
    },
    {
      "epoch": 1.4319018404907975,
      "grad_norm": 2.351503468437324,
      "learning_rate": 7.921487603305786e-07,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.3515625,
      "logps/chosen": -404.0,
      "logps/rejected": -410.0,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2734375,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.4375,
      "step": 1167
    },
    {
      "epoch": 1.4331288343558282,
      "grad_norm": 10.275425558023933,
      "learning_rate": 7.915289256198348e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -372.0,
      "logps/rejected": -374.0,
      "loss": 0.0434,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.357421875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.5625,
      "step": 1168
    },
    {
      "epoch": 1.4343558282208588,
      "grad_norm": 9.69113419509306,
      "learning_rate": 7.909090909090909e-07,
      "logits/chosen": -0.07568359375,
      "logits/rejected": -0.236328125,
      "logps/chosen": -376.0,
      "logps/rejected": -398.0,
      "loss": 0.0378,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.390625,
      "rewards/margins": 10.25,
      "rewards/rejected": -9.875,
      "step": 1169
    },
    {
      "epoch": 1.4355828220858895,
      "grad_norm": 11.100318005337881,
      "learning_rate": 7.90289256198347e-07,
      "logits/chosen": -0.171875,
      "logits/rejected": -0.421875,
      "logps/chosen": -396.0,
      "logps/rejected": -364.0,
      "loss": 0.0327,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.095703125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.75,
      "step": 1170
    },
    {
      "epoch": 1.4368098159509202,
      "grad_norm": 13.42384166771764,
      "learning_rate": 7.896694214876033e-07,
      "logits/chosen": -0.126953125,
      "logits/rejected": -0.28515625,
      "logps/chosen": -380.0,
      "logps/rejected": -372.0,
      "loss": 0.0555,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.3125,
      "step": 1171
    },
    {
      "epoch": 1.438036809815951,
      "grad_norm": 16.33374781104785,
      "learning_rate": 7.890495867768595e-07,
      "logits/chosen": -0.055419921875,
      "logits/rejected": -0.173828125,
      "logps/chosen": -436.0,
      "logps/rejected": -400.0,
      "loss": 0.0706,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2421875,
      "rewards/margins": 9.25,
      "rewards/rejected": -9.4375,
      "step": 1172
    },
    {
      "epoch": 1.4392638036809817,
      "grad_norm": 17.67963845713936,
      "learning_rate": 7.884297520661158e-07,
      "logits/chosen": -0.125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -428.0,
      "logps/rejected": -410.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.275390625,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.625,
      "step": 1173
    },
    {
      "epoch": 1.4404907975460124,
      "grad_norm": 5.1384201359953,
      "learning_rate": 7.878099173553719e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.166015625,
      "logps/chosen": -362.0,
      "logps/rejected": -366.0,
      "loss": 0.0129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.017578125,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.5625,
      "step": 1174
    },
    {
      "epoch": 1.441717791411043,
      "grad_norm": 5.148940884458012,
      "learning_rate": 7.871900826446282e-07,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.173828125,
      "logps/chosen": -346.0,
      "logps/rejected": -364.0,
      "loss": 0.0273,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.35546875,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.375,
      "step": 1175
    },
    {
      "epoch": 1.4429447852760737,
      "grad_norm": 11.603523639095071,
      "learning_rate": 7.865702479338844e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.310546875,
      "logps/chosen": -392.0,
      "logps/rejected": -402.0,
      "loss": 0.0389,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.306640625,
      "rewards/margins": 9.875,
      "rewards/rejected": -10.1875,
      "step": 1176
    },
    {
      "epoch": 1.4441717791411044,
      "grad_norm": 10.040926511598048,
      "learning_rate": 7.859504132231406e-07,
      "logits/chosen": -0.0517578125,
      "logits/rejected": -0.244140625,
      "logps/chosen": -384.0,
      "logps/rejected": -354.0,
      "loss": 0.0383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04296875,
      "rewards/margins": 9.5,
      "rewards/rejected": -9.5,
      "step": 1177
    },
    {
      "epoch": 1.445398773006135,
      "grad_norm": 11.028023088957683,
      "learning_rate": 7.853305785123967e-07,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.35546875,
      "logps/chosen": -396.0,
      "logps/rejected": -364.0,
      "loss": 0.0425,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -9.875,
      "step": 1178
    },
    {
      "epoch": 1.4466257668711657,
      "grad_norm": 8.150158411239879,
      "learning_rate": 7.847107438016528e-07,
      "logits/chosen": -0.2412109375,
      "logits/rejected": -0.443359375,
      "logps/chosen": -438.0,
      "logps/rejected": -394.0,
      "loss": 0.0374,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.515625,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.1875,
      "step": 1179
    },
    {
      "epoch": 1.4478527607361964,
      "grad_norm": 6.894255501689617,
      "learning_rate": 7.840909090909091e-07,
      "logits/chosen": -0.1279296875,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -344.0,
      "logps/rejected": -386.0,
      "loss": 0.0309,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.05029296875,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.6875,
      "step": 1180
    },
    {
      "epoch": 1.449079754601227,
      "grad_norm": 8.019249451553959,
      "learning_rate": 7.834710743801653e-07,
      "logits/chosen": -0.162109375,
      "logits/rejected": -0.2890625,
      "logps/chosen": -434.0,
      "logps/rejected": -430.0,
      "loss": 0.0239,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.87890625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -9.9375,
      "step": 1181
    },
    {
      "epoch": 1.4503067484662577,
      "grad_norm": 8.340607467826109,
      "learning_rate": 7.828512396694216e-07,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -426.0,
      "logps/rejected": -404.0,
      "loss": 0.0307,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -10.625,
      "step": 1182
    },
    {
      "epoch": 1.4515337423312884,
      "grad_norm": 11.72673950966423,
      "learning_rate": 7.822314049586777e-07,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.19921875,
      "logps/chosen": -326.0,
      "logps/rejected": -372.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.06787109375,
      "rewards/margins": 9.875,
      "rewards/rejected": -9.9375,
      "step": 1183
    },
    {
      "epoch": 1.452760736196319,
      "grad_norm": 9.058514886167021,
      "learning_rate": 7.81611570247934e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.34765625,
      "logps/chosen": -366.0,
      "logps/rejected": -352.0,
      "loss": 0.0407,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.330078125,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.625,
      "step": 1184
    },
    {
      "epoch": 1.4539877300613497,
      "grad_norm": 11.422467086533596,
      "learning_rate": 7.809917355371902e-07,
      "logits/chosen": -0.041259765625,
      "logits/rejected": -0.19140625,
      "logps/chosen": -420.0,
      "logps/rejected": -402.0,
      "loss": 0.0448,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.5625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.25,
      "step": 1185
    },
    {
      "epoch": 1.4552147239263804,
      "grad_norm": 9.735079419768947,
      "learning_rate": 7.803719008264463e-07,
      "logits/chosen": -0.11328125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -378.0,
      "logps/rejected": -390.0,
      "loss": 0.0379,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.11328125,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.25,
      "step": 1186
    },
    {
      "epoch": 1.456441717791411,
      "grad_norm": 6.162772380389009,
      "learning_rate": 7.797520661157025e-07,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.3359375,
      "logps/chosen": -456.0,
      "logps/rejected": -410.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.6875,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.6875,
      "step": 1187
    },
    {
      "epoch": 1.4576687116564417,
      "grad_norm": 15.719129543958939,
      "learning_rate": 7.791322314049586e-07,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -432.0,
      "logps/rejected": -434.0,
      "loss": 0.0721,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.08447265625,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.125,
      "step": 1188
    },
    {
      "epoch": 1.4588957055214724,
      "grad_norm": 12.022319651435222,
      "learning_rate": 7.785123966942149e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -428.0,
      "logps/rejected": -400.0,
      "loss": 0.082,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.380859375,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.5,
      "step": 1189
    },
    {
      "epoch": 1.460122699386503,
      "grad_norm": 10.515256662579745,
      "learning_rate": 7.778925619834711e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.39453125,
      "logps/chosen": -436.0,
      "logps/rejected": -410.0,
      "loss": 0.0408,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.80078125,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.3125,
      "step": 1190
    },
    {
      "epoch": 1.4613496932515337,
      "grad_norm": 14.208293904428151,
      "learning_rate": 7.772727272727273e-07,
      "logits/chosen": 0.002960205078125,
      "logits/rejected": -0.13671875,
      "logps/chosen": -370.0,
      "logps/rejected": -390.0,
      "loss": 0.0511,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.140625,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.25,
      "step": 1191
    },
    {
      "epoch": 1.4625766871165644,
      "grad_norm": 4.726495134099289,
      "learning_rate": 7.766528925619835e-07,
      "logits/chosen": -0.1162109375,
      "logits/rejected": -0.29296875,
      "logps/chosen": -400.0,
      "logps/rejected": -406.0,
      "loss": 0.0213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0208740234375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.375,
      "step": 1192
    },
    {
      "epoch": 1.463803680981595,
      "grad_norm": 7.11137377163575,
      "learning_rate": 7.760330578512398e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.373046875,
      "logps/chosen": -458.0,
      "logps/rejected": -420.0,
      "loss": 0.0249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.0,
      "step": 1193
    },
    {
      "epoch": 1.4650306748466257,
      "grad_norm": 7.389252182902934,
      "learning_rate": 7.75413223140496e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.275390625,
      "logps/chosen": -420.0,
      "logps/rejected": -404.0,
      "loss": 0.0133,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.326171875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.625,
      "step": 1194
    },
    {
      "epoch": 1.4662576687116564,
      "grad_norm": 10.994717707834955,
      "learning_rate": 7.74793388429752e-07,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -402.0,
      "logps/rejected": -410.0,
      "loss": 0.0418,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.08251953125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.8125,
      "step": 1195
    },
    {
      "epoch": 1.467484662576687,
      "grad_norm": 8.858341862725984,
      "learning_rate": 7.741735537190082e-07,
      "logits/chosen": -0.04638671875,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -390.0,
      "logps/rejected": -392.0,
      "loss": 0.0296,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1884765625,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.3125,
      "step": 1196
    },
    {
      "epoch": 1.4687116564417177,
      "grad_norm": 15.001467453574548,
      "learning_rate": 7.735537190082644e-07,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -422.0,
      "logps/rejected": -382.0,
      "loss": 0.0572,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.05810546875,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.8125,
      "step": 1197
    },
    {
      "epoch": 1.4699386503067484,
      "grad_norm": 11.467996330718492,
      "learning_rate": 7.729338842975207e-07,
      "logits/chosen": 0.01531982421875,
      "logits/rejected": -0.126953125,
      "logps/chosen": -384.0,
      "logps/rejected": -356.0,
      "loss": 0.0766,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.07177734375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.6875,
      "step": 1198
    },
    {
      "epoch": 1.471165644171779,
      "grad_norm": 8.297507128344717,
      "learning_rate": 7.723140495867769e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.3671875,
      "logps/chosen": -394.0,
      "logps/rejected": -376.0,
      "loss": 0.0247,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.376953125,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.1875,
      "step": 1199
    },
    {
      "epoch": 1.4723926380368098,
      "grad_norm": 6.708462240020914,
      "learning_rate": 7.716942148760331e-07,
      "logits/chosen": -0.13671875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -330.0,
      "logps/rejected": -350.0,
      "loss": 0.0204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1572265625,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.25,
      "step": 1200
    },
    {
      "epoch": 1.4736196319018404,
      "grad_norm": 10.300151959699189,
      "learning_rate": 7.710743801652893e-07,
      "logits/chosen": -0.041015625,
      "logits/rejected": -0.18359375,
      "logps/chosen": -378.0,
      "logps/rejected": -410.0,
      "loss": 0.0294,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.6875,
      "step": 1201
    },
    {
      "epoch": 1.474846625766871,
      "grad_norm": 10.130166213309426,
      "learning_rate": 7.704545454545456e-07,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -428.0,
      "logps/rejected": -422.0,
      "loss": 0.0315,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.75,
      "step": 1202
    },
    {
      "epoch": 1.4760736196319018,
      "grad_norm": 9.219567443331156,
      "learning_rate": 7.698347107438017e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.408203125,
      "logps/chosen": -410.0,
      "logps/rejected": -372.0,
      "loss": 0.0433,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.2060546875,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.9375,
      "step": 1203
    },
    {
      "epoch": 1.4773006134969324,
      "grad_norm": 9.51752284318554,
      "learning_rate": 7.692148760330578e-07,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -374.0,
      "logps/rejected": -392.0,
      "loss": 0.041,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.546875,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.625,
      "step": 1204
    },
    {
      "epoch": 1.478527607361963,
      "grad_norm": 23.546428660464922,
      "learning_rate": 7.68595041322314e-07,
      "logits/chosen": 0.02734375,
      "logits/rejected": -0.2021484375,
      "logps/chosen": -364.0,
      "logps/rejected": -378.0,
      "loss": 0.1268,
      "rewards/accuracies": 0.9453125,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.1875,
      "step": 1205
    },
    {
      "epoch": 1.4797546012269938,
      "grad_norm": 6.082107723283603,
      "learning_rate": 7.679752066115702e-07,
      "logits/chosen": -0.02685546875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -350.0,
      "logps/rejected": -344.0,
      "loss": 0.0275,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.0791015625,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.1875,
      "step": 1206
    },
    {
      "epoch": 1.4809815950920244,
      "grad_norm": 10.622559047787306,
      "learning_rate": 7.673553719008265e-07,
      "logits/chosen": -0.09375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.0353,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.05859375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.875,
      "step": 1207
    },
    {
      "epoch": 1.482208588957055,
      "grad_norm": 13.683817866460686,
      "learning_rate": 7.667355371900826e-07,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -376.0,
      "logps/rejected": -382.0,
      "loss": 0.0619,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.11474609375,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.3125,
      "step": 1208
    },
    {
      "epoch": 1.4834355828220858,
      "grad_norm": 8.88934565774754,
      "learning_rate": 7.661157024793389e-07,
      "logits/chosen": -0.08154296875,
      "logits/rejected": -0.26171875,
      "logps/chosen": -362.0,
      "logps/rejected": -364.0,
      "loss": 0.042,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.0166015625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.375,
      "step": 1209
    },
    {
      "epoch": 1.4846625766871164,
      "grad_norm": 13.517803461526613,
      "learning_rate": 7.654958677685951e-07,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.19140625,
      "logps/chosen": -366.0,
      "logps/rejected": -364.0,
      "loss": 0.048,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -10.875,
      "step": 1210
    },
    {
      "epoch": 1.4858895705521473,
      "grad_norm": 3.0410283674490257,
      "learning_rate": 7.648760330578514e-07,
      "logits/chosen": -0.06640625,
      "logits/rejected": -0.283203125,
      "logps/chosen": -358.0,
      "logps/rejected": -382.0,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2021484375,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.8125,
      "step": 1211
    },
    {
      "epoch": 1.487116564417178,
      "grad_norm": 5.560187356994683,
      "learning_rate": 7.642561983471075e-07,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.28125,
      "logps/chosen": -356.0,
      "logps/rejected": -386.0,
      "loss": 0.0259,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12158203125,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.375,
      "step": 1212
    },
    {
      "epoch": 1.4883435582822087,
      "grad_norm": 7.567720346296838,
      "learning_rate": 7.636363636363635e-07,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.42578125,
      "logps/chosen": -432.0,
      "logps/rejected": -396.0,
      "loss": 0.0322,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.7890625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.75,
      "step": 1213
    },
    {
      "epoch": 1.4895705521472393,
      "grad_norm": 10.044279259721861,
      "learning_rate": 7.630165289256198e-07,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.2421875,
      "logps/chosen": -348.0,
      "logps/rejected": -380.0,
      "loss": 0.0415,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.369140625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.4375,
      "step": 1214
    },
    {
      "epoch": 1.49079754601227,
      "grad_norm": 11.888900857643913,
      "learning_rate": 7.62396694214876e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.390625,
      "logps/chosen": -382.0,
      "logps/rejected": -392.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.9375,
      "step": 1215
    },
    {
      "epoch": 1.4920245398773007,
      "grad_norm": 11.234520606273014,
      "learning_rate": 7.617768595041323e-07,
      "logits/chosen": -0.140625,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -378.0,
      "logps/rejected": -370.0,
      "loss": 0.0389,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.396484375,
      "rewards/margins": 10.625,
      "rewards/rejected": -11.0,
      "step": 1216
    },
    {
      "epoch": 1.4932515337423313,
      "grad_norm": 5.600087268075799,
      "learning_rate": 7.611570247933884e-07,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.26953125,
      "logps/chosen": -404.0,
      "logps/rejected": -406.0,
      "loss": 0.0147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.078125,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.0,
      "step": 1217
    },
    {
      "epoch": 1.494478527607362,
      "grad_norm": 13.512440969064604,
      "learning_rate": 7.605371900826447e-07,
      "logits/chosen": -0.0634765625,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -374.0,
      "logps/rejected": -386.0,
      "loss": 0.0486,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.76171875,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.8125,
      "step": 1218
    },
    {
      "epoch": 1.4957055214723927,
      "grad_norm": 11.083011010632404,
      "learning_rate": 7.599173553719009e-07,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -394.0,
      "logps/rejected": -408.0,
      "loss": 0.0355,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1767578125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.9375,
      "step": 1219
    },
    {
      "epoch": 1.4969325153374233,
      "grad_norm": 12.719289884310554,
      "learning_rate": 7.59297520661157e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.271484375,
      "logps/chosen": -404.0,
      "logps/rejected": -420.0,
      "loss": 0.0511,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.5,
      "step": 1220
    },
    {
      "epoch": 1.498159509202454,
      "grad_norm": 11.10401374516513,
      "learning_rate": 7.586776859504133e-07,
      "logits/chosen": -0.2109375,
      "logits/rejected": -0.3671875,
      "logps/chosen": -412.0,
      "logps/rejected": -382.0,
      "loss": 0.038,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.087890625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.4375,
      "step": 1221
    },
    {
      "epoch": 1.4993865030674847,
      "grad_norm": 10.550256846021844,
      "learning_rate": 7.580578512396693e-07,
      "logits/chosen": -0.08935546875,
      "logits/rejected": -0.263671875,
      "logps/chosen": -342.0,
      "logps/rejected": -370.0,
      "loss": 0.0459,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.015625,
      "rewards/margins": 10.0625,
      "rewards/rejected": -11.0625,
      "step": 1222
    },
    {
      "epoch": 1.5006134969325153,
      "grad_norm": 4.907032447202447,
      "learning_rate": 7.574380165289256e-07,
      "logits/chosen": -0.11328125,
      "logits/rejected": -0.2421875,
      "logps/chosen": -428.0,
      "logps/rejected": -406.0,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4609375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.3125,
      "step": 1223
    },
    {
      "epoch": 1.501840490797546,
      "grad_norm": 7.212415747597125,
      "learning_rate": 7.568181818181818e-07,
      "logits/chosen": -0.1044921875,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -360.0,
      "logps/rejected": -374.0,
      "loss": 0.0386,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.96875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -10.8125,
      "step": 1224
    },
    {
      "epoch": 1.5030674846625767,
      "grad_norm": 5.207118696961701,
      "learning_rate": 7.56198347107438e-07,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.220703125,
      "logps/chosen": -362.0,
      "logps/rejected": -378.0,
      "loss": 0.0192,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35546875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.9375,
      "step": 1225
    },
    {
      "epoch": 1.5042944785276073,
      "grad_norm": 12.469106555425835,
      "learning_rate": 7.555785123966942e-07,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.34765625,
      "logps/chosen": -394.0,
      "logps/rejected": -384.0,
      "loss": 0.0364,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.3125,
      "step": 1226
    },
    {
      "epoch": 1.505521472392638,
      "grad_norm": 6.703095964441331,
      "learning_rate": 7.549586776859505e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.396484375,
      "logps/chosen": -398.0,
      "logps/rejected": -422.0,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.228515625,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.375,
      "step": 1227
    },
    {
      "epoch": 1.5067484662576687,
      "grad_norm": 20.182875127429853,
      "learning_rate": 7.543388429752067e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.251953125,
      "logps/chosen": -416.0,
      "logps/rejected": -426.0,
      "loss": 0.1114,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.458984375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.875,
      "step": 1228
    },
    {
      "epoch": 1.5079754601226993,
      "grad_norm": 7.168144452671762,
      "learning_rate": 7.537190082644629e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.34765625,
      "logps/chosen": -382.0,
      "logps/rejected": -394.0,
      "loss": 0.0249,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.330078125,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.3125,
      "step": 1229
    },
    {
      "epoch": 1.50920245398773,
      "grad_norm": 8.739084266889561,
      "learning_rate": 7.530991735537191e-07,
      "logits/chosen": -0.1220703125,
      "logits/rejected": -0.29296875,
      "logps/chosen": -370.0,
      "logps/rejected": -370.0,
      "loss": 0.041,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.0230712890625,
      "rewards/margins": 10.5,
      "rewards/rejected": -10.5625,
      "step": 1230
    },
    {
      "epoch": 1.510429447852761,
      "grad_norm": 14.93632729231571,
      "learning_rate": 7.524793388429751e-07,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.23046875,
      "logps/chosen": -356.0,
      "logps/rejected": -356.0,
      "loss": 0.0501,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.1796875,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.5625,
      "step": 1231
    },
    {
      "epoch": 1.5116564417177916,
      "grad_norm": 6.602158622702926,
      "learning_rate": 7.518595041322314e-07,
      "logits/chosen": -0.296875,
      "logits/rejected": -0.486328125,
      "logps/chosen": -400.0,
      "logps/rejected": -396.0,
      "loss": 0.0195,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.482421875,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.75,
      "step": 1232
    },
    {
      "epoch": 1.5128834355828222,
      "grad_norm": 6.2838103080905015,
      "learning_rate": 7.512396694214876e-07,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.37890625,
      "logps/chosen": -384.0,
      "logps/rejected": -374.0,
      "loss": 0.0189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.302734375,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.1875,
      "step": 1233
    },
    {
      "epoch": 1.514110429447853,
      "grad_norm": 18.0136951443616,
      "learning_rate": 7.506198347107438e-07,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.328125,
      "logps/chosen": -426.0,
      "logps/rejected": -394.0,
      "loss": 0.0583,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.65234375,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.6875,
      "step": 1234
    },
    {
      "epoch": 1.5153374233128836,
      "grad_norm": 7.846641320419013,
      "learning_rate": 7.5e-07,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.19140625,
      "logps/chosen": -392.0,
      "logps/rejected": -382.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.9609375,
      "rewards/margins": 10.375,
      "rewards/rejected": -11.375,
      "step": 1235
    },
    {
      "epoch": 1.5165644171779142,
      "grad_norm": 14.299732747614243,
      "learning_rate": 7.493801652892563e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.310546875,
      "logps/chosen": -392.0,
      "logps/rejected": -388.0,
      "loss": 0.0557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0291748046875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.8125,
      "step": 1236
    },
    {
      "epoch": 1.517791411042945,
      "grad_norm": 15.326417785804917,
      "learning_rate": 7.487603305785124e-07,
      "logits/chosen": -0.24609375,
      "logits/rejected": -0.40625,
      "logps/chosen": -434.0,
      "logps/rejected": -402.0,
      "loss": 0.0642,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.23046875,
      "rewards/margins": 12.125,
      "rewards/rejected": -12.3125,
      "step": 1237
    },
    {
      "epoch": 1.5190184049079756,
      "grad_norm": 10.442301727950197,
      "learning_rate": 7.481404958677685e-07,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -464.0,
      "logps/rejected": -396.0,
      "loss": 0.0259,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.875,
      "step": 1238
    },
    {
      "epoch": 1.5202453987730062,
      "grad_norm": 10.812028014205586,
      "learning_rate": 7.475206611570248e-07,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.40625,
      "logps/chosen": -404.0,
      "logps/rejected": -382.0,
      "loss": 0.0373,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.10302734375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.1875,
      "step": 1239
    },
    {
      "epoch": 1.521472392638037,
      "grad_norm": 7.399322537588399,
      "learning_rate": 7.46900826446281e-07,
      "logits/chosen": -0.0830078125,
      "logits/rejected": -0.28125,
      "logps/chosen": -410.0,
      "logps/rejected": -402.0,
      "loss": 0.0326,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.353515625,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.25,
      "step": 1240
    },
    {
      "epoch": 1.5226993865030676,
      "grad_norm": 11.737470905903368,
      "learning_rate": 7.462809917355373e-07,
      "logits/chosen": -0.033447265625,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -360.0,
      "logps/rejected": -368.0,
      "loss": 0.0451,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.46484375,
      "rewards/margins": 10.0625,
      "rewards/rejected": -10.5625,
      "step": 1241
    },
    {
      "epoch": 1.5239263803680982,
      "grad_norm": 11.68475946384956,
      "learning_rate": 7.456611570247933e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.42578125,
      "logps/chosen": -388.0,
      "logps/rejected": -386.0,
      "loss": 0.0484,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.5703125,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.8125,
      "step": 1242
    },
    {
      "epoch": 1.525153374233129,
      "grad_norm": 7.544728556493131,
      "learning_rate": 7.450413223140496e-07,
      "logits/chosen": -0.059814453125,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -334.0,
      "logps/rejected": -354.0,
      "loss": 0.028,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.73828125,
      "rewards/margins": 10.125,
      "rewards/rejected": -10.875,
      "step": 1243
    },
    {
      "epoch": 1.5263803680981596,
      "grad_norm": 8.662327485339514,
      "learning_rate": 7.444214876033058e-07,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.16796875,
      "logps/chosen": -342.0,
      "logps/rejected": -376.0,
      "loss": 0.0232,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.59375,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.5625,
      "step": 1244
    },
    {
      "epoch": 1.5276073619631902,
      "grad_norm": 14.324574848335546,
      "learning_rate": 7.438016528925621e-07,
      "logits/chosen": -0.10498046875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -422.0,
      "logps/rejected": -418.0,
      "loss": 0.0653,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08154296875,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.3125,
      "step": 1245
    },
    {
      "epoch": 1.528834355828221,
      "grad_norm": 8.939887633566299,
      "learning_rate": 7.431818181818182e-07,
      "logits/chosen": -0.23046875,
      "logits/rejected": -0.396484375,
      "logps/chosen": -352.0,
      "logps/rejected": -370.0,
      "loss": 0.0281,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.05908203125,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.1875,
      "step": 1246
    },
    {
      "epoch": 1.5300613496932516,
      "grad_norm": 2.737977434539333,
      "learning_rate": 7.425619834710743e-07,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -434.0,
      "logps/rejected": -404.0,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.318359375,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.75,
      "step": 1247
    },
    {
      "epoch": 1.5312883435582823,
      "grad_norm": 7.955215828013632,
      "learning_rate": 7.419421487603306e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -376.0,
      "logps/rejected": -386.0,
      "loss": 0.0229,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.3828125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.0,
      "step": 1248
    },
    {
      "epoch": 1.532515337423313,
      "grad_norm": 13.148164884666349,
      "learning_rate": 7.413223140495868e-07,
      "logits/chosen": 0.036376953125,
      "logits/rejected": -0.0673828125,
      "logps/chosen": -350.0,
      "logps/rejected": -378.0,
      "loss": 0.0618,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.54296875,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.4375,
      "step": 1249
    },
    {
      "epoch": 1.5337423312883436,
      "grad_norm": 6.7014580907570505,
      "learning_rate": 7.40702479338843e-07,
      "logits/chosen": -0.2333984375,
      "logits/rejected": -0.384765625,
      "logps/chosen": -446.0,
      "logps/rejected": -472.0,
      "loss": 0.028,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.39453125,
      "rewards/margins": 13.125,
      "rewards/rejected": -12.75,
      "step": 1250
    },
    {
      "epoch": 1.5349693251533743,
      "grad_norm": 9.188465721829859,
      "learning_rate": 7.400826446280991e-07,
      "logits/chosen": -0.291015625,
      "logits/rejected": -0.39453125,
      "logps/chosen": -376.0,
      "logps/rejected": -414.0,
      "loss": 0.0271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 12.625,
      "rewards/rejected": -11.9375,
      "step": 1251
    },
    {
      "epoch": 1.536196319018405,
      "grad_norm": 8.458678355370827,
      "learning_rate": 7.394628099173554e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -390.0,
      "logps/rejected": -372.0,
      "loss": 0.0304,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 12.375,
      "rewards/rejected": -12.1875,
      "step": 1252
    },
    {
      "epoch": 1.5374233128834356,
      "grad_norm": 8.012255387049295,
      "learning_rate": 7.388429752066116e-07,
      "logits/chosen": -0.16796875,
      "logits/rejected": -0.326171875,
      "logps/chosen": -420.0,
      "logps/rejected": -410.0,
      "loss": 0.0317,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.1875,
      "step": 1253
    },
    {
      "epoch": 1.5386503067484663,
      "grad_norm": 9.41643727038603,
      "learning_rate": 7.382231404958678e-07,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.19921875,
      "logps/chosen": -378.0,
      "logps/rejected": -360.0,
      "loss": 0.0366,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.75,
      "step": 1254
    },
    {
      "epoch": 1.539877300613497,
      "grad_norm": 7.434007112296638,
      "learning_rate": 7.37603305785124e-07,
      "logits/chosen": -0.203125,
      "logits/rejected": -0.427734375,
      "logps/chosen": -410.0,
      "logps/rejected": -432.0,
      "loss": 0.0221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.0625,
      "step": 1255
    },
    {
      "epoch": 1.5411042944785276,
      "grad_norm": 8.974145176497167,
      "learning_rate": 7.369834710743802e-07,
      "logits/chosen": -0.1142578125,
      "logits/rejected": -0.328125,
      "logps/chosen": -410.0,
      "logps/rejected": -398.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.06494140625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.625,
      "step": 1256
    },
    {
      "epoch": 1.5423312883435583,
      "grad_norm": 13.782445851616703,
      "learning_rate": 7.363636363636364e-07,
      "logits/chosen": -0.0732421875,
      "logits/rejected": -0.26171875,
      "logps/chosen": -390.0,
      "logps/rejected": -386.0,
      "loss": 0.0591,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.0693359375,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.9375,
      "step": 1257
    },
    {
      "epoch": 1.543558282208589,
      "grad_norm": 8.478001588142458,
      "learning_rate": 7.357438016528926e-07,
      "logits/chosen": -0.10791015625,
      "logits/rejected": -0.3125,
      "logps/chosen": -404.0,
      "logps/rejected": -368.0,
      "loss": 0.0315,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.5625,
      "step": 1258
    },
    {
      "epoch": 1.5447852760736196,
      "grad_norm": 12.343587239938957,
      "learning_rate": 7.351239669421488e-07,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.34375,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.0382,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.625,
      "step": 1259
    },
    {
      "epoch": 1.5460122699386503,
      "grad_norm": 10.786647686676162,
      "learning_rate": 7.345041322314049e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.34765625,
      "logps/chosen": -450.0,
      "logps/rejected": -396.0,
      "loss": 0.0273,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 11.6875,
      "rewards/rejected": -10.75,
      "step": 1260
    },
    {
      "epoch": 1.547239263803681,
      "grad_norm": 11.69554028398202,
      "learning_rate": 7.338842975206612e-07,
      "logits/chosen": -0.2236328125,
      "logits/rejected": -0.345703125,
      "logps/chosen": -406.0,
      "logps/rejected": -388.0,
      "loss": 0.0333,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.490234375,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.875,
      "step": 1261
    },
    {
      "epoch": 1.5484662576687116,
      "grad_norm": 7.864292149216722,
      "learning_rate": 7.332644628099174e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.333984375,
      "logps/chosen": -386.0,
      "logps/rejected": -362.0,
      "loss": 0.038,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.13671875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.0625,
      "step": 1262
    },
    {
      "epoch": 1.5496932515337423,
      "grad_norm": 13.897917738806647,
      "learning_rate": 7.326446280991736e-07,
      "logits/chosen": -0.0830078125,
      "logits/rejected": -0.298828125,
      "logps/chosen": -394.0,
      "logps/rejected": -380.0,
      "loss": 0.0455,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.9375,
      "step": 1263
    },
    {
      "epoch": 1.550920245398773,
      "grad_norm": 11.512996209757597,
      "learning_rate": 7.320247933884297e-07,
      "logits/chosen": -0.0218505859375,
      "logits/rejected": -0.1875,
      "logps/chosen": -390.0,
      "logps/rejected": -398.0,
      "loss": 0.0581,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1591796875,
      "rewards/margins": 10.1875,
      "rewards/rejected": -10.3125,
      "step": 1264
    },
    {
      "epoch": 1.5521472392638036,
      "grad_norm": 13.705005476191044,
      "learning_rate": 7.31404958677686e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.341796875,
      "logps/chosen": -380.0,
      "logps/rejected": -390.0,
      "loss": 0.0413,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.3359375,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.8125,
      "step": 1265
    },
    {
      "epoch": 1.5533742331288343,
      "grad_norm": 10.295352755658561,
      "learning_rate": 7.307851239669422e-07,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.296875,
      "logps/chosen": -352.0,
      "logps/rejected": -372.0,
      "loss": 0.0297,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.09130859375,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.5,
      "step": 1266
    },
    {
      "epoch": 1.554601226993865,
      "grad_norm": 18.45895537867682,
      "learning_rate": 7.301652892561983e-07,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.271484375,
      "logps/chosen": -368.0,
      "logps/rejected": -366.0,
      "loss": 0.1103,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.5,
      "step": 1267
    },
    {
      "epoch": 1.5558282208588956,
      "grad_norm": 14.03503002507992,
      "learning_rate": 7.295454545454546e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.255859375,
      "logps/chosen": -498.0,
      "logps/rejected": -422.0,
      "loss": 0.0495,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.1875,
      "step": 1268
    },
    {
      "epoch": 1.5570552147239263,
      "grad_norm": 3.7230278336867006,
      "learning_rate": 7.289256198347107e-07,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.314453125,
      "logps/chosen": -378.0,
      "logps/rejected": -362.0,
      "loss": 0.0144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.75,
      "step": 1269
    },
    {
      "epoch": 1.558282208588957,
      "grad_norm": 8.890488407699403,
      "learning_rate": 7.28305785123967e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.22265625,
      "logps/chosen": -406.0,
      "logps/rejected": -406.0,
      "loss": 0.026,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.375,
      "step": 1270
    },
    {
      "epoch": 1.5595092024539876,
      "grad_norm": 8.516198696845315,
      "learning_rate": 7.276859504132231e-07,
      "logits/chosen": -0.0830078125,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -356.0,
      "logps/rejected": -366.0,
      "loss": 0.0302,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.25,
      "step": 1271
    },
    {
      "epoch": 1.5607361963190183,
      "grad_norm": 3.950798580884482,
      "learning_rate": 7.270661157024794e-07,
      "logits/chosen": -0.2578125,
      "logits/rejected": -0.412109375,
      "logps/chosen": -426.0,
      "logps/rejected": -392.0,
      "loss": 0.0131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.46875,
      "rewards/margins": 12.25,
      "rewards/rejected": -10.75,
      "step": 1272
    },
    {
      "epoch": 1.561963190184049,
      "grad_norm": 12.063277382793855,
      "learning_rate": 7.264462809917355e-07,
      "logits/chosen": -0.2080078125,
      "logits/rejected": -0.3046875,
      "logps/chosen": -426.0,
      "logps/rejected": -412.0,
      "loss": 0.0509,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.3125,
      "step": 1273
    },
    {
      "epoch": 1.5631901840490796,
      "grad_norm": 5.052885121682078,
      "learning_rate": 7.258264462809918e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.5078125,
      "logps/chosen": -400.0,
      "logps/rejected": -388.0,
      "loss": 0.0173,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.75,
      "step": 1274
    },
    {
      "epoch": 1.5644171779141103,
      "grad_norm": 15.33141936648023,
      "learning_rate": 7.25206611570248e-07,
      "logits/chosen": -0.10791015625,
      "logits/rejected": -0.2578125,
      "logps/chosen": -388.0,
      "logps/rejected": -394.0,
      "loss": 0.0744,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.111328125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.8125,
      "step": 1275
    },
    {
      "epoch": 1.565644171779141,
      "grad_norm": 7.988279468708803,
      "learning_rate": 7.245867768595041e-07,
      "logits/chosen": -0.025146484375,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -418.0,
      "logps/rejected": -374.0,
      "loss": 0.0247,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.48046875,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.5,
      "step": 1276
    },
    {
      "epoch": 1.5668711656441716,
      "grad_norm": 12.554173231892838,
      "learning_rate": 7.239669421487604e-07,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.314453125,
      "logps/chosen": -358.0,
      "logps/rejected": -354.0,
      "loss": 0.043,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.034423828125,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.0625,
      "step": 1277
    },
    {
      "epoch": 1.5680981595092025,
      "grad_norm": 15.815759980577592,
      "learning_rate": 7.233471074380165e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.365234375,
      "logps/chosen": -404.0,
      "logps/rejected": -396.0,
      "loss": 0.028,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.77734375,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.3125,
      "step": 1278
    },
    {
      "epoch": 1.5693251533742332,
      "grad_norm": 13.88863047530027,
      "learning_rate": 7.227272727272728e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.310546875,
      "logps/chosen": -356.0,
      "logps/rejected": -394.0,
      "loss": 0.0486,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1826171875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.5,
      "step": 1279
    },
    {
      "epoch": 1.5705521472392638,
      "grad_norm": 11.958915852962361,
      "learning_rate": 7.221074380165289e-07,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -434.0,
      "logps/rejected": -432.0,
      "loss": 0.0382,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.470703125,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.75,
      "step": 1280
    },
    {
      "epoch": 1.5717791411042945,
      "grad_norm": 13.174861138807707,
      "learning_rate": 7.214876033057852e-07,
      "logits/chosen": -0.2236328125,
      "logits/rejected": -0.384765625,
      "logps/chosen": -438.0,
      "logps/rejected": -440.0,
      "loss": 0.0556,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.125,
      "rewards/margins": 12.6875,
      "rewards/rejected": -11.5625,
      "step": 1281
    },
    {
      "epoch": 1.5730061349693252,
      "grad_norm": 9.377186147771738,
      "learning_rate": 7.208677685950413e-07,
      "logits/chosen": 0.048828125,
      "logits/rejected": -0.185546875,
      "logps/chosen": -394.0,
      "logps/rejected": -374.0,
      "loss": 0.0309,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.032470703125,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.125,
      "step": 1282
    },
    {
      "epoch": 1.5742331288343558,
      "grad_norm": 9.78074406651563,
      "learning_rate": 7.202479338842976e-07,
      "logits/chosen": -0.0028228759765625,
      "logits/rejected": -0.11865234375,
      "logps/chosen": -354.0,
      "logps/rejected": -380.0,
      "loss": 0.0371,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 10.8125,
      "rewards/rejected": -11.6875,
      "step": 1283
    },
    {
      "epoch": 1.5754601226993865,
      "grad_norm": 15.650011079839588,
      "learning_rate": 7.196280991735537e-07,
      "logits/chosen": -0.072265625,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -404.0,
      "logps/rejected": -420.0,
      "loss": 0.0681,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.33203125,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.5,
      "step": 1284
    },
    {
      "epoch": 1.5766871165644172,
      "grad_norm": 15.27608982764389,
      "learning_rate": 7.190082644628099e-07,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.373046875,
      "logps/chosen": -422.0,
      "logps/rejected": -420.0,
      "loss": 0.0468,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.109375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.3125,
      "step": 1285
    },
    {
      "epoch": 1.5779141104294478,
      "grad_norm": 15.001506307085636,
      "learning_rate": 7.183884297520662e-07,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.271484375,
      "logps/chosen": -372.0,
      "logps/rejected": -356.0,
      "loss": 0.0383,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.921875,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.6875,
      "step": 1286
    },
    {
      "epoch": 1.5791411042944785,
      "grad_norm": 5.225854839258308,
      "learning_rate": 7.177685950413223e-07,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -330.0,
      "logps/rejected": -380.0,
      "loss": 0.0262,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.375,
      "rewards/margins": 11.0,
      "rewards/rejected": -12.375,
      "step": 1287
    },
    {
      "epoch": 1.5803680981595092,
      "grad_norm": 11.803831089679342,
      "learning_rate": 7.171487603305785e-07,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.400390625,
      "logps/chosen": -394.0,
      "logps/rejected": -412.0,
      "loss": 0.0419,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2236328125,
      "rewards/margins": 12.125,
      "rewards/rejected": -12.3125,
      "step": 1288
    },
    {
      "epoch": 1.5815950920245399,
      "grad_norm": 9.347363939888405,
      "learning_rate": 7.165289256198347e-07,
      "logits/chosen": -0.087890625,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -334.0,
      "logps/rejected": -342.0,
      "loss": 0.0396,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.0390625,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.8125,
      "step": 1289
    },
    {
      "epoch": 1.5828220858895705,
      "grad_norm": 4.3767999812751714,
      "learning_rate": 7.15909090909091e-07,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.40234375,
      "logps/chosen": -396.0,
      "logps/rejected": -416.0,
      "loss": 0.0096,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8984375,
      "rewards/margins": 12.375,
      "rewards/rejected": -13.25,
      "step": 1290
    },
    {
      "epoch": 1.5840490797546012,
      "grad_norm": 4.228705291026931,
      "learning_rate": 7.152892561983471e-07,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.21484375,
      "logps/chosen": -372.0,
      "logps/rejected": -376.0,
      "loss": 0.0152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -12.1875,
      "step": 1291
    },
    {
      "epoch": 1.5852760736196319,
      "grad_norm": 7.584547777512873,
      "learning_rate": 7.146694214876034e-07,
      "logits/chosen": -0.25,
      "logits/rejected": -0.35546875,
      "logps/chosen": -390.0,
      "logps/rejected": -428.0,
      "loss": 0.0277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.67578125,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.875,
      "step": 1292
    },
    {
      "epoch": 1.5865030674846625,
      "grad_norm": 10.655223291637148,
      "learning_rate": 7.140495867768595e-07,
      "logits/chosen": -0.0137939453125,
      "logits/rejected": -0.28515625,
      "logps/chosen": -416.0,
      "logps/rejected": -384.0,
      "loss": 0.0202,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.25390625,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.625,
      "step": 1293
    },
    {
      "epoch": 1.5877300613496934,
      "grad_norm": 4.2898047053372155,
      "learning_rate": 7.134297520661157e-07,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.30078125,
      "logps/chosen": -412.0,
      "logps/rejected": -428.0,
      "loss": 0.0154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 12.25,
      "rewards/rejected": -12.8125,
      "step": 1294
    },
    {
      "epoch": 1.588957055214724,
      "grad_norm": 8.499456654824181,
      "learning_rate": 7.12809917355372e-07,
      "logits/chosen": -0.2470703125,
      "logits/rejected": -0.431640625,
      "logps/chosen": -370.0,
      "logps/rejected": -378.0,
      "loss": 0.0387,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.59375,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.5625,
      "step": 1295
    },
    {
      "epoch": 1.5901840490797547,
      "grad_norm": 16.676691340584743,
      "learning_rate": 7.121900826446281e-07,
      "logits/chosen": -0.1220703125,
      "logits/rejected": -0.2099609375,
      "logps/chosen": -394.0,
      "logps/rejected": -402.0,
      "loss": 0.0806,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.234375,
      "rewards/margins": 10.75,
      "rewards/rejected": -12.0,
      "step": 1296
    },
    {
      "epoch": 1.5914110429447854,
      "grad_norm": 9.185721305153498,
      "learning_rate": 7.115702479338843e-07,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -414.0,
      "logps/rejected": -384.0,
      "loss": 0.0365,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.41015625,
      "rewards/margins": 9.875,
      "rewards/rejected": -10.25,
      "step": 1297
    },
    {
      "epoch": 1.592638036809816,
      "grad_norm": 9.516915353409866,
      "learning_rate": 7.109504132231405e-07,
      "logits/chosen": -0.07568359375,
      "logits/rejected": -0.197265625,
      "logps/chosen": -410.0,
      "logps/rejected": -414.0,
      "loss": 0.0373,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.279296875,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.4375,
      "step": 1298
    },
    {
      "epoch": 1.5938650306748468,
      "grad_norm": 12.688329159049632,
      "learning_rate": 7.103305785123968e-07,
      "logits/chosen": -0.197265625,
      "logits/rejected": -0.28515625,
      "logps/chosen": -394.0,
      "logps/rejected": -402.0,
      "loss": 0.0515,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.1875,
      "step": 1299
    },
    {
      "epoch": 1.5950920245398774,
      "grad_norm": 9.480010594113034,
      "learning_rate": 7.097107438016529e-07,
      "logits/chosen": -0.08447265625,
      "logits/rejected": -0.283203125,
      "logps/chosen": -406.0,
      "logps/rejected": -410.0,
      "loss": 0.0238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.345703125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.4375,
      "step": 1300
    },
    {
      "epoch": 1.596319018404908,
      "grad_norm": 6.743619593496769,
      "learning_rate": 7.09090909090909e-07,
      "logits/chosen": -0.07958984375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -384.0,
      "logps/rejected": -394.0,
      "loss": 0.0206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.045166015625,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.25,
      "step": 1301
    },
    {
      "epoch": 1.5975460122699388,
      "grad_norm": 10.079704999041756,
      "learning_rate": 7.084710743801653e-07,
      "logits/chosen": -0.283203125,
      "logits/rejected": -0.396484375,
      "logps/chosen": -404.0,
      "logps/rejected": -402.0,
      "loss": 0.0333,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.49609375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.3125,
      "step": 1302
    },
    {
      "epoch": 1.5987730061349694,
      "grad_norm": 4.128642305901009,
      "learning_rate": 7.078512396694215e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.369140625,
      "logps/chosen": -440.0,
      "logps/rejected": -408.0,
      "loss": 0.0206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6328125,
      "rewards/margins": 12.5625,
      "rewards/rejected": -11.9375,
      "step": 1303
    },
    {
      "epoch": 1.6,
      "grad_norm": 6.006214161041272,
      "learning_rate": 7.072314049586777e-07,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.388671875,
      "logps/chosen": -380.0,
      "logps/rejected": -408.0,
      "loss": 0.0223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7421875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.5625,
      "step": 1304
    },
    {
      "epoch": 1.6012269938650308,
      "grad_norm": 15.374011330047086,
      "learning_rate": 7.066115702479339e-07,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -346.0,
      "logps/rejected": -366.0,
      "loss": 0.068,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.423828125,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.4375,
      "step": 1305
    },
    {
      "epoch": 1.6024539877300614,
      "grad_norm": 10.751324968165987,
      "learning_rate": 7.059917355371901e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.294921875,
      "logps/chosen": -406.0,
      "logps/rejected": -384.0,
      "loss": 0.0244,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.875,
      "step": 1306
    },
    {
      "epoch": 1.603680981595092,
      "grad_norm": 7.9285821638566905,
      "learning_rate": 7.053719008264463e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.38671875,
      "logps/chosen": -440.0,
      "logps/rejected": -402.0,
      "loss": 0.0207,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 11.25,
      "rewards/rejected": -10.25,
      "step": 1307
    },
    {
      "epoch": 1.6049079754601228,
      "grad_norm": 4.924145204402143,
      "learning_rate": 7.047520661157026e-07,
      "logits/chosen": -0.25,
      "logits/rejected": -0.392578125,
      "logps/chosen": -380.0,
      "logps/rejected": -372.0,
      "loss": 0.0136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.6875,
      "step": 1308
    },
    {
      "epoch": 1.6061349693251534,
      "grad_norm": 11.95129952622703,
      "learning_rate": 7.041322314049587e-07,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -392.0,
      "logps/rejected": -394.0,
      "loss": 0.0371,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.0,
      "step": 1309
    },
    {
      "epoch": 1.607361963190184,
      "grad_norm": 6.935480593251807,
      "learning_rate": 7.035123966942149e-07,
      "logits/chosen": -0.09619140625,
      "logits/rejected": -0.2578125,
      "logps/chosen": -334.0,
      "logps/rejected": -356.0,
      "loss": 0.0257,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.4375,
      "step": 1310
    },
    {
      "epoch": 1.6085889570552148,
      "grad_norm": 7.070447805786784,
      "learning_rate": 7.028925619834711e-07,
      "logits/chosen": -0.046142578125,
      "logits/rejected": -0.24609375,
      "logps/chosen": -418.0,
      "logps/rejected": -380.0,
      "loss": 0.0364,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.0751953125,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.8125,
      "step": 1311
    },
    {
      "epoch": 1.6098159509202454,
      "grad_norm": 6.5989010041947065,
      "learning_rate": 7.022727272727273e-07,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.453125,
      "logps/chosen": -404.0,
      "logps/rejected": -392.0,
      "loss": 0.0213,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.5625,
      "step": 1312
    },
    {
      "epoch": 1.611042944785276,
      "grad_norm": 5.053991836327145,
      "learning_rate": 7.016528925619835e-07,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.28125,
      "logps/chosen": -366.0,
      "logps/rejected": -374.0,
      "loss": 0.0231,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2294921875,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.5,
      "step": 1313
    },
    {
      "epoch": 1.6122699386503068,
      "grad_norm": 7.196334947724689,
      "learning_rate": 7.010330578512396e-07,
      "logits/chosen": -0.310546875,
      "logits/rejected": -0.390625,
      "logps/chosen": -392.0,
      "logps/rejected": -408.0,
      "loss": 0.0192,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 12.4375,
      "rewards/rejected": -11.4375,
      "step": 1314
    },
    {
      "epoch": 1.6134969325153374,
      "grad_norm": 7.456124142101998,
      "learning_rate": 7.004132231404959e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.384765625,
      "logps/chosen": -452.0,
      "logps/rejected": -422.0,
      "loss": 0.0295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.1875,
      "step": 1315
    },
    {
      "epoch": 1.614723926380368,
      "grad_norm": 14.771588037378546,
      "learning_rate": 6.997933884297521e-07,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -378.0,
      "logps/rejected": -366.0,
      "loss": 0.0521,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.09912109375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -10.3125,
      "step": 1316
    },
    {
      "epoch": 1.6159509202453988,
      "grad_norm": 6.5021730031025555,
      "learning_rate": 6.991735537190084e-07,
      "logits/chosen": -0.19140625,
      "logits/rejected": -0.296875,
      "logps/chosen": -388.0,
      "logps/rejected": -414.0,
      "loss": 0.024,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.0625,
      "step": 1317
    },
    {
      "epoch": 1.6171779141104294,
      "grad_norm": 7.036484087537271,
      "learning_rate": 6.985537190082644e-07,
      "logits/chosen": -0.126953125,
      "logits/rejected": -0.333984375,
      "logps/chosen": -388.0,
      "logps/rejected": -382.0,
      "loss": 0.0165,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.037109375,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.3125,
      "step": 1318
    },
    {
      "epoch": 1.61840490797546,
      "grad_norm": 12.457147867029073,
      "learning_rate": 6.979338842975207e-07,
      "logits/chosen": -0.1552734375,
      "logits/rejected": -0.30859375,
      "logps/chosen": -390.0,
      "logps/rejected": -424.0,
      "loss": 0.0243,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.474609375,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.5625,
      "step": 1319
    },
    {
      "epoch": 1.6196319018404908,
      "grad_norm": 18.65684903039485,
      "learning_rate": 6.973140495867769e-07,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.314453125,
      "logps/chosen": -438.0,
      "logps/rejected": -406.0,
      "loss": 0.0705,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.4375,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.1875,
      "step": 1320
    },
    {
      "epoch": 1.6208588957055214,
      "grad_norm": 15.725190139889248,
      "learning_rate": 6.966942148760331e-07,
      "logits/chosen": -0.16796875,
      "logits/rejected": -0.296875,
      "logps/chosen": -384.0,
      "logps/rejected": -376.0,
      "loss": 0.0439,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.17578125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.25,
      "step": 1321
    },
    {
      "epoch": 1.622085889570552,
      "grad_norm": 13.624128902680484,
      "learning_rate": 6.960743801652893e-07,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.345703125,
      "logps/chosen": -366.0,
      "logps/rejected": -396.0,
      "loss": 0.0327,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1865234375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.125,
      "step": 1322
    },
    {
      "epoch": 1.6233128834355828,
      "grad_norm": 2.854089790732775,
      "learning_rate": 6.954545454545454e-07,
      "logits/chosen": -0.0157470703125,
      "logits/rejected": -0.208984375,
      "logps/chosen": -336.0,
      "logps/rejected": -376.0,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03955078125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.375,
      "step": 1323
    },
    {
      "epoch": 1.6245398773006134,
      "grad_norm": 13.205081682962938,
      "learning_rate": 6.948347107438017e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.404296875,
      "logps/chosen": -416.0,
      "logps/rejected": -394.0,
      "loss": 0.0504,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.265625,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.625,
      "step": 1324
    },
    {
      "epoch": 1.6257668711656441,
      "grad_norm": 10.512892902569012,
      "learning_rate": 6.942148760330579e-07,
      "logits/chosen": -0.07275390625,
      "logits/rejected": -0.33203125,
      "logps/chosen": -388.0,
      "logps/rejected": -390.0,
      "loss": 0.0409,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.75,
      "step": 1325
    },
    {
      "epoch": 1.6269938650306748,
      "grad_norm": 16.75758409246438,
      "learning_rate": 6.935950413223141e-07,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.33203125,
      "logps/chosen": -412.0,
      "logps/rejected": -418.0,
      "loss": 0.0433,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.076171875,
      "rewards/margins": 12.25,
      "rewards/rejected": -12.3125,
      "step": 1326
    },
    {
      "epoch": 1.6282208588957054,
      "grad_norm": 17.757254444197297,
      "learning_rate": 6.929752066115702e-07,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.2265625,
      "logps/chosen": -372.0,
      "logps/rejected": -406.0,
      "loss": 0.095,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.80859375,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.6875,
      "step": 1327
    },
    {
      "epoch": 1.6294478527607361,
      "grad_norm": 13.567940362966437,
      "learning_rate": 6.923553719008265e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.40625,
      "logps/chosen": -414.0,
      "logps/rejected": -398.0,
      "loss": 0.0402,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.330078125,
      "rewards/margins": 11.75,
      "rewards/rejected": -12.0625,
      "step": 1328
    },
    {
      "epoch": 1.6306748466257668,
      "grad_norm": 15.662951369347224,
      "learning_rate": 6.917355371900827e-07,
      "logits/chosen": -0.2353515625,
      "logits/rejected": -0.41015625,
      "logps/chosen": -422.0,
      "logps/rejected": -414.0,
      "loss": 0.0593,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6640625,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.4375,
      "step": 1329
    },
    {
      "epoch": 1.6319018404907975,
      "grad_norm": 10.57654744394464,
      "learning_rate": 6.911157024793388e-07,
      "logits/chosen": -0.12158203125,
      "logits/rejected": -0.33203125,
      "logps/chosen": -404.0,
      "logps/rejected": -404.0,
      "loss": 0.032,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.37109375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.875,
      "step": 1330
    },
    {
      "epoch": 1.6331288343558281,
      "grad_norm": 7.812525368210755,
      "learning_rate": 6.90495867768595e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -366.0,
      "logps/rejected": -388.0,
      "loss": 0.0254,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -1.0546875,
      "rewards/margins": 11.8125,
      "rewards/rejected": -12.875,
      "step": 1331
    },
    {
      "epoch": 1.6343558282208588,
      "grad_norm": 12.529370972774968,
      "learning_rate": 6.898760330578512e-07,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.416015625,
      "logps/chosen": -386.0,
      "logps/rejected": -390.0,
      "loss": 0.0267,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.328125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.75,
      "step": 1332
    },
    {
      "epoch": 1.6355828220858895,
      "grad_norm": 15.494182278895307,
      "learning_rate": 6.892561983471075e-07,
      "logits/chosen": -0.27734375,
      "logits/rejected": -0.416015625,
      "logps/chosen": -420.0,
      "logps/rejected": -440.0,
      "loss": 0.0527,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.875,
      "step": 1333
    },
    {
      "epoch": 1.6368098159509201,
      "grad_norm": 8.698893043286152,
      "learning_rate": 6.886363636363637e-07,
      "logits/chosen": -0.10009765625,
      "logits/rejected": -0.244140625,
      "logps/chosen": -382.0,
      "logps/rejected": -400.0,
      "loss": 0.0306,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.6875,
      "step": 1334
    },
    {
      "epoch": 1.6380368098159508,
      "grad_norm": 14.987956201874853,
      "learning_rate": 6.880165289256199e-07,
      "logits/chosen": -0.2451171875,
      "logits/rejected": -0.44921875,
      "logps/chosen": -432.0,
      "logps/rejected": -386.0,
      "loss": 0.0336,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.109375,
      "rewards/margins": 11.625,
      "rewards/rejected": -10.5625,
      "step": 1335
    },
    {
      "epoch": 1.6392638036809815,
      "grad_norm": 15.60191378628464,
      "learning_rate": 6.87396694214876e-07,
      "logits/chosen": -0.361328125,
      "logits/rejected": -0.53125,
      "logps/chosen": -436.0,
      "logps/rejected": -436.0,
      "loss": 0.0541,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 12.0625,
      "rewards/rejected": -12.5625,
      "step": 1336
    },
    {
      "epoch": 1.6404907975460121,
      "grad_norm": 6.108403854935857,
      "learning_rate": 6.867768595041323e-07,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.376953125,
      "logps/chosen": -414.0,
      "logps/rejected": -384.0,
      "loss": 0.0178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1640625,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.6875,
      "step": 1337
    },
    {
      "epoch": 1.6417177914110428,
      "grad_norm": 6.235103209750328,
      "learning_rate": 6.861570247933885e-07,
      "logits/chosen": -0.2734375,
      "logits/rejected": -0.400390625,
      "logps/chosen": -432.0,
      "logps/rejected": -410.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.435546875,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.1875,
      "step": 1338
    },
    {
      "epoch": 1.6429447852760735,
      "grad_norm": 13.809775053491462,
      "learning_rate": 6.855371900826446e-07,
      "logits/chosen": -0.13671875,
      "logits/rejected": -0.1923828125,
      "logps/chosen": -390.0,
      "logps/rejected": -346.0,
      "loss": 0.0495,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.033935546875,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.375,
      "step": 1339
    },
    {
      "epoch": 1.6441717791411041,
      "grad_norm": 14.649539845278797,
      "learning_rate": 6.849173553719008e-07,
      "logits/chosen": -0.060791015625,
      "logits/rejected": -0.220703125,
      "logps/chosen": -412.0,
      "logps/rejected": -394.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.1875,
      "step": 1340
    },
    {
      "epoch": 1.645398773006135,
      "grad_norm": 6.096103726931023,
      "learning_rate": 6.84297520661157e-07,
      "logits/chosen": -0.224609375,
      "logits/rejected": -0.453125,
      "logps/chosen": -438.0,
      "logps/rejected": -398.0,
      "loss": 0.0186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.5,
      "step": 1341
    },
    {
      "epoch": 1.6466257668711657,
      "grad_norm": 4.043549433090365,
      "learning_rate": 6.836776859504133e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.470703125,
      "logps/chosen": -410.0,
      "logps/rejected": -378.0,
      "loss": 0.0267,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.60546875,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.4375,
      "step": 1342
    },
    {
      "epoch": 1.6478527607361964,
      "grad_norm": 22.56474705026688,
      "learning_rate": 6.830578512396694e-07,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.40625,
      "logps/chosen": -408.0,
      "logps/rejected": -404.0,
      "loss": 0.0661,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.1650390625,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.9375,
      "step": 1343
    },
    {
      "epoch": 1.649079754601227,
      "grad_norm": 7.990667626479867,
      "learning_rate": 6.824380165289256e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.416015625,
      "logps/chosen": -434.0,
      "logps/rejected": -388.0,
      "loss": 0.0183,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.203125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -10.75,
      "step": 1344
    },
    {
      "epoch": 1.6503067484662577,
      "grad_norm": 9.134005678853116,
      "learning_rate": 6.818181818181818e-07,
      "logits/chosen": -0.1943359375,
      "logits/rejected": -0.34375,
      "logps/chosen": -422.0,
      "logps/rejected": -376.0,
      "loss": 0.0366,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.361328125,
      "rewards/margins": 10.0,
      "rewards/rejected": -9.625,
      "step": 1345
    },
    {
      "epoch": 1.6515337423312884,
      "grad_norm": 19.441190331880875,
      "learning_rate": 6.811983471074381e-07,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.32421875,
      "logps/chosen": -438.0,
      "logps/rejected": -396.0,
      "loss": 0.0749,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.8125,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.3125,
      "step": 1346
    },
    {
      "epoch": 1.652760736196319,
      "grad_norm": 14.390784632473302,
      "learning_rate": 6.805785123966942e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.3046875,
      "logps/chosen": -420.0,
      "logps/rejected": -420.0,
      "loss": 0.0671,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.55078125,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.4375,
      "step": 1347
    },
    {
      "epoch": 1.6539877300613497,
      "grad_norm": 8.537444327620346,
      "learning_rate": 6.799586776859504e-07,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.32421875,
      "logps/chosen": -340.0,
      "logps/rejected": -382.0,
      "loss": 0.0348,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.048095703125,
      "rewards/margins": 10.6875,
      "rewards/rejected": -10.6875,
      "step": 1348
    },
    {
      "epoch": 1.6552147239263804,
      "grad_norm": 7.326685705649235,
      "learning_rate": 6.793388429752066e-07,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.337890625,
      "logps/chosen": -368.0,
      "logps/rejected": -380.0,
      "loss": 0.0303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.80078125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.375,
      "step": 1349
    },
    {
      "epoch": 1.656441717791411,
      "grad_norm": 11.253855573956272,
      "learning_rate": 6.787190082644628e-07,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -422.0,
      "logps/rejected": -432.0,
      "loss": 0.0313,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.375,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.125,
      "step": 1350
    },
    {
      "epoch": 1.6576687116564417,
      "grad_norm": 20.220907692170936,
      "learning_rate": 6.780991735537191e-07,
      "logits/chosen": -0.11962890625,
      "logits/rejected": -0.26171875,
      "logps/chosen": -396.0,
      "logps/rejected": -392.0,
      "loss": 0.1149,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.1650390625,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.75,
      "step": 1351
    },
    {
      "epoch": 1.6588957055214724,
      "grad_norm": 8.156843128459297,
      "learning_rate": 6.774793388429752e-07,
      "logits/chosen": -0.2119140625,
      "logits/rejected": -0.369140625,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.0315,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2314453125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.25,
      "step": 1352
    },
    {
      "epoch": 1.660122699386503,
      "grad_norm": 6.558928813231135,
      "learning_rate": 6.768595041322314e-07,
      "logits/chosen": -0.10986328125,
      "logits/rejected": -0.146484375,
      "logps/chosen": -376.0,
      "logps/rejected": -412.0,
      "loss": 0.0286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0908203125,
      "rewards/margins": 10.75,
      "rewards/rejected": -10.8125,
      "step": 1353
    },
    {
      "epoch": 1.6613496932515337,
      "grad_norm": 4.479021963688224,
      "learning_rate": 6.762396694214876e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.333984375,
      "logps/chosen": -368.0,
      "logps/rejected": -394.0,
      "loss": 0.025,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.25,
      "step": 1354
    },
    {
      "epoch": 1.6625766871165644,
      "grad_norm": 12.69028856164417,
      "learning_rate": 6.756198347107439e-07,
      "logits/chosen": -0.20703125,
      "logits/rejected": -0.345703125,
      "logps/chosen": -424.0,
      "logps/rejected": -422.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.20703125,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.0625,
      "step": 1355
    },
    {
      "epoch": 1.6638036809815953,
      "grad_norm": 6.416852390478722,
      "learning_rate": 6.75e-07,
      "logits/chosen": -0.16015625,
      "logits/rejected": -0.322265625,
      "logps/chosen": -408.0,
      "logps/rejected": -412.0,
      "loss": 0.0263,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.396484375,
      "rewards/margins": 11.875,
      "rewards/rejected": -12.25,
      "step": 1356
    },
    {
      "epoch": 1.665030674846626,
      "grad_norm": 15.843916290648238,
      "learning_rate": 6.743801652892562e-07,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.353515625,
      "logps/chosen": -436.0,
      "logps/rejected": -410.0,
      "loss": 0.0297,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.30859375,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.6875,
      "step": 1357
    },
    {
      "epoch": 1.6662576687116566,
      "grad_norm": 15.902047502639894,
      "learning_rate": 6.737603305785124e-07,
      "logits/chosen": -0.2080078125,
      "logits/rejected": -0.453125,
      "logps/chosen": -400.0,
      "logps/rejected": -388.0,
      "loss": 0.06,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.11328125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.9375,
      "step": 1358
    },
    {
      "epoch": 1.6674846625766873,
      "grad_norm": 3.3586369592233085,
      "learning_rate": 6.731404958677686e-07,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.318359375,
      "logps/chosen": -410.0,
      "logps/rejected": -412.0,
      "loss": 0.0118,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6171875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.6875,
      "step": 1359
    },
    {
      "epoch": 1.668711656441718,
      "grad_norm": 12.937160200064577,
      "learning_rate": 6.725206611570248e-07,
      "logits/chosen": -0.388671875,
      "logits/rejected": -0.6171875,
      "logps/chosen": -392.0,
      "logps/rejected": -380.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.3125,
      "step": 1360
    },
    {
      "epoch": 1.6699386503067486,
      "grad_norm": 11.349224605689916,
      "learning_rate": 6.71900826446281e-07,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -416.0,
      "logps/rejected": -422.0,
      "loss": 0.0391,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.201171875,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.875,
      "step": 1361
    },
    {
      "epoch": 1.6711656441717793,
      "grad_norm": 16.146708757099134,
      "learning_rate": 6.712809917355372e-07,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.2578125,
      "logps/chosen": -402.0,
      "logps/rejected": -376.0,
      "loss": 0.051,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.474609375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.0625,
      "step": 1362
    },
    {
      "epoch": 1.67239263803681,
      "grad_norm": 3.943632661486204,
      "learning_rate": 6.706611570247934e-07,
      "logits/chosen": -0.224609375,
      "logits/rejected": -0.41796875,
      "logps/chosen": -398.0,
      "logps/rejected": -366.0,
      "loss": 0.0129,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48046875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.125,
      "step": 1363
    },
    {
      "epoch": 1.6736196319018406,
      "grad_norm": 4.1389200081135655,
      "learning_rate": 6.700413223140497e-07,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.279296875,
      "logps/chosen": -374.0,
      "logps/rejected": -356.0,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.453125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.125,
      "step": 1364
    },
    {
      "epoch": 1.6748466257668713,
      "grad_norm": 10.368495733373145,
      "learning_rate": 6.694214876033058e-07,
      "logits/chosen": -0.0458984375,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -408.0,
      "logps/rejected": -370.0,
      "loss": 0.033,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6640625,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.875,
      "step": 1365
    },
    {
      "epoch": 1.676073619631902,
      "grad_norm": 10.90227263887412,
      "learning_rate": 6.68801652892562e-07,
      "logits/chosen": -0.12353515625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -398.0,
      "logps/rejected": -368.0,
      "loss": 0.034,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.125,
      "step": 1366
    },
    {
      "epoch": 1.6773006134969326,
      "grad_norm": 14.33752123603866,
      "learning_rate": 6.681818181818182e-07,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.25390625,
      "logps/chosen": -394.0,
      "logps/rejected": -400.0,
      "loss": 0.0614,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.62109375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.125,
      "step": 1367
    },
    {
      "epoch": 1.6785276073619633,
      "grad_norm": 10.4754734935581,
      "learning_rate": 6.675619834710744e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": -0.462890625,
      "logps/chosen": -400.0,
      "logps/rejected": -390.0,
      "loss": 0.0374,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.125,
      "step": 1368
    },
    {
      "epoch": 1.679754601226994,
      "grad_norm": 13.976696266239935,
      "learning_rate": 6.669421487603306e-07,
      "logits/chosen": -0.2451171875,
      "logits/rejected": -0.4453125,
      "logps/chosen": -404.0,
      "logps/rejected": -394.0,
      "loss": 0.0565,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.10986328125,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.6875,
      "step": 1369
    },
    {
      "epoch": 1.6809815950920246,
      "grad_norm": 10.93083816897367,
      "learning_rate": 6.663223140495868e-07,
      "logits/chosen": -0.1865234375,
      "logits/rejected": -0.296875,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.0301,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 11.5,
      "rewards/rejected": -10.875,
      "step": 1370
    },
    {
      "epoch": 1.6822085889570553,
      "grad_norm": 23.02977282251546,
      "learning_rate": 6.65702479338843e-07,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.388671875,
      "logps/chosen": -468.0,
      "logps/rejected": -428.0,
      "loss": 0.0674,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.79296875,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.0,
      "step": 1371
    },
    {
      "epoch": 1.683435582822086,
      "grad_norm": 6.529767256435788,
      "learning_rate": 6.650826446280992e-07,
      "logits/chosen": -0.1787109375,
      "logits/rejected": -0.404296875,
      "logps/chosen": -392.0,
      "logps/rejected": -384.0,
      "loss": 0.0215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6328125,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.0625,
      "step": 1372
    },
    {
      "epoch": 1.6846625766871166,
      "grad_norm": 16.218767103827304,
      "learning_rate": 6.644628099173554e-07,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.216796875,
      "logps/chosen": -376.0,
      "logps/rejected": -410.0,
      "loss": 0.0695,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.5,
      "step": 1373
    },
    {
      "epoch": 1.6858895705521473,
      "grad_norm": 12.743040998410756,
      "learning_rate": 6.638429752066116e-07,
      "logits/chosen": -0.056396484375,
      "logits/rejected": -0.29296875,
      "logps/chosen": -394.0,
      "logps/rejected": -368.0,
      "loss": 0.0465,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.1943359375,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.4375,
      "step": 1374
    },
    {
      "epoch": 1.687116564417178,
      "grad_norm": 11.631482703134425,
      "learning_rate": 6.632231404958678e-07,
      "logits/chosen": -0.0947265625,
      "logits/rejected": -0.1962890625,
      "logps/chosen": -328.0,
      "logps/rejected": -358.0,
      "loss": 0.0442,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.162109375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.25,
      "step": 1375
    },
    {
      "epoch": 1.6883435582822086,
      "grad_norm": 8.378575051965539,
      "learning_rate": 6.62603305785124e-07,
      "logits/chosen": -0.25,
      "logits/rejected": -0.349609375,
      "logps/chosen": -358.0,
      "logps/rejected": -402.0,
      "loss": 0.0241,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.51953125,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.3125,
      "step": 1376
    },
    {
      "epoch": 1.6895705521472393,
      "grad_norm": 6.071796287867807,
      "learning_rate": 6.619834710743801e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -360.0,
      "logps/rejected": -380.0,
      "loss": 0.0204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.5,
      "step": 1377
    },
    {
      "epoch": 1.69079754601227,
      "grad_norm": 6.709699176048537,
      "learning_rate": 6.613636363636364e-07,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.35546875,
      "logps/chosen": -366.0,
      "logps/rejected": -362.0,
      "loss": 0.0212,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.259765625,
      "rewards/margins": 10.25,
      "rewards/rejected": -10.5,
      "step": 1378
    },
    {
      "epoch": 1.6920245398773006,
      "grad_norm": 9.699757128977033,
      "learning_rate": 6.607438016528926e-07,
      "logits/chosen": -0.1943359375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -410.0,
      "logps/rejected": -378.0,
      "loss": 0.0463,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.875,
      "step": 1379
    },
    {
      "epoch": 1.6932515337423313,
      "grad_norm": 13.747966139148284,
      "learning_rate": 6.601239669421488e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.37109375,
      "logps/chosen": -422.0,
      "logps/rejected": -354.0,
      "loss": 0.0229,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 10.8125,
      "rewards/rejected": -10.125,
      "step": 1380
    },
    {
      "epoch": 1.694478527607362,
      "grad_norm": 12.861297927766412,
      "learning_rate": 6.59504132231405e-07,
      "logits/chosen": -0.0712890625,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -362.0,
      "logps/rejected": -364.0,
      "loss": 0.0549,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.26171875,
      "rewards/margins": 9.9375,
      "rewards/rejected": -9.6875,
      "step": 1381
    },
    {
      "epoch": 1.6957055214723926,
      "grad_norm": 17.309330036067298,
      "learning_rate": 6.588842975206612e-07,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.25,
      "logps/chosen": -348.0,
      "logps/rejected": -376.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.046630859375,
      "rewards/margins": 10.1875,
      "rewards/rejected": -10.25,
      "step": 1382
    },
    {
      "epoch": 1.6969325153374233,
      "grad_norm": 9.03911314369842,
      "learning_rate": 6.582644628099174e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.33203125,
      "logps/chosen": -386.0,
      "logps/rejected": -392.0,
      "loss": 0.0248,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.95703125,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.375,
      "step": 1383
    },
    {
      "epoch": 1.698159509202454,
      "grad_norm": 16.803262144198918,
      "learning_rate": 6.576446280991736e-07,
      "logits/chosen": -0.01507568359375,
      "logits/rejected": -0.130859375,
      "logps/chosen": -370.0,
      "logps/rejected": -352.0,
      "loss": 0.0558,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.04931640625,
      "rewards/margins": 9.875,
      "rewards/rejected": -9.875,
      "step": 1384
    },
    {
      "epoch": 1.6993865030674846,
      "grad_norm": 9.539239180399765,
      "learning_rate": 6.570247933884298e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.265625,
      "logps/chosen": -328.0,
      "logps/rejected": -342.0,
      "loss": 0.0495,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 10.375,
      "rewards/rejected": -9.875,
      "step": 1385
    },
    {
      "epoch": 1.7006134969325153,
      "grad_norm": 8.811291557763298,
      "learning_rate": 6.564049586776859e-07,
      "logits/chosen": -0.294921875,
      "logits/rejected": -0.50390625,
      "logps/chosen": -416.0,
      "logps/rejected": -410.0,
      "loss": 0.0357,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.1484375,
      "rewards/margins": 12.0625,
      "rewards/rejected": -10.9375,
      "step": 1386
    },
    {
      "epoch": 1.701840490797546,
      "grad_norm": 9.305768357984167,
      "learning_rate": 6.557851239669422e-07,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.404296875,
      "logps/chosen": -428.0,
      "logps/rejected": -440.0,
      "loss": 0.029,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.6875,
      "step": 1387
    },
    {
      "epoch": 1.7030674846625766,
      "grad_norm": 7.800969393038809,
      "learning_rate": 6.551652892561984e-07,
      "logits/chosen": -0.01025390625,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -372.0,
      "logps/rejected": -350.0,
      "loss": 0.0219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 9.875,
      "rewards/rejected": -9.625,
      "step": 1388
    },
    {
      "epoch": 1.7042944785276073,
      "grad_norm": 7.529656036678668,
      "learning_rate": 6.545454545454546e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -408.0,
      "logps/rejected": -412.0,
      "loss": 0.0288,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 11.75,
      "rewards/rejected": -10.9375,
      "step": 1389
    },
    {
      "epoch": 1.705521472392638,
      "grad_norm": 10.948563401414935,
      "learning_rate": 6.539256198347107e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.29296875,
      "logps/chosen": -374.0,
      "logps/rejected": -386.0,
      "loss": 0.032,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.345703125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.0,
      "step": 1390
    },
    {
      "epoch": 1.7067484662576686,
      "grad_norm": 8.365223158634247,
      "learning_rate": 6.53305785123967e-07,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.40625,
      "logps/chosen": -392.0,
      "logps/rejected": -386.0,
      "loss": 0.031,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 1.125,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.0,
      "step": 1391
    },
    {
      "epoch": 1.7079754601226993,
      "grad_norm": 7.910192843345908,
      "learning_rate": 6.526859504132232e-07,
      "logits/chosen": 0.01153564453125,
      "logits/rejected": -0.162109375,
      "logps/chosen": -360.0,
      "logps/rejected": -346.0,
      "loss": 0.037,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.1953125,
      "rewards/margins": 9.5,
      "rewards/rejected": -9.6875,
      "step": 1392
    },
    {
      "epoch": 1.70920245398773,
      "grad_norm": 10.155639345314109,
      "learning_rate": 6.520661157024793e-07,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.388671875,
      "logps/chosen": -372.0,
      "logps/rejected": -344.0,
      "loss": 0.0319,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 10.5,
      "rewards/rejected": -9.75,
      "step": 1393
    },
    {
      "epoch": 1.7104294478527606,
      "grad_norm": 11.604341441937358,
      "learning_rate": 6.514462809917355e-07,
      "logits/chosen": -0.248046875,
      "logits/rejected": -0.41015625,
      "logps/chosen": -406.0,
      "logps/rejected": -382.0,
      "loss": 0.0466,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.2890625,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.8125,
      "step": 1394
    },
    {
      "epoch": 1.7116564417177913,
      "grad_norm": 6.312301480394843,
      "learning_rate": 6.508264462809917e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.349609375,
      "logps/chosen": -408.0,
      "logps/rejected": -382.0,
      "loss": 0.0145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 11.0,
      "rewards/rejected": -10.3125,
      "step": 1395
    },
    {
      "epoch": 1.712883435582822,
      "grad_norm": 2.1889604215084524,
      "learning_rate": 6.50206611570248e-07,
      "logits/chosen": 0.01226806640625,
      "logits/rejected": -0.2021484375,
      "logps/chosen": -382.0,
      "logps/rejected": -372.0,
      "loss": 0.0102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.60546875,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.4375,
      "step": 1396
    },
    {
      "epoch": 1.7141104294478526,
      "grad_norm": 8.251001626937791,
      "learning_rate": 6.495867768595042e-07,
      "logits/chosen": -0.1611328125,
      "logits/rejected": -0.306640625,
      "logps/chosen": -388.0,
      "logps/rejected": -388.0,
      "loss": 0.0266,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.09375,
      "rewards/margins": 12.5625,
      "rewards/rejected": -11.4375,
      "step": 1397
    },
    {
      "epoch": 1.7153374233128833,
      "grad_norm": 12.58694243429673,
      "learning_rate": 6.489669421487604e-07,
      "logits/chosen": -0.1044921875,
      "logits/rejected": -0.21484375,
      "logps/chosen": -432.0,
      "logps/rejected": -426.0,
      "loss": 0.0351,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.25,
      "step": 1398
    },
    {
      "epoch": 1.716564417177914,
      "grad_norm": 7.217759877360516,
      "learning_rate": 6.483471074380165e-07,
      "logits/chosen": -0.07275390625,
      "logits/rejected": -0.181640625,
      "logps/chosen": -362.0,
      "logps/rejected": -402.0,
      "loss": 0.0306,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.018798828125,
      "rewards/margins": 10.5,
      "rewards/rejected": -10.5625,
      "step": 1399
    },
    {
      "epoch": 1.7177914110429446,
      "grad_norm": 21.648162026893676,
      "learning_rate": 6.477272727272728e-07,
      "logits/chosen": -0.072265625,
      "logits/rejected": -0.271484375,
      "logps/chosen": -344.0,
      "logps/rejected": -352.0,
      "loss": 0.0312,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.5,
      "rewards/margins": 10.3125,
      "rewards/rejected": -10.8125,
      "step": 1400
    },
    {
      "epoch": 1.7190184049079753,
      "grad_norm": 6.36710851995934,
      "learning_rate": 6.47107438016529e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.36328125,
      "logps/chosen": -446.0,
      "logps/rejected": -442.0,
      "loss": 0.0253,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.8125,
      "step": 1401
    },
    {
      "epoch": 1.720245398773006,
      "grad_norm": 6.937087144639035,
      "learning_rate": 6.464876033057851e-07,
      "logits/chosen": -0.1611328125,
      "logits/rejected": -0.37890625,
      "logps/chosen": -406.0,
      "logps/rejected": -372.0,
      "loss": 0.0245,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.4453125,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.4375,
      "step": 1402
    },
    {
      "epoch": 1.7214723926380369,
      "grad_norm": 10.928603486238107,
      "learning_rate": 6.458677685950413e-07,
      "logits/chosen": -0.0498046875,
      "logits/rejected": -0.19921875,
      "logps/chosen": -406.0,
      "logps/rejected": -386.0,
      "loss": 0.036,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.10888671875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -10.8125,
      "step": 1403
    },
    {
      "epoch": 1.7226993865030675,
      "grad_norm": 15.583870620895178,
      "learning_rate": 6.452479338842975e-07,
      "logits/chosen": -0.07568359375,
      "logits/rejected": -0.228515625,
      "logps/chosen": -336.0,
      "logps/rejected": -330.0,
      "loss": 0.0482,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.0,
      "step": 1404
    },
    {
      "epoch": 1.7239263803680982,
      "grad_norm": 11.143077881808916,
      "learning_rate": 6.446280991735538e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.34765625,
      "logps/chosen": -428.0,
      "logps/rejected": -412.0,
      "loss": 0.0398,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.240234375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.5,
      "step": 1405
    },
    {
      "epoch": 1.7251533742331289,
      "grad_norm": 4.224299099232638,
      "learning_rate": 6.440082644628099e-07,
      "logits/chosen": -0.302734375,
      "logits/rejected": -0.421875,
      "logps/chosen": -328.0,
      "logps/rejected": -374.0,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39453125,
      "rewards/margins": 11.75,
      "rewards/rejected": -12.125,
      "step": 1406
    },
    {
      "epoch": 1.7263803680981595,
      "grad_norm": 13.330656715870173,
      "learning_rate": 6.433884297520661e-07,
      "logits/chosen": -0.111328125,
      "logits/rejected": -0.390625,
      "logps/chosen": -442.0,
      "logps/rejected": -426.0,
      "loss": 0.0462,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.3671875,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.375,
      "step": 1407
    },
    {
      "epoch": 1.7276073619631902,
      "grad_norm": 7.332633738244965,
      "learning_rate": 6.427685950413223e-07,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.30078125,
      "logps/chosen": -470.0,
      "logps/rejected": -440.0,
      "loss": 0.0309,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.765625,
      "rewards/margins": 13.0,
      "rewards/rejected": -12.1875,
      "step": 1408
    },
    {
      "epoch": 1.7288343558282209,
      "grad_norm": 12.35088317930436,
      "learning_rate": 6.421487603305786e-07,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -380.0,
      "logps/rejected": -376.0,
      "loss": 0.0446,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.0240478515625,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.75,
      "step": 1409
    },
    {
      "epoch": 1.7300613496932515,
      "grad_norm": 12.863248116987783,
      "learning_rate": 6.415289256198348e-07,
      "logits/chosen": -0.1787109375,
      "logits/rejected": -0.353515625,
      "logps/chosen": -400.0,
      "logps/rejected": -388.0,
      "loss": 0.0733,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": -0.5859375,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.5,
      "step": 1410
    },
    {
      "epoch": 1.7312883435582822,
      "grad_norm": 5.395301448670732,
      "learning_rate": 6.409090909090908e-07,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.279296875,
      "logps/chosen": -360.0,
      "logps/rejected": -382.0,
      "loss": 0.0197,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.451171875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.875,
      "step": 1411
    },
    {
      "epoch": 1.7325153374233129,
      "grad_norm": 11.512023543876147,
      "learning_rate": 6.402892561983471e-07,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.330078125,
      "logps/chosen": -430.0,
      "logps/rejected": -398.0,
      "loss": 0.063,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1904296875,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.625,
      "step": 1412
    },
    {
      "epoch": 1.7337423312883435,
      "grad_norm": 15.349502769641875,
      "learning_rate": 6.396694214876033e-07,
      "logits/chosen": -0.2333984375,
      "logits/rejected": -0.4453125,
      "logps/chosen": -400.0,
      "logps/rejected": -392.0,
      "loss": 0.0712,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.734375,
      "rewards/margins": 11.875,
      "rewards/rejected": -12.625,
      "step": 1413
    },
    {
      "epoch": 1.7349693251533742,
      "grad_norm": 10.27908988260348,
      "learning_rate": 6.390495867768596e-07,
      "logits/chosen": -0.10595703125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -436.0,
      "logps/rejected": -390.0,
      "loss": 0.0453,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.3125,
      "step": 1414
    },
    {
      "epoch": 1.7361963190184049,
      "grad_norm": 9.292914335874663,
      "learning_rate": 6.384297520661157e-07,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -350.0,
      "logps/rejected": -380.0,
      "loss": 0.041,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.15625,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.6875,
      "step": 1415
    },
    {
      "epoch": 1.7374233128834355,
      "grad_norm": 7.447145147683031,
      "learning_rate": 6.378099173553719e-07,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -420.0,
      "logps/rejected": -406.0,
      "loss": 0.027,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 12.625,
      "rewards/rejected": -12.0625,
      "step": 1416
    },
    {
      "epoch": 1.7386503067484662,
      "grad_norm": 8.792777557704559,
      "learning_rate": 6.371900826446281e-07,
      "logits/chosen": -0.2353515625,
      "logits/rejected": -0.37109375,
      "logps/chosen": -374.0,
      "logps/rejected": -386.0,
      "loss": 0.0291,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.162109375,
      "rewards/margins": 11.875,
      "rewards/rejected": -12.0,
      "step": 1417
    },
    {
      "epoch": 1.7398773006134969,
      "grad_norm": 13.38073206624744,
      "learning_rate": 6.365702479338844e-07,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.3671875,
      "logps/chosen": -396.0,
      "logps/rejected": -408.0,
      "loss": 0.0355,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.02587890625,
      "rewards/margins": 12.4375,
      "rewards/rejected": -12.4375,
      "step": 1418
    },
    {
      "epoch": 1.7411042944785278,
      "grad_norm": 10.607683414699668,
      "learning_rate": 6.359504132231405e-07,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.21875,
      "logps/chosen": -346.0,
      "logps/rejected": -342.0,
      "loss": 0.0453,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.63671875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -11.1875,
      "step": 1419
    },
    {
      "epoch": 1.7423312883435584,
      "grad_norm": 10.995863145556806,
      "learning_rate": 6.353305785123966e-07,
      "logits/chosen": -0.3203125,
      "logits/rejected": -0.466796875,
      "logps/chosen": -430.0,
      "logps/rejected": -422.0,
      "loss": 0.0369,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.00186920166015625,
      "rewards/margins": 12.375,
      "rewards/rejected": -12.375,
      "step": 1420
    },
    {
      "epoch": 1.743558282208589,
      "grad_norm": 4.993783324754888,
      "learning_rate": 6.347107438016529e-07,
      "logits/chosen": -0.2119140625,
      "logits/rejected": -0.412109375,
      "logps/chosen": -378.0,
      "logps/rejected": -392.0,
      "loss": 0.0189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1796875,
      "rewards/margins": 12.5625,
      "rewards/rejected": -12.375,
      "step": 1421
    },
    {
      "epoch": 1.7447852760736198,
      "grad_norm": 9.402562866512993,
      "learning_rate": 6.340909090909091e-07,
      "logits/chosen": -0.08935546875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -398.0,
      "logps/rejected": -392.0,
      "loss": 0.034,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1494140625,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.3125,
      "step": 1422
    },
    {
      "epoch": 1.7460122699386504,
      "grad_norm": 7.9284649955491595,
      "learning_rate": 6.334710743801654e-07,
      "logits/chosen": -0.16015625,
      "logits/rejected": -0.33984375,
      "logps/chosen": -382.0,
      "logps/rejected": -422.0,
      "loss": 0.0293,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.91796875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -13.25,
      "step": 1423
    },
    {
      "epoch": 1.747239263803681,
      "grad_norm": 6.1006118198416805,
      "learning_rate": 6.328512396694215e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": -0.49609375,
      "logps/chosen": -424.0,
      "logps/rejected": -410.0,
      "loss": 0.0223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23046875,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.875,
      "step": 1424
    },
    {
      "epoch": 1.7484662576687118,
      "grad_norm": 12.036819296468309,
      "learning_rate": 6.322314049586777e-07,
      "logits/chosen": -0.12255859375,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -386.0,
      "logps/rejected": -392.0,
      "loss": 0.0329,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1484375,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.75,
      "step": 1425
    },
    {
      "epoch": 1.7496932515337424,
      "grad_norm": 18.4014829822398,
      "learning_rate": 6.316115702479339e-07,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.30078125,
      "logps/chosen": -374.0,
      "logps/rejected": -388.0,
      "loss": 0.0692,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.1875,
      "step": 1426
    },
    {
      "epoch": 1.7509202453987731,
      "grad_norm": 12.513859109903631,
      "learning_rate": 6.309917355371902e-07,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.265625,
      "logps/chosen": -380.0,
      "logps/rejected": -386.0,
      "loss": 0.0557,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.427734375,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.875,
      "step": 1427
    },
    {
      "epoch": 1.7521472392638038,
      "grad_norm": 11.939507817491306,
      "learning_rate": 6.303719008264463e-07,
      "logits/chosen": -0.1845703125,
      "logits/rejected": -0.298828125,
      "logps/chosen": -420.0,
      "logps/rejected": -410.0,
      "loss": 0.0502,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.5,
      "step": 1428
    },
    {
      "epoch": 1.7533742331288344,
      "grad_norm": 6.914751025127418,
      "learning_rate": 6.297520661157024e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.296875,
      "logps/chosen": -354.0,
      "logps/rejected": -360.0,
      "loss": 0.0278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.37109375,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.125,
      "step": 1429
    },
    {
      "epoch": 1.7546012269938651,
      "grad_norm": 5.616187235790868,
      "learning_rate": 6.291322314049587e-07,
      "logits/chosen": -0.212890625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -388.0,
      "logps/rejected": -416.0,
      "loss": 0.0209,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.22265625,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.4375,
      "step": 1430
    },
    {
      "epoch": 1.7558282208588958,
      "grad_norm": 9.370787243161711,
      "learning_rate": 6.285123966942149e-07,
      "logits/chosen": -0.0196533203125,
      "logits/rejected": -0.181640625,
      "logps/chosen": -396.0,
      "logps/rejected": -380.0,
      "loss": 0.0259,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.0771484375,
      "rewards/margins": 10.4375,
      "rewards/rejected": -10.3125,
      "step": 1431
    },
    {
      "epoch": 1.7570552147239265,
      "grad_norm": 4.016040968990093,
      "learning_rate": 6.278925619834711e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.22265625,
      "logps/chosen": -336.0,
      "logps/rejected": -380.0,
      "loss": 0.012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.5,
      "step": 1432
    },
    {
      "epoch": 1.7582822085889571,
      "grad_norm": 12.729230743822482,
      "learning_rate": 6.272727272727272e-07,
      "logits/chosen": 0.0184326171875,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -360.0,
      "logps/rejected": -354.0,
      "loss": 0.0589,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.134765625,
      "rewards/margins": 10.0,
      "rewards/rejected": -10.1875,
      "step": 1433
    },
    {
      "epoch": 1.7595092024539878,
      "grad_norm": 18.378377289476965,
      "learning_rate": 6.266528925619835e-07,
      "logits/chosen": -0.0145263671875,
      "logits/rejected": -0.1103515625,
      "logps/chosen": -344.0,
      "logps/rejected": -410.0,
      "loss": 0.0787,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -10.125,
      "step": 1434
    },
    {
      "epoch": 1.7607361963190185,
      "grad_norm": 6.317649451474902,
      "learning_rate": 6.260330578512397e-07,
      "logits/chosen": -0.05810546875,
      "logits/rejected": -0.169921875,
      "logps/chosen": -352.0,
      "logps/rejected": -368.0,
      "loss": 0.0262,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.3125,
      "step": 1435
    },
    {
      "epoch": 1.7619631901840491,
      "grad_norm": 4.69256168204721,
      "learning_rate": 6.254132231404959e-07,
      "logits/chosen": -0.09619140625,
      "logits/rejected": -0.306640625,
      "logps/chosen": -388.0,
      "logps/rejected": -404.0,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.46875,
      "rewards/margins": 12.4375,
      "rewards/rejected": -10.9375,
      "step": 1436
    },
    {
      "epoch": 1.7631901840490798,
      "grad_norm": 29.257144870088368,
      "learning_rate": 6.247933884297521e-07,
      "logits/chosen": -0.248046875,
      "logits/rejected": -0.453125,
      "logps/chosen": -396.0,
      "logps/rejected": -416.0,
      "loss": 0.0505,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.53125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.375,
      "step": 1437
    },
    {
      "epoch": 1.7644171779141105,
      "grad_norm": 3.93681972201715,
      "learning_rate": 6.241735537190082e-07,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -382.0,
      "logps/rejected": -380.0,
      "loss": 0.0156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2138671875,
      "rewards/margins": 10.625,
      "rewards/rejected": -10.375,
      "step": 1438
    },
    {
      "epoch": 1.7656441717791411,
      "grad_norm": 7.385811382795677,
      "learning_rate": 6.235537190082645e-07,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.41015625,
      "logps/chosen": -368.0,
      "logps/rejected": -372.0,
      "loss": 0.0188,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.13671875,
      "rewards/margins": 10.5625,
      "rewards/rejected": -10.6875,
      "step": 1439
    },
    {
      "epoch": 1.7668711656441718,
      "grad_norm": 5.11550436529797,
      "learning_rate": 6.229338842975207e-07,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.34375,
      "logps/chosen": -470.0,
      "logps/rejected": -432.0,
      "loss": 0.0149,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.6875,
      "step": 1440
    },
    {
      "epoch": 1.7680981595092025,
      "grad_norm": 8.599376446703577,
      "learning_rate": 6.223140495867769e-07,
      "logits/chosen": -0.28515625,
      "logits/rejected": -0.359375,
      "logps/chosen": -360.0,
      "logps/rejected": -398.0,
      "loss": 0.0371,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.2001953125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.625,
      "step": 1441
    },
    {
      "epoch": 1.7693251533742331,
      "grad_norm": 20.234174712489157,
      "learning_rate": 6.21694214876033e-07,
      "logits/chosen": -0.21875,
      "logits/rejected": -0.43359375,
      "logps/chosen": -434.0,
      "logps/rejected": -404.0,
      "loss": 0.0897,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.057373046875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.375,
      "step": 1442
    },
    {
      "epoch": 1.7705521472392638,
      "grad_norm": 8.089818483065772,
      "learning_rate": 6.210743801652893e-07,
      "logits/chosen": -0.06640625,
      "logits/rejected": -0.21484375,
      "logps/chosen": -422.0,
      "logps/rejected": -416.0,
      "loss": 0.0331,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.46875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.5625,
      "step": 1443
    },
    {
      "epoch": 1.7717791411042945,
      "grad_norm": 7.490857640483026,
      "learning_rate": 6.204545454545455e-07,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -398.0,
      "logps/rejected": -398.0,
      "loss": 0.023,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.25,
      "step": 1444
    },
    {
      "epoch": 1.7730061349693251,
      "grad_norm": 4.507209000487265,
      "learning_rate": 6.198347107438017e-07,
      "logits/chosen": -0.09375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -330.0,
      "logps/rejected": -350.0,
      "loss": 0.0205,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.20703125,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.0,
      "step": 1445
    },
    {
      "epoch": 1.7742331288343558,
      "grad_norm": 9.506356568202962,
      "learning_rate": 6.192148760330579e-07,
      "logits/chosen": -0.0830078125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -418.0,
      "logps/rejected": -416.0,
      "loss": 0.0362,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.055908203125,
      "rewards/margins": 12.25,
      "rewards/rejected": -12.1875,
      "step": 1446
    },
    {
      "epoch": 1.7754601226993865,
      "grad_norm": 10.56612980003067,
      "learning_rate": 6.18595041322314e-07,
      "logits/chosen": -0.24609375,
      "logits/rejected": -0.40625,
      "logps/chosen": -386.0,
      "logps/rejected": -392.0,
      "loss": 0.0424,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2314453125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -12.0625,
      "step": 1447
    },
    {
      "epoch": 1.7766871165644171,
      "grad_norm": 17.112227162070322,
      "learning_rate": 6.179752066115703e-07,
      "logits/chosen": -0.0888671875,
      "logits/rejected": -0.208984375,
      "logps/chosen": -346.0,
      "logps/rejected": -376.0,
      "loss": 0.0438,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.75390625,
      "rewards/margins": 10.8125,
      "rewards/rejected": -11.5625,
      "step": 1448
    },
    {
      "epoch": 1.7779141104294478,
      "grad_norm": 7.673926079275721,
      "learning_rate": 6.173553719008264e-07,
      "logits/chosen": -0.003936767578125,
      "logits/rejected": -0.201171875,
      "logps/chosen": -430.0,
      "logps/rejected": -420.0,
      "loss": 0.0294,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.091796875,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.3125,
      "step": 1449
    },
    {
      "epoch": 1.7791411042944785,
      "grad_norm": 10.143124845251792,
      "learning_rate": 6.167355371900827e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -428.0,
      "logps/rejected": -398.0,
      "loss": 0.0361,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.099609375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.6875,
      "step": 1450
    },
    {
      "epoch": 1.7803680981595091,
      "grad_norm": 6.237913509142783,
      "learning_rate": 6.161157024793388e-07,
      "logits/chosen": -0.0751953125,
      "logits/rejected": -0.23046875,
      "logps/chosen": -422.0,
      "logps/rejected": -410.0,
      "loss": 0.0221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.8125,
      "step": 1451
    },
    {
      "epoch": 1.7815950920245398,
      "grad_norm": 13.734009163796884,
      "learning_rate": 6.154958677685951e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": -0.55078125,
      "logps/chosen": -458.0,
      "logps/rejected": -422.0,
      "loss": 0.0286,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 1.2265625,
      "rewards/margins": 13.5625,
      "rewards/rejected": -12.375,
      "step": 1452
    },
    {
      "epoch": 1.7828220858895705,
      "grad_norm": 5.142550278798512,
      "learning_rate": 6.148760330578512e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.3671875,
      "logps/chosen": -412.0,
      "logps/rejected": -388.0,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.265625,
      "rewards/margins": 12.375,
      "rewards/rejected": -12.125,
      "step": 1453
    },
    {
      "epoch": 1.7840490797546011,
      "grad_norm": 11.252004249668094,
      "learning_rate": 6.142561983471075e-07,
      "logits/chosen": -0.053955078125,
      "logits/rejected": -0.25390625,
      "logps/chosen": -402.0,
      "logps/rejected": -396.0,
      "loss": 0.0341,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.04833984375,
      "rewards/margins": 12.0,
      "rewards/rejected": -12.0,
      "step": 1454
    },
    {
      "epoch": 1.7852760736196318,
      "grad_norm": 5.828112394054352,
      "learning_rate": 6.136363636363637e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -398.0,
      "logps/rejected": -418.0,
      "loss": 0.024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1318359375,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.4375,
      "step": 1455
    },
    {
      "epoch": 1.7865030674846625,
      "grad_norm": 8.622121139819185,
      "learning_rate": 6.130165289256198e-07,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.29296875,
      "logps/chosen": -370.0,
      "logps/rejected": -366.0,
      "loss": 0.0437,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.232421875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -10.5625,
      "step": 1456
    },
    {
      "epoch": 1.7877300613496931,
      "grad_norm": 10.346638960597081,
      "learning_rate": 6.123966942148761e-07,
      "logits/chosen": -0.19140625,
      "logits/rejected": -0.41796875,
      "logps/chosen": -424.0,
      "logps/rejected": -418.0,
      "loss": 0.0402,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.09521484375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.75,
      "step": 1457
    },
    {
      "epoch": 1.7889570552147238,
      "grad_norm": 9.510862849605985,
      "learning_rate": 6.117768595041322e-07,
      "logits/chosen": -0.04736328125,
      "logits/rejected": -0.130859375,
      "logps/chosen": -360.0,
      "logps/rejected": -396.0,
      "loss": 0.0352,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.19921875,
      "rewards/margins": 10.3125,
      "rewards/rejected": -10.5,
      "step": 1458
    },
    {
      "epoch": 1.7901840490797545,
      "grad_norm": 2985.9302248963727,
      "learning_rate": 6.111570247933885e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.376953125,
      "logps/chosen": -404.0,
      "logps/rejected": -386.0,
      "loss": 1.516,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -1.1484375,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.8125,
      "step": 1459
    },
    {
      "epoch": 1.7914110429447851,
      "grad_norm": 6.456485664548984,
      "learning_rate": 6.105371900826446e-07,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.265625,
      "logps/chosen": -422.0,
      "logps/rejected": -406.0,
      "loss": 0.0201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.125,
      "step": 1460
    },
    {
      "epoch": 1.7926380368098158,
      "grad_norm": 12.803301057131963,
      "learning_rate": 6.099173553719009e-07,
      "logits/chosen": -0.11328125,
      "logits/rejected": -0.32421875,
      "logps/chosen": -430.0,
      "logps/rejected": -416.0,
      "loss": 0.0432,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 13.625,
      "rewards/rejected": -13.125,
      "step": 1461
    },
    {
      "epoch": 1.7938650306748465,
      "grad_norm": 8.137637416090907,
      "learning_rate": 6.09297520661157e-07,
      "logits/chosen": -0.13671875,
      "logits/rejected": -0.2021484375,
      "logps/chosen": -326.0,
      "logps/rejected": -372.0,
      "loss": 0.0317,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.201171875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.5,
      "step": 1462
    },
    {
      "epoch": 1.7950920245398772,
      "grad_norm": 9.5142759213274,
      "learning_rate": 6.086776859504133e-07,
      "logits/chosen": -0.058837890625,
      "logits/rejected": -0.29296875,
      "logps/chosen": -406.0,
      "logps/rejected": -392.0,
      "loss": 0.0229,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.33203125,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.0625,
      "step": 1463
    },
    {
      "epoch": 1.7963190184049078,
      "grad_norm": 16.72592677806428,
      "learning_rate": 6.080578512396695e-07,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.32421875,
      "logps/chosen": -346.0,
      "logps/rejected": -364.0,
      "loss": 0.0569,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.1328125,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.5,
      "step": 1464
    },
    {
      "epoch": 1.7975460122699385,
      "grad_norm": 8.34969796778253,
      "learning_rate": 6.074380165289256e-07,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.3515625,
      "logps/chosen": -396.0,
      "logps/rejected": -414.0,
      "loss": 0.0236,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 12.5625,
      "rewards/rejected": -12.0,
      "step": 1465
    },
    {
      "epoch": 1.7987730061349694,
      "grad_norm": 6.129551015730449,
      "learning_rate": 6.068181818181818e-07,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.359375,
      "logps/chosen": -364.0,
      "logps/rejected": -378.0,
      "loss": 0.0164,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.10205078125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -12.0,
      "step": 1466
    },
    {
      "epoch": 1.8,
      "grad_norm": 6.8967202729590245,
      "learning_rate": 6.06198347107438e-07,
      "logits/chosen": -0.2060546875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -426.0,
      "logps/rejected": -420.0,
      "loss": 0.0195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -10.9375,
      "step": 1467
    },
    {
      "epoch": 1.8012269938650307,
      "grad_norm": 8.361065202748346,
      "learning_rate": 6.055785123966943e-07,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.29296875,
      "logps/chosen": -412.0,
      "logps/rejected": -402.0,
      "loss": 0.0182,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.03125,
      "rewards/margins": 12.6875,
      "rewards/rejected": -11.625,
      "step": 1468
    },
    {
      "epoch": 1.8024539877300614,
      "grad_norm": 12.455616695494426,
      "learning_rate": 6.049586776859504e-07,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.224609375,
      "logps/chosen": -402.0,
      "logps/rejected": -390.0,
      "loss": 0.0396,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.06494140625,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.6875,
      "step": 1469
    },
    {
      "epoch": 1.803680981595092,
      "grad_norm": 9.292975088281931,
      "learning_rate": 6.043388429752066e-07,
      "logits/chosen": -0.05126953125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -404.0,
      "logps/rejected": -384.0,
      "loss": 0.0445,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.375,
      "step": 1470
    },
    {
      "epoch": 1.8049079754601227,
      "grad_norm": 9.047461001821528,
      "learning_rate": 6.037190082644628e-07,
      "logits/chosen": -0.07177734375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -372.0,
      "logps/rejected": -396.0,
      "loss": 0.0295,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.478515625,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.6875,
      "step": 1471
    },
    {
      "epoch": 1.8061349693251534,
      "grad_norm": 13.42466742331635,
      "learning_rate": 6.030991735537191e-07,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -392.0,
      "logps/rejected": -398.0,
      "loss": 0.04,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.34765625,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.875,
      "step": 1472
    },
    {
      "epoch": 1.807361963190184,
      "grad_norm": 6.913536686973756,
      "learning_rate": 6.024793388429752e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.40234375,
      "logps/chosen": -384.0,
      "logps/rejected": -390.0,
      "loss": 0.0335,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 12.5625,
      "rewards/rejected": -12.25,
      "step": 1473
    },
    {
      "epoch": 1.8085889570552147,
      "grad_norm": 17.128950617994402,
      "learning_rate": 6.018595041322314e-07,
      "logits/chosen": -0.017822265625,
      "logits/rejected": -0.234375,
      "logps/chosen": -426.0,
      "logps/rejected": -418.0,
      "loss": 0.0425,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.318359375,
      "rewards/margins": 12.25,
      "rewards/rejected": -11.9375,
      "step": 1474
    },
    {
      "epoch": 1.8098159509202454,
      "grad_norm": 13.951944722139784,
      "learning_rate": 6.012396694214876e-07,
      "logits/chosen": -0.154296875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -402.0,
      "logps/rejected": -398.0,
      "loss": 0.0357,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1357421875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.9375,
      "step": 1475
    },
    {
      "epoch": 1.811042944785276,
      "grad_norm": 12.939142286970506,
      "learning_rate": 6.006198347107438e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -368.0,
      "logps/rejected": -392.0,
      "loss": 0.0343,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.032470703125,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.4375,
      "step": 1476
    },
    {
      "epoch": 1.8122699386503067,
      "grad_norm": 15.184402213006727,
      "learning_rate": 6.000000000000001e-07,
      "logits/chosen": -0.057861328125,
      "logits/rejected": -0.185546875,
      "logps/chosen": -390.0,
      "logps/rejected": -404.0,
      "loss": 0.0564,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.54296875,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.6875,
      "step": 1477
    },
    {
      "epoch": 1.8134969325153374,
      "grad_norm": 7.953555230394229,
      "learning_rate": 5.993801652892562e-07,
      "logits/chosen": -0.09619140625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -402.0,
      "logps/rejected": -424.0,
      "loss": 0.027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8828125,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.5,
      "step": 1478
    },
    {
      "epoch": 1.814723926380368,
      "grad_norm": 11.656764281029574,
      "learning_rate": 5.987603305785124e-07,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.359375,
      "logps/chosen": -384.0,
      "logps/rejected": -396.0,
      "loss": 0.0344,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.25,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.9375,
      "step": 1479
    },
    {
      "epoch": 1.8159509202453987,
      "grad_norm": 10.973049014636235,
      "learning_rate": 5.981404958677686e-07,
      "logits/chosen": -0.267578125,
      "logits/rejected": -0.408203125,
      "logps/chosen": -396.0,
      "logps/rejected": -424.0,
      "loss": 0.0437,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.72265625,
      "rewards/margins": 11.375,
      "rewards/rejected": -10.6875,
      "step": 1480
    },
    {
      "epoch": 1.8171779141104294,
      "grad_norm": 16.33917392069797,
      "learning_rate": 5.975206611570249e-07,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -434.0,
      "logps/rejected": -434.0,
      "loss": 0.0484,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.1962890625,
      "rewards/margins": 11.9375,
      "rewards/rejected": -12.125,
      "step": 1481
    },
    {
      "epoch": 1.8184049079754603,
      "grad_norm": 13.067184042335276,
      "learning_rate": 5.96900826446281e-07,
      "logits/chosen": -0.046142578125,
      "logits/rejected": -0.203125,
      "logps/chosen": -374.0,
      "logps/rejected": -376.0,
      "loss": 0.0686,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.08984375,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.25,
      "step": 1482
    },
    {
      "epoch": 1.819631901840491,
      "grad_norm": 14.380040955958671,
      "learning_rate": 5.962809917355371e-07,
      "logits/chosen": 0.0086669921875,
      "logits/rejected": -0.146484375,
      "logps/chosen": -356.0,
      "logps/rejected": -342.0,
      "loss": 0.0636,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.322265625,
      "rewards/margins": 11.0,
      "rewards/rejected": -11.3125,
      "step": 1483
    },
    {
      "epoch": 1.8208588957055216,
      "grad_norm": 13.600405309118111,
      "learning_rate": 5.956611570247934e-07,
      "logits/chosen": -0.2158203125,
      "logits/rejected": -0.373046875,
      "logps/chosen": -354.0,
      "logps/rejected": -374.0,
      "loss": 0.0529,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.4375,
      "step": 1484
    },
    {
      "epoch": 1.8220858895705523,
      "grad_norm": 19.36891506988035,
      "learning_rate": 5.950413223140496e-07,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.23046875,
      "logps/chosen": -350.0,
      "logps/rejected": -424.0,
      "loss": 0.0361,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.5859375,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.6875,
      "step": 1485
    },
    {
      "epoch": 1.823312883435583,
      "grad_norm": 13.246751492795608,
      "learning_rate": 5.944214876033059e-07,
      "logits/chosen": -0.2890625,
      "logits/rejected": -0.447265625,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.0601,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.38671875,
      "rewards/margins": 11.75,
      "rewards/rejected": -12.125,
      "step": 1486
    },
    {
      "epoch": 1.8245398773006136,
      "grad_norm": 3.3945752544087964,
      "learning_rate": 5.93801652892562e-07,
      "logits/chosen": -0.34375,
      "logits/rejected": -0.5078125,
      "logps/chosen": -416.0,
      "logps/rejected": -388.0,
      "loss": 0.013,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.07958984375,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.4375,
      "step": 1487
    },
    {
      "epoch": 1.8257668711656443,
      "grad_norm": 8.043522667448203,
      "learning_rate": 5.931818181818182e-07,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.462890625,
      "logps/chosen": -416.0,
      "logps/rejected": -378.0,
      "loss": 0.0227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1240234375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.0625,
      "step": 1488
    },
    {
      "epoch": 1.826993865030675,
      "grad_norm": 10.658513612446942,
      "learning_rate": 5.925619834710744e-07,
      "logits/chosen": -0.333984375,
      "logits/rejected": -0.54296875,
      "logps/chosen": -410.0,
      "logps/rejected": -408.0,
      "loss": 0.0417,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.162109375,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.5625,
      "step": 1489
    },
    {
      "epoch": 1.8282208588957056,
      "grad_norm": 5.907167171601785,
      "learning_rate": 5.919421487603307e-07,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.416015625,
      "logps/chosen": -398.0,
      "logps/rejected": -404.0,
      "loss": 0.0174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.71875,
      "rewards/margins": 12.875,
      "rewards/rejected": -12.125,
      "step": 1490
    },
    {
      "epoch": 1.8294478527607363,
      "grad_norm": 7.708311095918759,
      "learning_rate": 5.913223140495868e-07,
      "logits/chosen": -0.2109375,
      "logits/rejected": -0.466796875,
      "logps/chosen": -484.0,
      "logps/rejected": -444.0,
      "loss": 0.0265,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 13.0625,
      "rewards/rejected": -12.125,
      "step": 1491
    },
    {
      "epoch": 1.830674846625767,
      "grad_norm": 11.650261479643984,
      "learning_rate": 5.907024793388429e-07,
      "logits/chosen": -0.349609375,
      "logits/rejected": -0.498046875,
      "logps/chosen": -390.0,
      "logps/rejected": -396.0,
      "loss": 0.0365,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.03369140625,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.8125,
      "step": 1492
    },
    {
      "epoch": 1.8319018404907976,
      "grad_norm": 6.3612166084027235,
      "learning_rate": 5.900826446280992e-07,
      "logits/chosen": -0.3125,
      "logits/rejected": -0.42578125,
      "logps/chosen": -330.0,
      "logps/rejected": -366.0,
      "loss": 0.0247,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.97265625,
      "rewards/margins": 11.375,
      "rewards/rejected": -12.375,
      "step": 1493
    },
    {
      "epoch": 1.8331288343558283,
      "grad_norm": 7.617904477738232,
      "learning_rate": 5.894628099173554e-07,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.38671875,
      "logps/chosen": -442.0,
      "logps/rejected": -438.0,
      "loss": 0.0286,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.1708984375,
      "rewards/margins": 12.125,
      "rewards/rejected": -12.3125,
      "step": 1494
    },
    {
      "epoch": 1.834355828220859,
      "grad_norm": 14.805530582972839,
      "learning_rate": 5.888429752066116e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.369140625,
      "logps/chosen": -366.0,
      "logps/rejected": -378.0,
      "loss": 0.0427,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.051025390625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.0,
      "step": 1495
    },
    {
      "epoch": 1.8355828220858896,
      "grad_norm": 8.603548186700504,
      "learning_rate": 5.882231404958677e-07,
      "logits/chosen": -0.1533203125,
      "logits/rejected": -0.2890625,
      "logps/chosen": -374.0,
      "logps/rejected": -394.0,
      "loss": 0.0246,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.3125,
      "step": 1496
    },
    {
      "epoch": 1.8368098159509203,
      "grad_norm": 5.119681631341246,
      "learning_rate": 5.87603305785124e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.375,
      "logps/chosen": -414.0,
      "logps/rejected": -380.0,
      "loss": 0.0172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.466796875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.0625,
      "step": 1497
    },
    {
      "epoch": 1.838036809815951,
      "grad_norm": 13.256747203318977,
      "learning_rate": 5.869834710743802e-07,
      "logits/chosen": -0.31640625,
      "logits/rejected": -0.49609375,
      "logps/chosen": -424.0,
      "logps/rejected": -436.0,
      "loss": 0.0546,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.328125,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.3125,
      "step": 1498
    },
    {
      "epoch": 1.8392638036809816,
      "grad_norm": 12.35912732127943,
      "learning_rate": 5.863636363636365e-07,
      "logits/chosen": -0.1181640625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -388.0,
      "logps/rejected": -396.0,
      "loss": 0.0348,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.58984375,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.375,
      "step": 1499
    },
    {
      "epoch": 1.8404907975460123,
      "grad_norm": 7.9066533942342945,
      "learning_rate": 5.857438016528925e-07,
      "logits/chosen": -0.1474609375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -370.0,
      "logps/rejected": -348.0,
      "loss": 0.034,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.015625,
      "rewards/margins": 11.75,
      "rewards/rejected": -10.75,
      "step": 1500
    },
    {
      "epoch": 1.841717791411043,
      "grad_norm": 7.790234578512824,
      "learning_rate": 5.851239669421487e-07,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.27734375,
      "logps/chosen": -374.0,
      "logps/rejected": -398.0,
      "loss": 0.0245,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.61328125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -11.25,
      "step": 1501
    },
    {
      "epoch": 1.8429447852760736,
      "grad_norm": 8.476340764027082,
      "learning_rate": 5.84504132231405e-07,
      "logits/chosen": -0.087890625,
      "logits/rejected": -0.1953125,
      "logps/chosen": -366.0,
      "logps/rejected": -394.0,
      "loss": 0.0284,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.5625,
      "step": 1502
    },
    {
      "epoch": 1.8441717791411043,
      "grad_norm": 8.931095134479317,
      "learning_rate": 5.838842975206612e-07,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.412109375,
      "logps/chosen": -430.0,
      "logps/rejected": -456.0,
      "loss": 0.0258,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 12.125,
      "rewards/rejected": -11.6875,
      "step": 1503
    },
    {
      "epoch": 1.845398773006135,
      "grad_norm": 9.61981018831853,
      "learning_rate": 5.832644628099174e-07,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.408203125,
      "logps/chosen": -388.0,
      "logps/rejected": -408.0,
      "loss": 0.0334,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.58203125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -10.9375,
      "step": 1504
    },
    {
      "epoch": 1.8466257668711656,
      "grad_norm": 8.898568846872344,
      "learning_rate": 5.826446280991735e-07,
      "logits/chosen": -0.23828125,
      "logits/rejected": -0.404296875,
      "logps/chosen": -416.0,
      "logps/rejected": -420.0,
      "loss": 0.0376,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2099609375,
      "rewards/margins": 11.625,
      "rewards/rejected": -11.4375,
      "step": 1505
    },
    {
      "epoch": 1.8478527607361963,
      "grad_norm": 8.234391016597,
      "learning_rate": 5.820247933884298e-07,
      "logits/chosen": -0.1787109375,
      "logits/rejected": -0.353515625,
      "logps/chosen": -358.0,
      "logps/rejected": -398.0,
      "loss": 0.0295,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.48046875,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.625,
      "step": 1506
    },
    {
      "epoch": 1.849079754601227,
      "grad_norm": 12.615811463348589,
      "learning_rate": 5.81404958677686e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.365234375,
      "logps/chosen": -400.0,
      "logps/rejected": -390.0,
      "loss": 0.0434,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.609375,
      "rewards/margins": 11.1875,
      "rewards/rejected": -10.625,
      "step": 1507
    },
    {
      "epoch": 1.8503067484662576,
      "grad_norm": 11.413630239199696,
      "learning_rate": 5.807851239669422e-07,
      "logits/chosen": -0.059326171875,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -392.0,
      "logps/rejected": -404.0,
      "loss": 0.0325,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.55859375,
      "rewards/margins": 10.875,
      "rewards/rejected": -11.4375,
      "step": 1508
    },
    {
      "epoch": 1.8515337423312883,
      "grad_norm": 7.238823334535515,
      "learning_rate": 5.801652892561983e-07,
      "logits/chosen": -0.1611328125,
      "logits/rejected": -0.361328125,
      "logps/chosen": -444.0,
      "logps/rejected": -422.0,
      "loss": 0.0186,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.453125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -10.875,
      "step": 1509
    },
    {
      "epoch": 1.852760736196319,
      "grad_norm": 4.424946915077485,
      "learning_rate": 5.795454545454545e-07,
      "logits/chosen": -0.126953125,
      "logits/rejected": -0.375,
      "logps/chosen": -432.0,
      "logps/rejected": -398.0,
      "loss": 0.0131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.53515625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.625,
      "step": 1510
    },
    {
      "epoch": 1.8539877300613496,
      "grad_norm": 7.053221824008459,
      "learning_rate": 5.789256198347108e-07,
      "logits/chosen": -0.203125,
      "logits/rejected": -0.267578125,
      "logps/chosen": -420.0,
      "logps/rejected": -456.0,
      "loss": 0.0169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.322265625,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.1875,
      "step": 1511
    },
    {
      "epoch": 1.8552147239263803,
      "grad_norm": 3.5547744931254903,
      "learning_rate": 5.783057851239669e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.373046875,
      "logps/chosen": -358.0,
      "logps/rejected": -364.0,
      "loss": 0.0135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9921875,
      "rewards/margins": 13.0,
      "rewards/rejected": -12.0,
      "step": 1512
    },
    {
      "epoch": 1.856441717791411,
      "grad_norm": 15.688605244420566,
      "learning_rate": 5.776859504132232e-07,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.376953125,
      "logps/chosen": -404.0,
      "logps/rejected": -378.0,
      "loss": 0.0586,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.12451171875,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.625,
      "step": 1513
    },
    {
      "epoch": 1.8576687116564417,
      "grad_norm": 5.4110839912069,
      "learning_rate": 5.770661157024793e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.37109375,
      "logps/chosen": -412.0,
      "logps/rejected": -426.0,
      "loss": 0.0181,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.369140625,
      "rewards/margins": 13.125,
      "rewards/rejected": -12.75,
      "step": 1514
    },
    {
      "epoch": 1.8588957055214723,
      "grad_norm": 10.478869181509733,
      "learning_rate": 5.764462809917356e-07,
      "logits/chosen": -0.2265625,
      "logits/rejected": -0.4453125,
      "logps/chosen": -408.0,
      "logps/rejected": -390.0,
      "loss": 0.0311,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 13.0625,
      "rewards/rejected": -12.125,
      "step": 1515
    },
    {
      "epoch": 1.860122699386503,
      "grad_norm": 9.184505008497789,
      "learning_rate": 5.758264462809918e-07,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.328125,
      "logps/chosen": -420.0,
      "logps/rejected": -416.0,
      "loss": 0.0329,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 11.125,
      "rewards/rejected": -11.9375,
      "step": 1516
    },
    {
      "epoch": 1.8613496932515337,
      "grad_norm": 4.539913274268795,
      "learning_rate": 5.75206611570248e-07,
      "logits/chosen": -0.224609375,
      "logits/rejected": -0.42578125,
      "logps/chosen": -418.0,
      "logps/rejected": -398.0,
      "loss": 0.0172,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 13.0,
      "rewards/rejected": -12.3125,
      "step": 1517
    },
    {
      "epoch": 1.8625766871165643,
      "grad_norm": 14.818256983584963,
      "learning_rate": 5.745867768595041e-07,
      "logits/chosen": -0.2451171875,
      "logits/rejected": -0.443359375,
      "logps/chosen": -384.0,
      "logps/rejected": -372.0,
      "loss": 0.0612,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3125,
      "rewards/margins": 12.3125,
      "rewards/rejected": -12.625,
      "step": 1518
    },
    {
      "epoch": 1.863803680981595,
      "grad_norm": 21.719699854423048,
      "learning_rate": 5.739669421487603e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -374.0,
      "logps/rejected": -412.0,
      "loss": 0.0598,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 10.875,
      "rewards/rejected": -12.125,
      "step": 1519
    },
    {
      "epoch": 1.8650306748466257,
      "grad_norm": 4.643580726862002,
      "learning_rate": 5.733471074380166e-07,
      "logits/chosen": -0.2236328125,
      "logits/rejected": -0.30078125,
      "logps/chosen": -396.0,
      "logps/rejected": -404.0,
      "loss": 0.0153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41796875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -12.5,
      "step": 1520
    },
    {
      "epoch": 1.8662576687116563,
      "grad_norm": 9.963670178751276,
      "learning_rate": 5.727272727272727e-07,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -374.0,
      "logps/rejected": -372.0,
      "loss": 0.0385,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.28125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.8125,
      "step": 1521
    },
    {
      "epoch": 1.867484662576687,
      "grad_norm": 13.279186623430544,
      "learning_rate": 5.721074380165289e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.35546875,
      "logps/chosen": -370.0,
      "logps/rejected": -374.0,
      "loss": 0.0478,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.0673828125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.5,
      "step": 1522
    },
    {
      "epoch": 1.8687116564417177,
      "grad_norm": 6.578971949059929,
      "learning_rate": 5.714876033057851e-07,
      "logits/chosen": -0.2412109375,
      "logits/rejected": -0.373046875,
      "logps/chosen": -376.0,
      "logps/rejected": -376.0,
      "loss": 0.0198,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1591796875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.875,
      "step": 1523
    },
    {
      "epoch": 1.8699386503067483,
      "grad_norm": 13.581236746412541,
      "learning_rate": 5.708677685950414e-07,
      "logits/chosen": -0.083984375,
      "logits/rejected": -0.236328125,
      "logps/chosen": -348.0,
      "logps/rejected": -400.0,
      "loss": 0.04,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.75,
      "rewards/margins": 11.4375,
      "rewards/rejected": -12.1875,
      "step": 1524
    },
    {
      "epoch": 1.871165644171779,
      "grad_norm": 8.911113085416726,
      "learning_rate": 5.702479338842975e-07,
      "logits/chosen": -0.2099609375,
      "logits/rejected": -0.36328125,
      "logps/chosen": -352.0,
      "logps/rejected": -390.0,
      "loss": 0.0247,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.470703125,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.9375,
      "step": 1525
    },
    {
      "epoch": 1.8723926380368097,
      "grad_norm": 12.467591280037412,
      "learning_rate": 5.696280991735538e-07,
      "logits/chosen": -0.07470703125,
      "logits/rejected": -0.26953125,
      "logps/chosen": -410.0,
      "logps/rejected": -390.0,
      "loss": 0.0599,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.671875,
      "rewards/margins": 11.6875,
      "rewards/rejected": -11.0625,
      "step": 1526
    },
    {
      "epoch": 1.8736196319018403,
      "grad_norm": 7.9523490622772,
      "learning_rate": 5.690082644628099e-07,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.3203125,
      "logps/chosen": -412.0,
      "logps/rejected": -366.0,
      "loss": 0.0304,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.48046875,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.25,
      "step": 1527
    },
    {
      "epoch": 1.874846625766871,
      "grad_norm": 16.90843634388907,
      "learning_rate": 5.683884297520661e-07,
      "logits/chosen": -0.07470703125,
      "logits/rejected": -0.29296875,
      "logps/chosen": -398.0,
      "logps/rejected": -362.0,
      "loss": 0.0687,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.5,
      "step": 1528
    },
    {
      "epoch": 1.876073619631902,
      "grad_norm": 5.8636813967032415,
      "learning_rate": 5.677685950413223e-07,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.447265625,
      "logps/chosen": -422.0,
      "logps/rejected": -402.0,
      "loss": 0.0195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.359375,
      "rewards/margins": 13.8125,
      "rewards/rejected": -12.4375,
      "step": 1529
    },
    {
      "epoch": 1.8773006134969326,
      "grad_norm": 10.180871320115374,
      "learning_rate": 5.671487603305785e-07,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.326171875,
      "logps/chosen": -388.0,
      "logps/rejected": -368.0,
      "loss": 0.0404,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.7265625,
      "rewards/margins": 11.8125,
      "rewards/rejected": -12.5625,
      "step": 1530
    },
    {
      "epoch": 1.8785276073619632,
      "grad_norm": 9.962751714095937,
      "learning_rate": 5.665289256198347e-07,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.1953125,
      "logps/chosen": -378.0,
      "logps/rejected": -374.0,
      "loss": 0.0435,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.3984375,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.875,
      "step": 1531
    },
    {
      "epoch": 1.879754601226994,
      "grad_norm": 13.344235509044795,
      "learning_rate": 5.659090909090909e-07,
      "logits/chosen": -0.328125,
      "logits/rejected": -0.4921875,
      "logps/chosen": -436.0,
      "logps/rejected": -450.0,
      "loss": 0.0678,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.6953125,
      "rewards/margins": 13.625,
      "rewards/rejected": -12.9375,
      "step": 1532
    },
    {
      "epoch": 1.8809815950920246,
      "grad_norm": 12.313294846355545,
      "learning_rate": 5.652892561983472e-07,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.26171875,
      "logps/chosen": -382.0,
      "logps/rejected": -402.0,
      "loss": 0.0423,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.057861328125,
      "rewards/margins": 11.0625,
      "rewards/rejected": -11.0,
      "step": 1533
    },
    {
      "epoch": 1.8822085889570552,
      "grad_norm": 7.89566619418515,
      "learning_rate": 5.646694214876033e-07,
      "logits/chosen": -0.205078125,
      "logits/rejected": -0.337890625,
      "logps/chosen": -392.0,
      "logps/rejected": -414.0,
      "loss": 0.025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.59375,
      "rewards/margins": 12.0,
      "rewards/rejected": -12.5625,
      "step": 1534
    },
    {
      "epoch": 1.883435582822086,
      "grad_norm": 11.645948617484482,
      "learning_rate": 5.640495867768596e-07,
      "logits/chosen": -0.046875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -374.0,
      "logps/rejected": -360.0,
      "loss": 0.0563,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.404296875,
      "rewards/margins": 11.9375,
      "rewards/rejected": -11.5625,
      "step": 1535
    },
    {
      "epoch": 1.8846625766871166,
      "grad_norm": 12.956631655705353,
      "learning_rate": 5.634297520661157e-07,
      "logits/chosen": -0.056640625,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -458.0,
      "logps/rejected": -440.0,
      "loss": 0.0515,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.34375,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.5625,
      "step": 1536
    },
    {
      "epoch": 1.8858895705521472,
      "grad_norm": 4.634384436264029,
      "learning_rate": 5.628099173553719e-07,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.34375,
      "logps/chosen": -380.0,
      "logps/rejected": -390.0,
      "loss": 0.0124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.037353515625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.125,
      "step": 1537
    },
    {
      "epoch": 1.887116564417178,
      "grad_norm": 5.6435058037855965,
      "learning_rate": 5.621900826446281e-07,
      "logits/chosen": -0.10498046875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -366.0,
      "logps/rejected": -394.0,
      "loss": 0.0103,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.51953125,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.875,
      "step": 1538
    },
    {
      "epoch": 1.8883435582822086,
      "grad_norm": 8.930748067631301,
      "learning_rate": 5.615702479338843e-07,
      "logits/chosen": -0.01275634765625,
      "logits/rejected": -0.240234375,
      "logps/chosen": -380.0,
      "logps/rejected": -390.0,
      "loss": 0.0294,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.201171875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.375,
      "step": 1539
    },
    {
      "epoch": 1.8895705521472392,
      "grad_norm": 10.323806155698314,
      "learning_rate": 5.609504132231405e-07,
      "logits/chosen": -0.302734375,
      "logits/rejected": -0.4375,
      "logps/chosen": -438.0,
      "logps/rejected": -468.0,
      "loss": 0.0427,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.486328125,
      "rewards/margins": 12.625,
      "rewards/rejected": -12.125,
      "step": 1540
    },
    {
      "epoch": 1.89079754601227,
      "grad_norm": 11.351287245365558,
      "learning_rate": 5.603305785123967e-07,
      "logits/chosen": -0.2421875,
      "logits/rejected": -0.396484375,
      "logps/chosen": -384.0,
      "logps/rejected": -406.0,
      "loss": 0.0483,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.11669921875,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.875,
      "step": 1541
    },
    {
      "epoch": 1.8920245398773006,
      "grad_norm": 3.2489476684935736,
      "learning_rate": 5.597107438016529e-07,
      "logits/chosen": -0.2041015625,
      "logits/rejected": -0.375,
      "logps/chosen": -404.0,
      "logps/rejected": -406.0,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.80078125,
      "rewards/margins": 13.1875,
      "rewards/rejected": -12.375,
      "step": 1542
    },
    {
      "epoch": 1.8932515337423312,
      "grad_norm": 8.504627067419271,
      "learning_rate": 5.590909090909091e-07,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.357421875,
      "logps/chosen": -378.0,
      "logps/rejected": -396.0,
      "loss": 0.0291,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.3125,
      "step": 1543
    },
    {
      "epoch": 1.8944785276073621,
      "grad_norm": 8.19609184280983,
      "learning_rate": 5.584710743801654e-07,
      "logits/chosen": -0.2451171875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -422.0,
      "logps/rejected": -450.0,
      "loss": 0.0282,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.314453125,
      "rewards/margins": 12.625,
      "rewards/rejected": -12.3125,
      "step": 1544
    },
    {
      "epoch": 1.8957055214723928,
      "grad_norm": 8.691100794961908,
      "learning_rate": 5.578512396694215e-07,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.326171875,
      "logps/chosen": -392.0,
      "logps/rejected": -358.0,
      "loss": 0.0275,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 10.75,
      "rewards/rejected": -11.0625,
      "step": 1545
    },
    {
      "epoch": 1.8969325153374235,
      "grad_norm": 8.614624685931496,
      "learning_rate": 5.572314049586777e-07,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -376.0,
      "logps/rejected": -372.0,
      "loss": 0.024,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1435546875,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.375,
      "step": 1546
    },
    {
      "epoch": 1.8981595092024541,
      "grad_norm": 15.386153310990915,
      "learning_rate": 5.566115702479339e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.484375,
      "logps/chosen": -490.0,
      "logps/rejected": -418.0,
      "loss": 0.0277,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.361328125,
      "rewards/margins": 13.1875,
      "rewards/rejected": -12.8125,
      "step": 1547
    },
    {
      "epoch": 1.8993865030674848,
      "grad_norm": 21.182540544956645,
      "learning_rate": 5.559917355371901e-07,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -400.0,
      "logps/rejected": -430.0,
      "loss": 0.0662,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.11376953125,
      "rewards/margins": 11.9375,
      "rewards/rejected": -12.0625,
      "step": 1548
    },
    {
      "epoch": 1.9006134969325155,
      "grad_norm": 9.056705219686792,
      "learning_rate": 5.553719008264463e-07,
      "logits/chosen": -0.046875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -388.0,
      "logps/rejected": -374.0,
      "loss": 0.032,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.0185546875,
      "rewards/margins": 11.4375,
      "rewards/rejected": -11.4375,
      "step": 1549
    },
    {
      "epoch": 1.9018404907975461,
      "grad_norm": 12.174899827400916,
      "learning_rate": 5.547520661157025e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.2890625,
      "logps/chosen": -394.0,
      "logps/rejected": -396.0,
      "loss": 0.0361,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.0478515625,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.6875,
      "step": 1550
    },
    {
      "epoch": 1.9030674846625768,
      "grad_norm": 11.776464119336328,
      "learning_rate": 5.541322314049587e-07,
      "logits/chosen": -0.2177734375,
      "logits/rejected": -0.42578125,
      "logps/chosen": -434.0,
      "logps/rejected": -416.0,
      "loss": 0.0399,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.466796875,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.8125,
      "step": 1551
    },
    {
      "epoch": 1.9042944785276075,
      "grad_norm": 15.301018527355753,
      "learning_rate": 5.535123966942149e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.26953125,
      "logps/chosen": -358.0,
      "logps/rejected": -362.0,
      "loss": 0.071,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.19140625,
      "rewards/margins": 10.375,
      "rewards/rejected": -10.5625,
      "step": 1552
    },
    {
      "epoch": 1.9055214723926381,
      "grad_norm": 13.713477834316057,
      "learning_rate": 5.528925619834712e-07,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -392.0,
      "logps/rejected": -356.0,
      "loss": 0.0288,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.015625,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.5,
      "step": 1553
    },
    {
      "epoch": 1.9067484662576688,
      "grad_norm": 13.970935293353676,
      "learning_rate": 5.522727272727273e-07,
      "logits/chosen": -0.29296875,
      "logits/rejected": -0.55078125,
      "logps/chosen": -426.0,
      "logps/rejected": -406.0,
      "loss": 0.0417,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.05810546875,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.125,
      "step": 1554
    },
    {
      "epoch": 1.9079754601226995,
      "grad_norm": 17.08022297387505,
      "learning_rate": 5.516528925619834e-07,
      "logits/chosen": -0.306640625,
      "logits/rejected": -0.318359375,
      "logps/chosen": -388.0,
      "logps/rejected": -414.0,
      "loss": 0.0317,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.18359375,
      "rewards/margins": 11.875,
      "rewards/rejected": -12.0625,
      "step": 1555
    },
    {
      "epoch": 1.9092024539877301,
      "grad_norm": 5.512042362005046,
      "learning_rate": 5.510330578512397e-07,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.3359375,
      "logps/chosen": -410.0,
      "logps/rejected": -392.0,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23828125,
      "rewards/margins": 11.5625,
      "rewards/rejected": -11.375,
      "step": 1556
    },
    {
      "epoch": 1.9104294478527608,
      "grad_norm": 9.862650604155213,
      "learning_rate": 5.504132231404959e-07,
      "logits/chosen": -0.01397705078125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -404.0,
      "logps/rejected": -414.0,
      "loss": 0.0364,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.041748046875,
      "rewards/margins": 11.25,
      "rewards/rejected": -11.1875,
      "step": 1557
    },
    {
      "epoch": 1.9116564417177915,
      "grad_norm": 9.10335392738119,
      "learning_rate": 5.497933884297521e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.337890625,
      "logps/chosen": -394.0,
      "logps/rejected": -432.0,
      "loss": 0.0229,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.296875,
      "rewards/margins": 12.375,
      "rewards/rejected": -12.0625,
      "step": 1558
    },
    {
      "epoch": 1.9128834355828221,
      "grad_norm": 10.053624701482729,
      "learning_rate": 5.491735537190082e-07,
      "logits/chosen": -0.032958984375,
      "logits/rejected": -0.1162109375,
      "logps/chosen": -396.0,
      "logps/rejected": -354.0,
      "loss": 0.0385,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1142578125,
      "rewards/margins": 10.9375,
      "rewards/rejected": -11.0625,
      "step": 1559
    },
    {
      "epoch": 1.9141104294478528,
      "grad_norm": 6.811810662950314,
      "learning_rate": 5.485537190082645e-07,
      "logits/chosen": -0.09228515625,
      "logits/rejected": -0.287109375,
      "logps/chosen": -376.0,
      "logps/rejected": -370.0,
      "loss": 0.0255,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.91796875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -12.125,
      "step": 1560
    },
    {
      "epoch": 1.9153374233128835,
      "grad_norm": 10.499380910825229,
      "learning_rate": 5.479338842975207e-07,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.470703125,
      "logps/chosen": -456.0,
      "logps/rejected": -410.0,
      "loss": 0.0363,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.734375,
      "rewards/margins": 12.9375,
      "rewards/rejected": -12.1875,
      "step": 1561
    },
    {
      "epoch": 1.9165644171779141,
      "grad_norm": 12.232131041485115,
      "learning_rate": 5.473140495867769e-07,
      "logits/chosen": -0.21875,
      "logits/rejected": -0.4296875,
      "logps/chosen": -438.0,
      "logps/rejected": -414.0,
      "loss": 0.03,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 1.34375,
      "rewards/margins": 12.625,
      "rewards/rejected": -11.25,
      "step": 1562
    },
    {
      "epoch": 1.9177914110429448,
      "grad_norm": 14.943326703058906,
      "learning_rate": 5.466942148760331e-07,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.408203125,
      "logps/chosen": -404.0,
      "logps/rejected": -404.0,
      "loss": 0.0469,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7421875,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.75,
      "step": 1563
    },
    {
      "epoch": 1.9190184049079755,
      "grad_norm": 2.974048087039378,
      "learning_rate": 5.460743801652892e-07,
      "logits/chosen": -0.1650390625,
      "logits/rejected": -0.361328125,
      "logps/chosen": -388.0,
      "logps/rejected": -402.0,
      "loss": 0.0101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10302734375,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.25,
      "step": 1564
    },
    {
      "epoch": 1.9202453987730062,
      "grad_norm": 6.834929386098197,
      "learning_rate": 5.454545454545455e-07,
      "logits/chosen": -0.298828125,
      "logits/rejected": -0.490234375,
      "logps/chosen": -322.0,
      "logps/rejected": -342.0,
      "loss": 0.0246,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.474609375,
      "rewards/margins": 12.0625,
      "rewards/rejected": -11.5625,
      "step": 1565
    },
    {
      "epoch": 1.9214723926380368,
      "grad_norm": 19.709537367801847,
      "learning_rate": 5.448347107438017e-07,
      "logits/chosen": -0.1279296875,
      "logits/rejected": -0.267578125,
      "logps/chosen": -406.0,
      "logps/rejected": -422.0,
      "loss": 0.0777,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.890625,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.4375,
      "step": 1566
    },
    {
      "epoch": 1.9226993865030675,
      "grad_norm": 10.759644336799223,
      "learning_rate": 5.442148760330579e-07,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.302734375,
      "logps/chosen": -376.0,
      "logps/rejected": -388.0,
      "loss": 0.0364,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.220703125,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.75,
      "step": 1567
    },
    {
      "epoch": 1.9239263803680982,
      "grad_norm": 20.138956412255148,
      "learning_rate": 5.43595041322314e-07,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.328125,
      "logps/chosen": -392.0,
      "logps/rejected": -404.0,
      "loss": 0.0545,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 12.75,
      "rewards/rejected": -11.4375,
      "step": 1568
    },
    {
      "epoch": 1.9251533742331288,
      "grad_norm": 4.621297903485169,
      "learning_rate": 5.429752066115703e-07,
      "logits/chosen": 0.022705078125,
      "logits/rejected": -0.1220703125,
      "logps/chosen": -390.0,
      "logps/rejected": -370.0,
      "loss": 0.0098,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.8125,
      "step": 1569
    },
    {
      "epoch": 1.9263803680981595,
      "grad_norm": 12.208310607847823,
      "learning_rate": 5.423553719008265e-07,
      "logits/chosen": -0.109375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -384.0,
      "logps/rejected": -388.0,
      "loss": 0.0424,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.12158203125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.3125,
      "step": 1570
    },
    {
      "epoch": 1.9276073619631902,
      "grad_norm": 13.248924763237904,
      "learning_rate": 5.417355371900827e-07,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.294921875,
      "logps/chosen": -426.0,
      "logps/rejected": -430.0,
      "loss": 0.0522,
      "rewards/accuracies": 0.9609375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 12.3125,
      "rewards/rejected": -11.5,
      "step": 1571
    },
    {
      "epoch": 1.9288343558282208,
      "grad_norm": 7.855413156125594,
      "learning_rate": 5.411157024793388e-07,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.392578125,
      "logps/chosen": -414.0,
      "logps/rejected": -406.0,
      "loss": 0.0196,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.1494140625,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.5625,
      "step": 1572
    },
    {
      "epoch": 1.9300613496932515,
      "grad_norm": 11.846684123578562,
      "learning_rate": 5.40495867768595e-07,
      "logits/chosen": -0.17578125,
      "logits/rejected": -0.3046875,
      "logps/chosen": -402.0,
      "logps/rejected": -396.0,
      "loss": 0.0502,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.279296875,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.625,
      "step": 1573
    },
    {
      "epoch": 1.9312883435582822,
      "grad_norm": 15.381676115792011,
      "learning_rate": 5.398760330578513e-07,
      "logits/chosen": -0.02880859375,
      "logits/rejected": -0.162109375,
      "logps/chosen": -366.0,
      "logps/rejected": -404.0,
      "loss": 0.0444,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.078125,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.1875,
      "step": 1574
    },
    {
      "epoch": 1.9325153374233128,
      "grad_norm": 16.45905252471069,
      "learning_rate": 5.392561983471075e-07,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.30078125,
      "logps/chosen": -398.0,
      "logps/rejected": -384.0,
      "loss": 0.0753,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.9375,
      "step": 1575
    },
    {
      "epoch": 1.9337423312883435,
      "grad_norm": 5.91372888725709,
      "learning_rate": 5.386363636363636e-07,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.37890625,
      "logps/chosen": -378.0,
      "logps/rejected": -384.0,
      "loss": 0.0169,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.69921875,
      "rewards/margins": 11.5625,
      "rewards/rejected": -12.3125,
      "step": 1576
    },
    {
      "epoch": 1.9349693251533742,
      "grad_norm": 13.068737835207726,
      "learning_rate": 5.380165289256198e-07,
      "logits/chosen": -0.10693359375,
      "logits/rejected": -0.294921875,
      "logps/chosen": -388.0,
      "logps/rejected": -346.0,
      "loss": 0.0301,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 12.4375,
      "rewards/rejected": -11.625,
      "step": 1577
    },
    {
      "epoch": 1.9361963190184048,
      "grad_norm": 12.574938700274403,
      "learning_rate": 5.373966942148761e-07,
      "logits/chosen": -0.12255859375,
      "logits/rejected": -0.296875,
      "logps/chosen": -382.0,
      "logps/rejected": -420.0,
      "loss": 0.0401,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.0927734375,
      "rewards/margins": 12.125,
      "rewards/rejected": -12.1875,
      "step": 1578
    },
    {
      "epoch": 1.9374233128834355,
      "grad_norm": 6.1456398779932035,
      "learning_rate": 5.367768595041323e-07,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.255859375,
      "logps/chosen": -384.0,
      "logps/rejected": -386.0,
      "loss": 0.0222,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.19140625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -12.0,
      "step": 1579
    },
    {
      "epoch": 1.9386503067484662,
      "grad_norm": 7.144532600637711,
      "learning_rate": 5.361570247933885e-07,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.376953125,
      "logps/chosen": -444.0,
      "logps/rejected": -416.0,
      "loss": 0.0192,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.78515625,
      "rewards/margins": 12.4375,
      "rewards/rejected": -11.6875,
      "step": 1580
    },
    {
      "epoch": 1.9398773006134968,
      "grad_norm": 5.065784309723247,
      "learning_rate": 5.355371900826446e-07,
      "logits/chosen": -0.09765625,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -416.0,
      "logps/rejected": -408.0,
      "loss": 0.0165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.75,
      "step": 1581
    },
    {
      "epoch": 1.9411042944785275,
      "grad_norm": 8.432710268891027,
      "learning_rate": 5.349173553719008e-07,
      "logits/chosen": -0.0242919921875,
      "logits/rejected": -0.142578125,
      "logps/chosen": -332.0,
      "logps/rejected": -352.0,
      "loss": 0.0494,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.31640625,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.0,
      "step": 1582
    },
    {
      "epoch": 1.9423312883435582,
      "grad_norm": 14.414561246083233,
      "learning_rate": 5.342975206611571e-07,
      "logits/chosen": -0.12255859375,
      "logits/rejected": -0.30078125,
      "logps/chosen": -418.0,
      "logps/rejected": -400.0,
      "loss": 0.0637,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.4375,
      "step": 1583
    },
    {
      "epoch": 1.9435582822085888,
      "grad_norm": 13.517917688736201,
      "learning_rate": 5.336776859504132e-07,
      "logits/chosen": -0.08740234375,
      "logits/rejected": -0.271484375,
      "logps/chosen": -436.0,
      "logps/rejected": -394.0,
      "loss": 0.064,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.37890625,
      "rewards/margins": 11.0625,
      "rewards/rejected": -10.6875,
      "step": 1584
    },
    {
      "epoch": 1.9447852760736195,
      "grad_norm": 18.795243167760674,
      "learning_rate": 5.330578512396694e-07,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.326171875,
      "logps/chosen": -418.0,
      "logps/rejected": -434.0,
      "loss": 0.073,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 13.1875,
      "rewards/rejected": -12.375,
      "step": 1585
    },
    {
      "epoch": 1.9460122699386502,
      "grad_norm": 9.671395753436496,
      "learning_rate": 5.324380165289256e-07,
      "logits/chosen": -0.203125,
      "logits/rejected": -0.37890625,
      "logps/chosen": -404.0,
      "logps/rejected": -386.0,
      "loss": 0.0373,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.169921875,
      "rewards/margins": 11.125,
      "rewards/rejected": -10.9375,
      "step": 1586
    },
    {
      "epoch": 1.9472392638036808,
      "grad_norm": 10.924001936653777,
      "learning_rate": 5.318181818181819e-07,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.34765625,
      "logps/chosen": -358.0,
      "logps/rejected": -366.0,
      "loss": 0.0462,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.13671875,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.3125,
      "step": 1587
    },
    {
      "epoch": 1.9484662576687115,
      "grad_norm": 10.287013784424305,
      "learning_rate": 5.31198347107438e-07,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -374.0,
      "logps/rejected": -378.0,
      "loss": 0.0323,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.35546875,
      "rewards/margins": 11.375,
      "rewards/rejected": -11.0,
      "step": 1588
    },
    {
      "epoch": 1.9496932515337422,
      "grad_norm": 11.345438684638955,
      "learning_rate": 5.305785123966941e-07,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.33984375,
      "logps/chosen": -360.0,
      "logps/rejected": -388.0,
      "loss": 0.028,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.26953125,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.1875,
      "step": 1589
    },
    {
      "epoch": 1.9509202453987728,
      "grad_norm": 12.380473368422328,
      "learning_rate": 5.299586776859504e-07,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.232421875,
      "logps/chosen": -398.0,
      "logps/rejected": -420.0,
      "loss": 0.0306,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.4765625,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.875,
      "step": 1590
    },
    {
      "epoch": 1.9521472392638037,
      "grad_norm": 6.3846910410496855,
      "learning_rate": 5.293388429752066e-07,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.328125,
      "logps/chosen": -364.0,
      "logps/rejected": -368.0,
      "loss": 0.0163,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.02490234375,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.875,
      "step": 1591
    },
    {
      "epoch": 1.9533742331288344,
      "grad_norm": 12.161899209714043,
      "learning_rate": 5.287190082644629e-07,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -378.0,
      "logps/rejected": -402.0,
      "loss": 0.0384,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 11.5,
      "rewards/rejected": -12.0625,
      "step": 1592
    },
    {
      "epoch": 1.954601226993865,
      "grad_norm": 16.096985565614556,
      "learning_rate": 5.28099173553719e-07,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.263671875,
      "logps/chosen": -406.0,
      "logps/rejected": -396.0,
      "loss": 0.0467,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.478515625,
      "rewards/margins": 11.75,
      "rewards/rejected": -12.25,
      "step": 1593
    },
    {
      "epoch": 1.9558282208588957,
      "grad_norm": 8.516823030452967,
      "learning_rate": 5.274793388429752e-07,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.34765625,
      "logps/chosen": -332.0,
      "logps/rejected": -344.0,
      "loss": 0.0568,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.1103515625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.4375,
      "step": 1594
    },
    {
      "epoch": 1.9570552147239264,
      "grad_norm": 20.676589588590403,
      "learning_rate": 5.268595041322314e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.21875,
      "logps/chosen": -382.0,
      "logps/rejected": -404.0,
      "loss": 0.0608,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.953125,
      "rewards/margins": 11.8125,
      "rewards/rejected": -12.8125,
      "step": 1595
    },
    {
      "epoch": 1.958282208588957,
      "grad_norm": 11.402306615191995,
      "learning_rate": 5.262396694214877e-07,
      "logits/chosen": -0.2119140625,
      "logits/rejected": -0.25390625,
      "logps/chosen": -362.0,
      "logps/rejected": -370.0,
      "loss": 0.0339,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.5234375,
      "rewards/margins": 11.8125,
      "rewards/rejected": -12.375,
      "step": 1596
    },
    {
      "epoch": 1.9595092024539877,
      "grad_norm": 20.832383714357192,
      "learning_rate": 5.256198347107438e-07,
      "logits/chosen": -0.12353515625,
      "logits/rejected": -0.330078125,
      "logps/chosen": -402.0,
      "logps/rejected": -394.0,
      "loss": 0.0374,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.25390625,
      "rewards/margins": 12.5,
      "rewards/rejected": -12.75,
      "step": 1597
    },
    {
      "epoch": 1.9607361963190184,
      "grad_norm": 17.67799399509982,
      "learning_rate": 5.25e-07,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.30078125,
      "logps/chosen": -396.0,
      "logps/rejected": -432.0,
      "loss": 0.0746,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 11.625,
      "rewards/rejected": -12.6875,
      "step": 1598
    },
    {
      "epoch": 1.961963190184049,
      "grad_norm": 2.4424485737982744,
      "learning_rate": 5.243801652892562e-07,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.36328125,
      "logps/chosen": -402.0,
      "logps/rejected": -388.0,
      "loss": 0.011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.27734375,
      "rewards/margins": 12.0,
      "rewards/rejected": -12.25,
      "step": 1599
    },
    {
      "epoch": 1.9631901840490797,
      "grad_norm": 13.907963965939771,
      "learning_rate": 5.237603305785124e-07,
      "logits/chosen": -0.06884765625,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -416.0,
      "logps/rejected": -410.0,
      "loss": 0.0418,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 11.625,
      "rewards/rejected": -12.125,
      "step": 1600
    },
    {
      "epoch": 1.9644171779141104,
      "grad_norm": 13.814689419274158,
      "learning_rate": 5.231404958677686e-07,
      "logits/chosen": -0.033447265625,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -346.0,
      "logps/rejected": -372.0,
      "loss": 0.0472,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.98046875,
      "rewards/margins": 10.6875,
      "rewards/rejected": -11.6875,
      "step": 1601
    },
    {
      "epoch": 1.965644171779141,
      "grad_norm": 11.545621891190732,
      "learning_rate": 5.225206611570248e-07,
      "logits/chosen": -0.11962890625,
      "logits/rejected": -0.326171875,
      "logps/chosen": -442.0,
      "logps/rejected": -380.0,
      "loss": 0.0297,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2578125,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.9375,
      "step": 1602
    },
    {
      "epoch": 1.9668711656441717,
      "grad_norm": 8.879321275799807,
      "learning_rate": 5.21900826446281e-07,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -472.0,
      "logps/rejected": -436.0,
      "loss": 0.0337,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.373046875,
      "rewards/margins": 13.0,
      "rewards/rejected": -13.3125,
      "step": 1603
    },
    {
      "epoch": 1.9680981595092024,
      "grad_norm": 10.570209103673362,
      "learning_rate": 5.212809917355372e-07,
      "logits/chosen": -0.080078125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -394.0,
      "logps/rejected": -392.0,
      "loss": 0.0386,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.07861328125,
      "rewards/margins": 11.1875,
      "rewards/rejected": -11.3125,
      "step": 1604
    },
    {
      "epoch": 1.969325153374233,
      "grad_norm": 11.659167158794315,
      "learning_rate": 5.206611570247935e-07,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.232421875,
      "logps/chosen": -434.0,
      "logps/rejected": -436.0,
      "loss": 0.0439,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.875,
      "rewards/margins": 11.625,
      "rewards/rejected": -12.5,
      "step": 1605
    },
    {
      "epoch": 1.9705521472392638,
      "grad_norm": 4.504606116838098,
      "learning_rate": 5.200413223140496e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.427734375,
      "logps/chosen": -418.0,
      "logps/rejected": -434.0,
      "loss": 0.0155,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 12.5625,
      "rewards/rejected": -12.3125,
      "step": 1606
    },
    {
      "epoch": 1.9717791411042946,
      "grad_norm": 3.9125260909977104,
      "learning_rate": 5.194214876033058e-07,
      "logits/chosen": -0.16015625,
      "logits/rejected": -0.396484375,
      "logps/chosen": -384.0,
      "logps/rejected": -370.0,
      "loss": 0.0139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2578125,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.4375,
      "step": 1607
    },
    {
      "epoch": 1.9730061349693253,
      "grad_norm": 12.198720753980819,
      "learning_rate": 5.18801652892562e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.369140625,
      "logps/chosen": -392.0,
      "logps/rejected": -414.0,
      "loss": 0.0264,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.2236328125,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.5625,
      "step": 1608
    },
    {
      "epoch": 1.974233128834356,
      "grad_norm": 13.187714303157518,
      "learning_rate": 5.181818181818182e-07,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.3125,
      "logps/chosen": -378.0,
      "logps/rejected": -398.0,
      "loss": 0.0497,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.57421875,
      "rewards/margins": 12.0625,
      "rewards/rejected": -12.625,
      "step": 1609
    },
    {
      "epoch": 1.9754601226993866,
      "grad_norm": 13.077595266689482,
      "learning_rate": 5.175619834710744e-07,
      "logits/chosen": -0.22265625,
      "logits/rejected": -0.3125,
      "logps/chosen": -366.0,
      "logps/rejected": -362.0,
      "loss": 0.0473,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 12.0,
      "rewards/rejected": -12.0625,
      "step": 1610
    },
    {
      "epoch": 1.9766871165644173,
      "grad_norm": 11.034197753449089,
      "learning_rate": 5.169421487603305e-07,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.216796875,
      "logps/chosen": -328.0,
      "logps/rejected": -342.0,
      "loss": 0.0548,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.78125,
      "rewards/margins": 10.5,
      "rewards/rejected": -11.25,
      "step": 1611
    },
    {
      "epoch": 1.977914110429448,
      "grad_norm": 19.25493001940352,
      "learning_rate": 5.163223140495868e-07,
      "logits/chosen": -0.07275390625,
      "logits/rejected": -0.275390625,
      "logps/chosen": -436.0,
      "logps/rejected": -462.0,
      "loss": 0.0788,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.478515625,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.25,
      "step": 1612
    },
    {
      "epoch": 1.9791411042944786,
      "grad_norm": 2.50058852484782,
      "learning_rate": 5.15702479338843e-07,
      "logits/chosen": -0.041259765625,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -442.0,
      "logps/rejected": -420.0,
      "loss": 0.0116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1943359375,
      "rewards/margins": 13.5,
      "rewards/rejected": -13.3125,
      "step": 1613
    },
    {
      "epoch": 1.9803680981595093,
      "grad_norm": 9.641767117339855,
      "learning_rate": 5.150826446280992e-07,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.470703125,
      "logps/chosen": -436.0,
      "logps/rejected": -432.0,
      "loss": 0.0409,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.1953125,
      "rewards/margins": 13.0,
      "rewards/rejected": -12.8125,
      "step": 1614
    },
    {
      "epoch": 1.98159509202454,
      "grad_norm": 13.7846054101889,
      "learning_rate": 5.144628099173554e-07,
      "logits/chosen": -0.283203125,
      "logits/rejected": -0.37890625,
      "logps/chosen": -376.0,
      "logps/rejected": -414.0,
      "loss": 0.0534,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.5625,
      "step": 1615
    },
    {
      "epoch": 1.9828220858895707,
      "grad_norm": 10.800122873462064,
      "learning_rate": 5.138429752066116e-07,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -352.0,
      "logps/rejected": -348.0,
      "loss": 0.0361,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": -0.28515625,
      "rewards/margins": 11.3125,
      "rewards/rejected": -11.625,
      "step": 1616
    },
    {
      "epoch": 1.9840490797546013,
      "grad_norm": 10.77210403419169,
      "learning_rate": 5.132231404958678e-07,
      "logits/chosen": -0.033935546875,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -360.0,
      "logps/rejected": -372.0,
      "loss": 0.0398,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 10.875,
      "rewards/rejected": -10.9375,
      "step": 1617
    },
    {
      "epoch": 1.985276073619632,
      "grad_norm": 9.529792792248653,
      "learning_rate": 5.126033057851239e-07,
      "logits/chosen": -0.2080078125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -370.0,
      "logps/rejected": -378.0,
      "loss": 0.037,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.2373046875,
      "rewards/margins": 11.875,
      "rewards/rejected": -11.6875,
      "step": 1618
    },
    {
      "epoch": 1.9865030674846627,
      "grad_norm": 8.741040335110851,
      "learning_rate": 5.119834710743802e-07,
      "logits/chosen": -0.2314453125,
      "logits/rejected": -0.384765625,
      "logps/chosen": -416.0,
      "logps/rejected": -388.0,
      "loss": 0.0321,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": -0.0927734375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.875,
      "step": 1619
    },
    {
      "epoch": 1.9877300613496933,
      "grad_norm": 7.785518167433976,
      "learning_rate": 5.113636363636363e-07,
      "logits/chosen": -0.134765625,
      "logits/rejected": -0.3203125,
      "logps/chosen": -372.0,
      "logps/rejected": -372.0,
      "loss": 0.0185,
      "rewards/accuracies": 0.9921875,
      "rewards/chosen": 0.33984375,
      "rewards/margins": 11.75,
      "rewards/rejected": -11.375,
      "step": 1620
    },
    {
      "epoch": 1.988957055214724,
      "grad_norm": 5.464211487320141,
      "learning_rate": 5.107438016528926e-07,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -408.0,
      "logps/rejected": -422.0,
      "loss": 0.0187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39453125,
      "rewards/margins": 12.4375,
      "rewards/rejected": -12.0625,
      "step": 1621
    },
    {
      "epoch": 1.9901840490797547,
      "grad_norm": 13.071939990292023,
      "learning_rate": 5.101239669421488e-07,
      "logits/chosen": -0.3828125,
      "logits/rejected": -0.5078125,
      "logps/chosen": -408.0,
      "logps/rejected": -442.0,
      "loss": 0.0443,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.10546875,
      "rewards/margins": 11.9375,
      "rewards/rejected": -12.0625,
      "step": 1622
    },
    {
      "epoch": 1.9914110429447853,
      "grad_norm": 3.434637787021447,
      "learning_rate": 5.09504132231405e-07,
      "logits/chosen": -0.0947265625,
      "logits/rejected": -0.30078125,
      "logps/chosen": -402.0,
      "logps/rejected": -402.0,
      "loss": 0.0095,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.61328125,
      "rewards/margins": 12.5,
      "rewards/rejected": -11.875,
      "step": 1623
    },
    {
      "epoch": 1.992638036809816,
      "grad_norm": 3.315305852103551,
      "learning_rate": 5.088842975206612e-07,
      "logits/chosen": -0.212890625,
      "logits/rejected": -0.474609375,
      "logps/chosen": -376.0,
      "logps/rejected": -394.0,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.0,
      "step": 1624
    },
    {
      "epoch": 1.9938650306748467,
      "grad_norm": 22.7278950349499,
      "learning_rate": 5.082644628099174e-07,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.2578125,
      "logps/chosen": -440.0,
      "logps/rejected": -408.0,
      "loss": 0.0652,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 12.375,
      "rewards/rejected": -11.625,
      "step": 1625
    },
    {
      "epoch": 1.9950920245398773,
      "grad_norm": 7.501231527411867,
      "learning_rate": 5.076446280991736e-07,
      "logits/chosen": -0.17578125,
      "logits/rejected": -0.28125,
      "logps/chosen": -432.0,
      "logps/rejected": -426.0,
      "loss": 0.0195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.251953125,
      "rewards/margins": 12.0,
      "rewards/rejected": -11.75,
      "step": 1626
    },
    {
      "epoch": 1.996319018404908,
      "grad_norm": 8.75791968734159,
      "learning_rate": 5.070247933884297e-07,
      "logits/chosen": -0.24609375,
      "logits/rejected": -0.345703125,
      "logps/chosen": -372.0,
      "logps/rejected": -368.0,
      "loss": 0.0302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4765625,
      "rewards/margins": 12.1875,
      "rewards/rejected": -11.6875,
      "step": 1627
    },
    {
      "epoch": 1.9975460122699387,
      "grad_norm": 12.320799734184225,
      "learning_rate": 5.06404958677686e-07,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.296875,
      "logps/chosen": -368.0,
      "logps/rejected": -406.0,
      "loss": 0.0386,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": 0.65625,
      "rewards/margins": 12.6875,
      "rewards/rejected": -12.0,
      "step": 1628
    },
    {
      "epoch": 1.9987730061349693,
      "grad_norm": 16.231968818300622,
      "learning_rate": 5.057851239669421e-07,
      "logits/chosen": -0.05126953125,
      "logits/rejected": -0.216796875,
      "logps/chosen": -398.0,
      "logps/rejected": -408.0,
      "loss": 0.0634,
      "rewards/accuracies": 0.9765625,
      "rewards/chosen": -0.11181640625,
      "rewards/margins": 11.5,
      "rewards/rejected": -11.625,
      "step": 1629
    },
    {
      "epoch": 2.0,
      "grad_norm": 7.251135077046663,
      "learning_rate": 5.051652892561984e-07,
      "logits/chosen": -0.1787109375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -428.0,
      "logps/rejected": -424.0,
      "loss": 0.0224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1337890625,
      "rewards/margins": 12.75,
      "rewards/rejected": -12.625,
      "step": 1630
    }
  ],
  "logging_steps": 1,
  "max_steps": 2445,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}