{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 815,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.001226993865030675,
      "grad_norm": 151.20719434086217,
      "learning_rate": 6.000000000000001e-08,
      "logits/chosen": -0.32421875,
      "logits/rejected": -0.515625,
      "logps/chosen": -474.0,
      "logps/rejected": -316.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.00245398773006135,
      "grad_norm": 162.171350331327,
      "learning_rate": 1.2000000000000002e-07,
      "logits/chosen": -0.22265625,
      "logits/rejected": -0.439453125,
      "logps/chosen": -436.0,
      "logps/rejected": -292.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2
    },
    {
      "epoch": 0.0036809815950920245,
      "grad_norm": 121.91181472288706,
      "learning_rate": 1.8e-07,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2578125,
      "logps/chosen": -368.0,
      "logps/rejected": -272.0,
      "loss": 0.7012,
      "rewards/accuracies": 0.3359375,
      "rewards/chosen": -0.0006103515625,
      "rewards/margins": -0.00506591796875,
      "rewards/rejected": 0.00445556640625,
      "step": 3
    },
    {
      "epoch": 0.0049079754601227,
      "grad_norm": 138.61232520616198,
      "learning_rate": 2.4000000000000003e-07,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.326171875,
      "logps/chosen": -422.0,
      "logps/rejected": -242.0,
      "loss": 0.7003,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": 0.0057373046875,
      "rewards/margins": -0.00762939453125,
      "rewards/rejected": 0.01336669921875,
      "step": 4
    },
    {
      "epoch": 0.006134969325153374,
      "grad_norm": 318.4843338984337,
      "learning_rate": 3.0000000000000004e-07,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.32421875,
      "logps/chosen": -380.0,
      "logps/rejected": -270.0,
      "loss": 0.7002,
      "rewards/accuracies": 0.296875,
      "rewards/chosen": -0.0037994384765625,
      "rewards/margins": -0.01123046875,
      "rewards/rejected": 0.007415771484375,
      "step": 5
    },
    {
      "epoch": 0.007361963190184049,
      "grad_norm": 356.6573592678301,
      "learning_rate": 3.6e-07,
      "logits/chosen": -0.255859375,
      "logits/rejected": -0.345703125,
      "logps/chosen": -412.0,
      "logps/rejected": -306.0,
      "loss": 0.692,
      "rewards/accuracies": 0.359375,
      "rewards/chosen": 0.0208740234375,
      "rewards/margins": 0.01153564453125,
      "rewards/rejected": 0.00927734375,
      "step": 6
    },
    {
      "epoch": 0.008588957055214725,
      "grad_norm": 228.0843622624859,
      "learning_rate": 4.2000000000000006e-07,
      "logits/chosen": -0.2060546875,
      "logits/rejected": -0.38671875,
      "logps/chosen": -378.0,
      "logps/rejected": -266.0,
      "loss": 0.6962,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": 0.006683349609375,
      "rewards/margins": -0.001617431640625,
      "rewards/rejected": 0.00830078125,
      "step": 7
    },
    {
      "epoch": 0.0098159509202454,
      "grad_norm": 139.9677750400214,
      "learning_rate": 4.800000000000001e-07,
      "logits/chosen": -0.111328125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -432.0,
      "logps/rejected": -292.0,
      "loss": 0.6667,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": 0.054443359375,
      "rewards/margins": 0.06201171875,
      "rewards/rejected": -0.00762939453125,
      "step": 8
    },
    {
      "epoch": 0.011042944785276074,
      "grad_norm": 114.26865689975604,
      "learning_rate": 5.4e-07,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.359375,
      "logps/chosen": -402.0,
      "logps/rejected": -294.0,
      "loss": 0.6492,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 0.0654296875,
      "rewards/margins": 0.10986328125,
      "rewards/rejected": -0.044677734375,
      "step": 9
    },
    {
      "epoch": 0.012269938650306749,
      "grad_norm": 150.56729793626556,
      "learning_rate": 6.000000000000001e-07,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -402.0,
      "logps/rejected": -278.0,
      "loss": 0.6146,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.158203125,
      "rewards/margins": 0.1943359375,
      "rewards/rejected": -0.0361328125,
      "step": 10
    },
    {
      "epoch": 0.013496932515337423,
      "grad_norm": 85.94823037139018,
      "learning_rate": 6.6e-07,
      "logits/chosen": -0.375,
      "logits/rejected": -0.515625,
      "logps/chosen": -420.0,
      "logps/rejected": -298.0,
      "loss": 0.5535,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2451171875,
      "rewards/margins": 0.3984375,
      "rewards/rejected": -0.1533203125,
      "step": 11
    },
    {
      "epoch": 0.014723926380368098,
      "grad_norm": 60.96373680958018,
      "learning_rate": 7.2e-07,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.244140625,
      "logps/chosen": -372.0,
      "logps/rejected": -262.0,
      "loss": 0.548,
      "rewards/accuracies": 0.6953125,
      "rewards/chosen": 0.255859375,
      "rewards/margins": 0.435546875,
      "rewards/rejected": -0.1796875,
      "step": 12
    },
    {
      "epoch": 0.015950920245398775,
      "grad_norm": 92.70871898741228,
      "learning_rate": 7.8e-07,
      "logits/chosen": -0.240234375,
      "logits/rejected": -0.408203125,
      "logps/chosen": -390.0,
      "logps/rejected": -251.0,
      "loss": 0.5255,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.302734375,
      "rewards/margins": 0.56640625,
      "rewards/rejected": -0.26171875,
      "step": 13
    },
    {
      "epoch": 0.01717791411042945,
      "grad_norm": 48.59299098972598,
      "learning_rate": 8.400000000000001e-07,
      "logits/chosen": -0.2138671875,
      "logits/rejected": -0.3984375,
      "logps/chosen": -466.0,
      "logps/rejected": -314.0,
      "loss": 0.4612,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.765625,
      "rewards/margins": 1.078125,
      "rewards/rejected": -0.31640625,
      "step": 14
    },
    {
      "epoch": 0.018404907975460124,
      "grad_norm": 59.699183777930486,
      "learning_rate": 9e-07,
      "logits/chosen": -0.10400390625,
      "logits/rejected": -0.177734375,
      "logps/chosen": -414.0,
      "logps/rejected": -292.0,
      "loss": 0.5014,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 0.96875,
      "rewards/rejected": -0.31640625,
      "step": 15
    },
    {
      "epoch": 0.0196319018404908,
      "grad_norm": 52.10075632980986,
      "learning_rate": 9.600000000000001e-07,
      "logits/chosen": -0.25,
      "logits/rejected": -0.42578125,
      "logps/chosen": -400.0,
      "logps/rejected": -310.0,
      "loss": 0.4424,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": 0.84375,
      "rewards/margins": 1.390625,
      "rewards/rejected": -0.54296875,
      "step": 16
    },
    {
      "epoch": 0.020858895705521473,
      "grad_norm": 42.16097538458702,
      "learning_rate": 1.0200000000000002e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -426.0,
      "logps/rejected": -260.0,
      "loss": 0.4308,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 1.578125,
      "rewards/rejected": -0.58984375,
      "step": 17
    },
    {
      "epoch": 0.022085889570552148,
      "grad_norm": 46.56295761289983,
      "learning_rate": 1.08e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.330078125,
      "logps/chosen": -398.0,
      "logps/rejected": -282.0,
      "loss": 0.432,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.875,
      "rewards/margins": 1.6875,
      "rewards/rejected": -0.8125,
      "step": 18
    },
    {
      "epoch": 0.023312883435582823,
      "grad_norm": 49.57615257325465,
      "learning_rate": 1.14e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.3671875,
      "logps/chosen": -382.0,
      "logps/rejected": -296.0,
      "loss": 0.4802,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 1.8125,
      "rewards/rejected": -1.0546875,
      "step": 19
    },
    {
      "epoch": 0.024539877300613498,
      "grad_norm": 42.63506109619643,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.439453125,
      "logps/chosen": -456.0,
      "logps/rejected": -324.0,
      "loss": 0.3801,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.4375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -1.703125,
      "step": 20
    },
    {
      "epoch": 0.025766871165644172,
      "grad_norm": 34.456600308133275,
      "learning_rate": 1.26e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -404.0,
      "logps/rejected": -272.0,
      "loss": 0.3497,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 2.640625,
      "rewards/rejected": -1.53125,
      "step": 21
    },
    {
      "epoch": 0.026993865030674847,
      "grad_norm": 44.983878720566445,
      "learning_rate": 1.32e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.271484375,
      "logps/chosen": -374.0,
      "logps/rejected": -306.0,
      "loss": 0.4156,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -1.7890625,
      "step": 22
    },
    {
      "epoch": 0.02822085889570552,
      "grad_norm": 56.72818428668396,
      "learning_rate": 1.3800000000000001e-06,
      "logits/chosen": -0.251953125,
      "logits/rejected": -0.314453125,
      "logps/chosen": -370.0,
      "logps/rejected": -318.0,
      "loss": 0.4566,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.46875,
      "rewards/margins": 3.1875,
      "rewards/rejected": -1.7265625,
      "step": 23
    },
    {
      "epoch": 0.029447852760736196,
      "grad_norm": 55.662509241103734,
      "learning_rate": 1.44e-06,
      "logits/chosen": -0.1884765625,
      "logits/rejected": -0.375,
      "logps/chosen": -376.0,
      "logps/rejected": -288.0,
      "loss": 0.4763,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 2.265625,
      "rewards/rejected": -1.625,
      "step": 24
    },
    {
      "epoch": 0.03067484662576687,
      "grad_norm": 35.38065334196675,
      "learning_rate": 1.5e-06,
      "logits/chosen": -0.2578125,
      "logits/rejected": -0.4453125,
      "logps/chosen": -430.0,
      "logps/rejected": -312.0,
      "loss": 0.3378,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -2.21875,
      "step": 25
    },
    {
      "epoch": 0.03190184049079755,
      "grad_norm": 42.887450752414956,
      "learning_rate": 1.4993801652892562e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -380.0,
      "logps/rejected": -288.0,
      "loss": 0.4058,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.640625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -1.9609375,
      "step": 26
    },
    {
      "epoch": 0.033128834355828224,
      "grad_norm": 37.07120513953294,
      "learning_rate": 1.4987603305785125e-06,
      "logits/chosen": -0.095703125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -354.0,
      "logps/rejected": -292.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -2.34375,
      "step": 27
    },
    {
      "epoch": 0.0343558282208589,
      "grad_norm": 35.87136073029194,
      "learning_rate": 1.4981404958677687e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -374.0,
      "logps/rejected": -274.0,
      "loss": 0.3615,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -1.796875,
      "step": 28
    },
    {
      "epoch": 0.03558282208588957,
      "grad_norm": 38.17662218776474,
      "learning_rate": 1.4975206611570248e-06,
      "logits/chosen": -0.263671875,
      "logits/rejected": -0.4140625,
      "logps/chosen": -414.0,
      "logps/rejected": -334.0,
      "loss": 0.3367,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.765625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -2.296875,
      "step": 29
    },
    {
      "epoch": 0.03680981595092025,
      "grad_norm": 39.994932388621756,
      "learning_rate": 1.496900826446281e-06,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.248046875,
      "logps/chosen": -332.0,
      "logps/rejected": -278.0,
      "loss": 0.3648,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.203125,
      "rewards/margins": 3.375,
      "rewards/rejected": -2.171875,
      "step": 30
    },
    {
      "epoch": 0.03803680981595092,
      "grad_norm": 42.39396872212231,
      "learning_rate": 1.496280991735537e-06,
      "logits/chosen": -0.10986328125,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -378.0,
      "logps/rejected": -286.0,
      "loss": 0.364,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.9765625,
      "rewards/margins": 4.25,
      "rewards/rejected": -2.265625,
      "step": 31
    },
    {
      "epoch": 0.0392638036809816,
      "grad_norm": 43.84495796047598,
      "learning_rate": 1.4956611570247935e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.333984375,
      "logps/chosen": -388.0,
      "logps/rejected": -298.0,
      "loss": 0.3861,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.71875,
      "rewards/margins": 4.0,
      "rewards/rejected": -2.28125,
      "step": 32
    },
    {
      "epoch": 0.04049079754601227,
      "grad_norm": 38.440307864705964,
      "learning_rate": 1.4950413223140496e-06,
      "logits/chosen": -0.04443359375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -328.0,
      "logps/rejected": -248.0,
      "loss": 0.3456,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.078125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -2.484375,
      "step": 33
    },
    {
      "epoch": 0.04171779141104295,
      "grad_norm": 42.669938783190766,
      "learning_rate": 1.4944214876033057e-06,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.296875,
      "logps/chosen": -384.0,
      "logps/rejected": -316.0,
      "loss": 0.3991,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -2.8125,
      "step": 34
    },
    {
      "epoch": 0.04294478527607362,
      "grad_norm": 39.82238013261856,
      "learning_rate": 1.493801652892562e-06,
      "logits/chosen": -0.0810546875,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -356.0,
      "logps/rejected": -278.0,
      "loss": 0.3621,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 4.125,
      "rewards/rejected": -2.9375,
      "step": 35
    },
    {
      "epoch": 0.044171779141104296,
      "grad_norm": 34.79756258065961,
      "learning_rate": 1.4931818181818182e-06,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.20703125,
      "logps/chosen": -398.0,
      "logps/rejected": -310.0,
      "loss": 0.3113,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.6640625,
      "rewards/margins": 4.78125,
      "rewards/rejected": -3.125,
      "step": 36
    },
    {
      "epoch": 0.04539877300613497,
      "grad_norm": 40.21812818356396,
      "learning_rate": 1.4925619834710746e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -384.0,
      "logps/rejected": -322.0,
      "loss": 0.3512,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 4.625,
      "rewards/rejected": -2.328125,
      "step": 37
    },
    {
      "epoch": 0.046625766871165646,
      "grad_norm": 38.815201724599156,
      "learning_rate": 1.4919421487603307e-06,
      "logits/chosen": -0.032470703125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -390.0,
      "logps/rejected": -312.0,
      "loss": 0.36,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.546875,
      "rewards/margins": 5.09375,
      "rewards/rejected": -2.546875,
      "step": 38
    },
    {
      "epoch": 0.04785276073619632,
      "grad_norm": 36.01668225984001,
      "learning_rate": 1.4913223140495867e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.2890625,
      "logps/chosen": -418.0,
      "logps/rejected": -318.0,
      "loss": 0.3347,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.296875,
      "rewards/margins": 5.21875,
      "rewards/rejected": -2.921875,
      "step": 39
    },
    {
      "epoch": 0.049079754601226995,
      "grad_norm": 30.447104323671855,
      "learning_rate": 1.490702479338843e-06,
      "logits/chosen": -0.0712890625,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -370.0,
      "logps/rejected": -296.0,
      "loss": 0.2729,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.09375,
      "rewards/margins": 4.90625,
      "rewards/rejected": -2.796875,
      "step": 40
    },
    {
      "epoch": 0.05030674846625767,
      "grad_norm": 36.96390592719149,
      "learning_rate": 1.4900826446280991e-06,
      "logits/chosen": -0.0751953125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -374.0,
      "logps/rejected": -286.0,
      "loss": 0.3822,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -2.671875,
      "step": 41
    },
    {
      "epoch": 0.051533742331288344,
      "grad_norm": 33.41932525699284,
      "learning_rate": 1.4894628099173555e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -360.0,
      "logps/rejected": -294.0,
      "loss": 0.3014,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.75,
      "rewards/margins": 4.3125,
      "rewards/rejected": -2.5625,
      "step": 42
    },
    {
      "epoch": 0.05276073619631902,
      "grad_norm": 38.41247549359973,
      "learning_rate": 1.4888429752066116e-06,
      "logits/chosen": -0.09033203125,
      "logits/rejected": -0.248046875,
      "logps/chosen": -412.0,
      "logps/rejected": -336.0,
      "loss": 0.2954,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.09375,
      "rewards/margins": 5.09375,
      "rewards/rejected": -3.0,
      "step": 43
    },
    {
      "epoch": 0.053987730061349694,
      "grad_norm": 43.07116536752478,
      "learning_rate": 1.4882231404958678e-06,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -384.0,
      "logps/rejected": -304.0,
      "loss": 0.3938,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.578125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -2.96875,
      "step": 44
    },
    {
      "epoch": 0.05521472392638037,
      "grad_norm": 39.15973652461684,
      "learning_rate": 1.4876033057851241e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.294921875,
      "logps/chosen": -350.0,
      "logps/rejected": -300.0,
      "loss": 0.3339,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.828125,
      "rewards/margins": 4.9375,
      "rewards/rejected": -3.109375,
      "step": 45
    },
    {
      "epoch": 0.05644171779141104,
      "grad_norm": 37.261369579802675,
      "learning_rate": 1.4869834710743803e-06,
      "logits/chosen": -0.09716796875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -416.0,
      "logps/rejected": -302.0,
      "loss": 0.2861,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.46875,
      "rewards/margins": 5.375,
      "rewards/rejected": -2.890625,
      "step": 46
    },
    {
      "epoch": 0.05766871165644172,
      "grad_norm": 54.78531831058221,
      "learning_rate": 1.4863636363636364e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -390.0,
      "logps/rejected": -372.0,
      "loss": 0.5073,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": 1.5234375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -2.6875,
      "step": 47
    },
    {
      "epoch": 0.05889570552147239,
      "grad_norm": 43.69804279742604,
      "learning_rate": 1.4857438016528926e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.26171875,
      "logps/chosen": -352.0,
      "logps/rejected": -288.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -2.734375,
      "step": 48
    },
    {
      "epoch": 0.06012269938650307,
      "grad_norm": 34.89468290879934,
      "learning_rate": 1.4851239669421487e-06,
      "logits/chosen": -0.146484375,
      "logits/rejected": -0.328125,
      "logps/chosen": -400.0,
      "logps/rejected": -300.0,
      "loss": 0.2721,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.25,
      "rewards/margins": 5.46875,
      "rewards/rejected": -3.21875,
      "step": 49
    },
    {
      "epoch": 0.06134969325153374,
      "grad_norm": 54.15414768657872,
      "learning_rate": 1.484504132231405e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.296875,
      "logps/chosen": -404.0,
      "logps/rejected": -324.0,
      "loss": 0.4387,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.515625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -3.171875,
      "step": 50
    },
    {
      "epoch": 0.06257668711656442,
      "grad_norm": 34.25072100129564,
      "learning_rate": 1.4838842975206612e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -440.0,
      "logps/rejected": -364.0,
      "loss": 0.2872,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.984375,
      "rewards/margins": 5.09375,
      "rewards/rejected": -3.125,
      "step": 51
    },
    {
      "epoch": 0.0638036809815951,
      "grad_norm": 38.20166040651749,
      "learning_rate": 1.4832644628099173e-06,
      "logits/chosen": -0.10693359375,
      "logits/rejected": -0.265625,
      "logps/chosen": -396.0,
      "logps/rejected": -338.0,
      "loss": 0.3424,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.65625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.171875,
      "step": 52
    },
    {
      "epoch": 0.06503067484662577,
      "grad_norm": 37.089420431960214,
      "learning_rate": 1.4826446280991737e-06,
      "logits/chosen": -0.055908203125,
      "logits/rejected": -0.189453125,
      "logps/chosen": -398.0,
      "logps/rejected": -326.0,
      "loss": 0.3026,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.953125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -2.75,
      "step": 53
    },
    {
      "epoch": 0.06625766871165645,
      "grad_norm": 35.67709547242796,
      "learning_rate": 1.4820247933884298e-06,
      "logits/chosen": 0.045654296875,
      "logits/rejected": -0.1162109375,
      "logps/chosen": -336.0,
      "logps/rejected": -266.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 4.625,
      "rewards/rejected": -3.296875,
      "step": 54
    },
    {
      "epoch": 0.06748466257668712,
      "grad_norm": 40.58362122970572,
      "learning_rate": 1.481404958677686e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -376.0,
      "logps/rejected": -302.0,
      "loss": 0.3515,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.8046875,
      "rewards/margins": 5.03125,
      "rewards/rejected": -3.234375,
      "step": 55
    },
    {
      "epoch": 0.0687116564417178,
      "grad_norm": 43.576103099929334,
      "learning_rate": 1.4807851239669423e-06,
      "logits/chosen": -0.034912109375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -384.0,
      "logps/rejected": -334.0,
      "loss": 0.4139,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.015625,
      "rewards/margins": 4.8125,
      "rewards/rejected": -2.796875,
      "step": 56
    },
    {
      "epoch": 0.06993865030674846,
      "grad_norm": 51.562196128944976,
      "learning_rate": 1.4801652892561983e-06,
      "logits/chosen": -0.042724609375,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -352.0,
      "logps/rejected": -326.0,
      "loss": 0.4667,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.6015625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -2.984375,
      "step": 57
    },
    {
      "epoch": 0.07116564417177915,
      "grad_norm": 49.38086728368548,
      "learning_rate": 1.4795454545454546e-06,
      "logits/chosen": -0.046630859375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.3869,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.640625,
      "rewards/margins": 4.875,
      "rewards/rejected": -3.234375,
      "step": 58
    },
    {
      "epoch": 0.07239263803680981,
      "grad_norm": 43.27222220894176,
      "learning_rate": 1.4789256198347107e-06,
      "logits/chosen": 0.0240478515625,
      "logits/rejected": -0.051025390625,
      "logps/chosen": -378.0,
      "logps/rejected": -302.0,
      "loss": 0.3983,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 1.3125,
      "rewards/margins": 4.25,
      "rewards/rejected": -2.953125,
      "step": 59
    },
    {
      "epoch": 0.0736196319018405,
      "grad_norm": 39.673404465674906,
      "learning_rate": 1.4783057851239669e-06,
      "logits/chosen": -0.049072265625,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -386.0,
      "logps/rejected": -294.0,
      "loss": 0.3927,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -3.1875,
      "step": 60
    },
    {
      "epoch": 0.07484662576687116,
      "grad_norm": 40.98812217259802,
      "learning_rate": 1.4776859504132232e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.173828125,
      "logps/chosen": -418.0,
      "logps/rejected": -328.0,
      "loss": 0.3496,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.890625,
      "rewards/margins": 5.65625,
      "rewards/rejected": -3.765625,
      "step": 61
    },
    {
      "epoch": 0.07607361963190185,
      "grad_norm": 40.09146702222396,
      "learning_rate": 1.4770661157024794e-06,
      "logits/chosen": -0.051513671875,
      "logits/rejected": -0.2275390625,
      "logps/chosen": -426.0,
      "logps/rejected": -316.0,
      "loss": 0.3416,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -3.640625,
      "step": 62
    },
    {
      "epoch": 0.07730061349693251,
      "grad_norm": 46.30805404508898,
      "learning_rate": 1.4764462809917355e-06,
      "logits/chosen": 0.031982421875,
      "logits/rejected": -0.11767578125,
      "logps/chosen": -418.0,
      "logps/rejected": -314.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.515625,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.328125,
      "step": 63
    },
    {
      "epoch": 0.0785276073619632,
      "grad_norm": 47.96974437982722,
      "learning_rate": 1.4758264462809919e-06,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.27734375,
      "logps/chosen": -402.0,
      "logps/rejected": -336.0,
      "loss": 0.4046,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.046875,
      "rewards/margins": 5.46875,
      "rewards/rejected": -3.4375,
      "step": 64
    },
    {
      "epoch": 0.07975460122699386,
      "grad_norm": 35.57381547040094,
      "learning_rate": 1.475206611570248e-06,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.25390625,
      "logps/chosen": -390.0,
      "logps/rejected": -310.0,
      "loss": 0.2899,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.046875,
      "rewards/margins": 5.53125,
      "rewards/rejected": -3.484375,
      "step": 65
    },
    {
      "epoch": 0.08098159509202454,
      "grad_norm": 33.414611619044024,
      "learning_rate": 1.4745867768595042e-06,
      "logits/chosen": -0.162109375,
      "logits/rejected": -0.287109375,
      "logps/chosen": -314.0,
      "logps/rejected": -270.0,
      "loss": 0.2704,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.78125,
      "step": 66
    },
    {
      "epoch": 0.08220858895705521,
      "grad_norm": 42.668537708095855,
      "learning_rate": 1.4739669421487603e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.1767578125,
      "logps/chosen": -332.0,
      "logps/rejected": -302.0,
      "loss": 0.3919,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.2578125,
      "rewards/margins": 4.84375,
      "rewards/rejected": -3.578125,
      "step": 67
    },
    {
      "epoch": 0.0834355828220859,
      "grad_norm": 40.70660622767979,
      "learning_rate": 1.4733471074380164e-06,
      "logits/chosen": -0.1455078125,
      "logits/rejected": -0.193359375,
      "logps/chosen": -358.0,
      "logps/rejected": -338.0,
      "loss": 0.3949,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.625,
      "rewards/margins": 4.96875,
      "rewards/rejected": -3.359375,
      "step": 68
    },
    {
      "epoch": 0.08466257668711656,
      "grad_norm": 42.29365611407333,
      "learning_rate": 1.4727272727272728e-06,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.36328125,
      "logps/chosen": -406.0,
      "logps/rejected": -290.0,
      "loss": 0.3163,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.484375,
      "rewards/margins": 6.0,
      "rewards/rejected": -3.515625,
      "step": 69
    },
    {
      "epoch": 0.08588957055214724,
      "grad_norm": 45.16433653526153,
      "learning_rate": 1.472107438016529e-06,
      "logits/chosen": -0.08447265625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -406.0,
      "logps/rejected": -346.0,
      "loss": 0.3817,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.515625,
      "rewards/margins": 5.6875,
      "rewards/rejected": -3.15625,
      "step": 70
    },
    {
      "epoch": 0.08711656441717791,
      "grad_norm": 37.819952615726415,
      "learning_rate": 1.4714876033057853e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.1005859375,
      "logps/chosen": -344.0,
      "logps/rejected": -286.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.46875,
      "rewards/margins": 4.625,
      "rewards/rejected": -3.15625,
      "step": 71
    },
    {
      "epoch": 0.08834355828220859,
      "grad_norm": 41.96140458025261,
      "learning_rate": 1.4708677685950414e-06,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.33203125,
      "logps/chosen": -428.0,
      "logps/rejected": -354.0,
      "loss": 0.3187,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.125,
      "rewards/margins": 5.1875,
      "rewards/rejected": -3.0625,
      "step": 72
    },
    {
      "epoch": 0.08957055214723926,
      "grad_norm": 45.003627256411235,
      "learning_rate": 1.4702479338842976e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.220703125,
      "logps/chosen": -346.0,
      "logps/rejected": -308.0,
      "loss": 0.4099,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 5.25,
      "rewards/rejected": -3.625,
      "step": 73
    },
    {
      "epoch": 0.09079754601226994,
      "grad_norm": 37.795821367857755,
      "learning_rate": 1.4696280991735537e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.212890625,
      "logps/chosen": -352.0,
      "logps/rejected": -338.0,
      "loss": 0.2989,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 5.0625,
      "rewards/rejected": -3.625,
      "step": 74
    },
    {
      "epoch": 0.09202453987730061,
      "grad_norm": 38.6325533255864,
      "learning_rate": 1.4690082644628099e-06,
      "logits/chosen": -0.1083984375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -376.0,
      "logps/rejected": -326.0,
      "loss": 0.3138,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.265625,
      "rewards/margins": 5.9375,
      "rewards/rejected": -3.671875,
      "step": 75
    },
    {
      "epoch": 0.09325153374233129,
      "grad_norm": 41.500214372239896,
      "learning_rate": 1.4683884297520662e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -394.0,
      "logps/rejected": -296.0,
      "loss": 0.3196,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.28125,
      "rewards/margins": 5.625,
      "rewards/rejected": -3.359375,
      "step": 76
    },
    {
      "epoch": 0.09447852760736196,
      "grad_norm": 37.661301568651126,
      "learning_rate": 1.4677685950413223e-06,
      "logits/chosen": -0.1337890625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -350.0,
      "logps/rejected": -308.0,
      "loss": 0.3298,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.0625,
      "step": 77
    },
    {
      "epoch": 0.09570552147239264,
      "grad_norm": 37.401749661903814,
      "learning_rate": 1.4671487603305785e-06,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -414.0,
      "logps/rejected": -324.0,
      "loss": 0.3295,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.015625,
      "rewards/margins": 5.90625,
      "rewards/rejected": -3.90625,
      "step": 78
    },
    {
      "epoch": 0.09693251533742331,
      "grad_norm": 41.34926796893681,
      "learning_rate": 1.4665289256198348e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2109375,
      "logps/chosen": -354.0,
      "logps/rejected": -300.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.25,
      "rewards/margins": 5.53125,
      "rewards/rejected": -4.28125,
      "step": 79
    },
    {
      "epoch": 0.09815950920245399,
      "grad_norm": 40.52807889159641,
      "learning_rate": 1.465909090909091e-06,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -404.0,
      "logps/rejected": -340.0,
      "loss": 0.3387,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.484375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -4.75,
      "step": 80
    },
    {
      "epoch": 0.09938650306748466,
      "grad_norm": 35.605812619698284,
      "learning_rate": 1.4652892561983471e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": -0.365234375,
      "logps/chosen": -436.0,
      "logps/rejected": -356.0,
      "loss": 0.3191,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.8828125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.78125,
      "step": 81
    },
    {
      "epoch": 0.10061349693251534,
      "grad_norm": 47.4617133344908,
      "learning_rate": 1.4646694214876035e-06,
      "logits/chosen": -0.1171875,
      "logits/rejected": -0.228515625,
      "logps/chosen": -430.0,
      "logps/rejected": -348.0,
      "loss": 0.3954,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.84375,
      "step": 82
    },
    {
      "epoch": 0.10184049079754601,
      "grad_norm": 42.72902928212931,
      "learning_rate": 1.4640495867768594e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.255859375,
      "logps/chosen": -352.0,
      "logps/rejected": -320.0,
      "loss": 0.3735,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.71875,
      "step": 83
    },
    {
      "epoch": 0.10306748466257669,
      "grad_norm": 59.351571106495314,
      "learning_rate": 1.4634297520661158e-06,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.310546875,
      "logps/chosen": -448.0,
      "logps/rejected": -360.0,
      "loss": 0.2887,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.0625,
      "step": 84
    },
    {
      "epoch": 0.10429447852760736,
      "grad_norm": 40.67504361329859,
      "learning_rate": 1.462809917355372e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -376.0,
      "logps/rejected": -338.0,
      "loss": 0.2933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.96875,
      "step": 85
    },
    {
      "epoch": 0.10552147239263804,
      "grad_norm": 44.45491286199163,
      "learning_rate": 1.462190082644628e-06,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.224609375,
      "logps/chosen": -424.0,
      "logps/rejected": -314.0,
      "loss": 0.4003,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.625,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.25,
      "step": 86
    },
    {
      "epoch": 0.1067484662576687,
      "grad_norm": 37.230987939956755,
      "learning_rate": 1.4615702479338844e-06,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.3046875,
      "logps/chosen": -354.0,
      "logps/rejected": -274.0,
      "loss": 0.3433,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0,
      "step": 87
    },
    {
      "epoch": 0.10797546012269939,
      "grad_norm": 35.435756690174514,
      "learning_rate": 1.4609504132231405e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -362.0,
      "logps/rejected": -286.0,
      "loss": 0.2796,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.5,
      "step": 88
    },
    {
      "epoch": 0.10920245398773006,
      "grad_norm": 39.218068339324205,
      "learning_rate": 1.4603305785123967e-06,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.2734375,
      "logps/chosen": -392.0,
      "logps/rejected": -340.0,
      "loss": 0.3661,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.609375,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.28125,
      "step": 89
    },
    {
      "epoch": 0.11042944785276074,
      "grad_norm": 35.86351693081174,
      "learning_rate": 1.459710743801653e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.375,
      "logps/chosen": -378.0,
      "logps/rejected": -334.0,
      "loss": 0.3305,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.0625,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.625,
      "step": 90
    },
    {
      "epoch": 0.1116564417177914,
      "grad_norm": 39.260039435042906,
      "learning_rate": 1.4590909090909092e-06,
      "logits/chosen": -0.028076171875,
      "logits/rejected": -0.28125,
      "logps/chosen": -392.0,
      "logps/rejected": -308.0,
      "loss": 0.3104,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.5234375,
      "rewards/margins": 5.9375,
      "rewards/rejected": -4.40625,
      "step": 91
    },
    {
      "epoch": 0.11288343558282209,
      "grad_norm": 35.99771043781269,
      "learning_rate": 1.4584710743801653e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.236328125,
      "logps/chosen": -390.0,
      "logps/rejected": -316.0,
      "loss": 0.304,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.484375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.8125,
      "step": 92
    },
    {
      "epoch": 0.11411042944785275,
      "grad_norm": 38.41867053252873,
      "learning_rate": 1.4578512396694215e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.36328125,
      "logps/chosen": -400.0,
      "logps/rejected": -310.0,
      "loss": 0.3311,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.328125,
      "rewards/margins": 6.625,
      "rewards/rejected": -4.3125,
      "step": 93
    },
    {
      "epoch": 0.11533742331288344,
      "grad_norm": 37.099757310606805,
      "learning_rate": 1.4572314049586776e-06,
      "logits/chosen": -0.171875,
      "logits/rejected": -0.390625,
      "logps/chosen": -426.0,
      "logps/rejected": -354.0,
      "loss": 0.2709,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.765625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -4.3125,
      "step": 94
    },
    {
      "epoch": 0.1165644171779141,
      "grad_norm": 36.042450420967036,
      "learning_rate": 1.456611570247934e-06,
      "logits/chosen": -0.2490234375,
      "logits/rejected": -0.392578125,
      "logps/chosen": -378.0,
      "logps/rejected": -320.0,
      "loss": 0.3138,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.0625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -4.625,
      "step": 95
    },
    {
      "epoch": 0.11779141104294479,
      "grad_norm": 37.51605196372948,
      "learning_rate": 1.45599173553719e-06,
      "logits/chosen": -0.2392578125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -366.0,
      "logps/rejected": -320.0,
      "loss": 0.2944,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.625,
      "rewards/margins": 7.125,
      "rewards/rejected": -4.5,
      "step": 96
    },
    {
      "epoch": 0.11901840490797547,
      "grad_norm": 47.87658843319023,
      "learning_rate": 1.4553719008264462e-06,
      "logits/chosen": -0.234375,
      "logits/rejected": -0.373046875,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4367,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.25,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.859375,
      "step": 97
    },
    {
      "epoch": 0.12024539877300613,
      "grad_norm": 37.61598969810632,
      "learning_rate": 1.4547520661157026e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.236328125,
      "logps/chosen": -354.0,
      "logps/rejected": -296.0,
      "loss": 0.3071,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.984375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -4.09375,
      "step": 98
    },
    {
      "epoch": 0.12147239263803682,
      "grad_norm": 35.33546250741774,
      "learning_rate": 1.4541322314049587e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -338.0,
      "logps/rejected": -258.0,
      "loss": 0.2669,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.359375,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.15625,
      "step": 99
    },
    {
      "epoch": 0.12269938650306748,
      "grad_norm": 32.14334216613253,
      "learning_rate": 1.453512396694215e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.404296875,
      "logps/chosen": -394.0,
      "logps/rejected": -332.0,
      "loss": 0.2858,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.8359375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.625,
      "step": 100
    },
    {
      "epoch": 0.12392638036809817,
      "grad_norm": 45.60783475518701,
      "learning_rate": 1.452892561983471e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.400390625,
      "logps/chosen": -412.0,
      "logps/rejected": -352.0,
      "loss": 0.4076,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.375,
      "step": 101
    },
    {
      "epoch": 0.12515337423312883,
      "grad_norm": 39.740396682339004,
      "learning_rate": 1.4522727272727272e-06,
      "logits/chosen": -0.1865234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -392.0,
      "logps/rejected": -330.0,
      "loss": 0.3534,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 6.0,
      "rewards/rejected": -3.9375,
      "step": 102
    },
    {
      "epoch": 0.1263803680981595,
      "grad_norm": 35.16122653572647,
      "learning_rate": 1.4516528925619835e-06,
      "logits/chosen": -0.17578125,
      "logits/rejected": -0.326171875,
      "logps/chosen": -350.0,
      "logps/rejected": -310.0,
      "loss": 0.3153,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.4765625,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.28125,
      "step": 103
    },
    {
      "epoch": 0.1276073619631902,
      "grad_norm": 46.63941850097279,
      "learning_rate": 1.4510330578512396e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.2255859375,
      "logps/chosen": -414.0,
      "logps/rejected": -348.0,
      "loss": 0.381,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.6484375,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.125,
      "step": 104
    },
    {
      "epoch": 0.12883435582822086,
      "grad_norm": 35.04246201772006,
      "learning_rate": 1.450413223140496e-06,
      "logits/chosen": -0.181640625,
      "logits/rejected": -0.3828125,
      "logps/chosen": -374.0,
      "logps/rejected": -322.0,
      "loss": 0.2864,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.03125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -4.78125,
      "step": 105
    },
    {
      "epoch": 0.13006134969325153,
      "grad_norm": 39.36563074587024,
      "learning_rate": 1.4497933884297521e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -392.0,
      "logps/rejected": -322.0,
      "loss": 0.3462,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.7890625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.25,
      "step": 106
    },
    {
      "epoch": 0.1312883435582822,
      "grad_norm": 35.07229699022507,
      "learning_rate": 1.4491735537190083e-06,
      "logits/chosen": -0.1865234375,
      "logits/rejected": -0.26953125,
      "logps/chosen": -352.0,
      "logps/rejected": -312.0,
      "loss": 0.2577,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.3515625,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.5,
      "step": 107
    },
    {
      "epoch": 0.1325153374233129,
      "grad_norm": 28.019513861884207,
      "learning_rate": 1.4485537190082646e-06,
      "logits/chosen": -0.19921875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.2438,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -5.25,
      "step": 108
    },
    {
      "epoch": 0.13374233128834356,
      "grad_norm": 35.40605672013622,
      "learning_rate": 1.4479338842975208e-06,
      "logits/chosen": -0.18359375,
      "logits/rejected": -0.359375,
      "logps/chosen": -412.0,
      "logps/rejected": -326.0,
      "loss": 0.3584,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.5625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -4.625,
      "step": 109
    },
    {
      "epoch": 0.13496932515337423,
      "grad_norm": 38.14439771999889,
      "learning_rate": 1.447314049586777e-06,
      "logits/chosen": -0.1728515625,
      "logits/rejected": -0.310546875,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.3481,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.53125,
      "step": 110
    },
    {
      "epoch": 0.1361963190184049,
      "grad_norm": 36.779042619503116,
      "learning_rate": 1.446694214876033e-06,
      "logits/chosen": -0.09716796875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -334.0,
      "logps/rejected": -282.0,
      "loss": 0.3363,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3671875,
      "rewards/margins": 5.4375,
      "rewards/rejected": -4.0625,
      "step": 111
    },
    {
      "epoch": 0.1374233128834356,
      "grad_norm": 36.86435500620325,
      "learning_rate": 1.4460743801652892e-06,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.29296875,
      "logps/chosen": -342.0,
      "logps/rejected": -292.0,
      "loss": 0.343,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.4375,
      "step": 112
    },
    {
      "epoch": 0.13865030674846626,
      "grad_norm": 38.139474814179394,
      "learning_rate": 1.4454545454545456e-06,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.169921875,
      "logps/chosen": -374.0,
      "logps/rejected": -312.0,
      "loss": 0.3952,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.859375,
      "rewards/margins": 5.75,
      "rewards/rejected": -3.90625,
      "step": 113
    },
    {
      "epoch": 0.13987730061349693,
      "grad_norm": 38.81539599706057,
      "learning_rate": 1.4448347107438017e-06,
      "logits/chosen": -0.1220703125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -372.0,
      "logps/rejected": -316.0,
      "loss": 0.3854,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 5.375,
      "rewards/rejected": -3.875,
      "step": 114
    },
    {
      "epoch": 0.1411042944785276,
      "grad_norm": 45.65683573404254,
      "learning_rate": 1.4442148760330578e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -338.0,
      "logps/rejected": -320.0,
      "loss": 0.4779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.046875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -3.65625,
      "step": 115
    },
    {
      "epoch": 0.1423312883435583,
      "grad_norm": 32.97644733528144,
      "learning_rate": 1.4435950413223142e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.408203125,
      "logps/chosen": -376.0,
      "logps/rejected": -324.0,
      "loss": 0.2978,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.40625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -3.625,
      "step": 116
    },
    {
      "epoch": 0.14355828220858896,
      "grad_norm": 38.235071091905795,
      "learning_rate": 1.4429752066115703e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.26953125,
      "logps/chosen": -324.0,
      "logps/rejected": -270.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -4.21875,
      "step": 117
    },
    {
      "epoch": 0.14478527607361963,
      "grad_norm": 41.12118912071453,
      "learning_rate": 1.4423553719008265e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -418.0,
      "logps/rejected": -360.0,
      "loss": 0.3658,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 2.28125,
      "rewards/margins": 6.125,
      "rewards/rejected": -3.84375,
      "step": 118
    },
    {
      "epoch": 0.1460122699386503,
      "grad_norm": 35.626041219957244,
      "learning_rate": 1.4417355371900826e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -352.0,
      "logps/rejected": -324.0,
      "loss": 0.3003,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -3.90625,
      "step": 119
    },
    {
      "epoch": 0.147239263803681,
      "grad_norm": 40.42176001046704,
      "learning_rate": 1.4411157024793388e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.37890625,
      "logps/chosen": -418.0,
      "logps/rejected": -354.0,
      "loss": 0.3506,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 2.6875,
      "rewards/margins": 6.625,
      "rewards/rejected": -3.9375,
      "step": 120
    },
    {
      "epoch": 0.14846625766871166,
      "grad_norm": 33.17627892781642,
      "learning_rate": 1.4404958677685951e-06,
      "logits/chosen": -0.12060546875,
      "logits/rejected": -0.27734375,
      "logps/chosen": -404.0,
      "logps/rejected": -336.0,
      "loss": 0.2824,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 2.625,
      "rewards/margins": 6.75,
      "rewards/rejected": -4.125,
      "step": 121
    },
    {
      "epoch": 0.14969325153374233,
      "grad_norm": 41.361282319998544,
      "learning_rate": 1.4398760330578512e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -378.0,
      "logps/rejected": -298.0,
      "loss": 0.3454,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.03125,
      "step": 122
    },
    {
      "epoch": 0.150920245398773,
      "grad_norm": 39.0679541541933,
      "learning_rate": 1.4392561983471074e-06,
      "logits/chosen": -0.0191650390625,
      "logits/rejected": -0.1884765625,
      "logps/chosen": -380.0,
      "logps/rejected": -290.0,
      "loss": 0.384,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.2734375,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.375,
      "step": 123
    },
    {
      "epoch": 0.1521472392638037,
      "grad_norm": 30.841344702577548,
      "learning_rate": 1.4386363636363637e-06,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -348.0,
      "logps/rejected": -294.0,
      "loss": 0.2792,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 2.203125,
      "rewards/margins": 7.0,
      "rewards/rejected": -4.8125,
      "step": 124
    },
    {
      "epoch": 0.15337423312883436,
      "grad_norm": 40.57809799655024,
      "learning_rate": 1.4380165289256199e-06,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.431640625,
      "logps/chosen": -434.0,
      "logps/rejected": -340.0,
      "loss": 0.3257,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.6015625,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.0,
      "step": 125
    },
    {
      "epoch": 0.15460122699386503,
      "grad_norm": 49.176910835221456,
      "learning_rate": 1.437396694214876e-06,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.30078125,
      "logps/chosen": -368.0,
      "logps/rejected": -320.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6328125,
      "rewards/margins": 5.59375,
      "rewards/rejected": -4.96875,
      "step": 126
    },
    {
      "epoch": 0.1558282208588957,
      "grad_norm": 42.06253723411312,
      "learning_rate": 1.4367768595041324e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -388.0,
      "logps/rejected": -340.0,
      "loss": 0.4543,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.40625,
      "rewards/rejected": -4.65625,
      "step": 127
    },
    {
      "epoch": 0.1570552147239264,
      "grad_norm": 49.94695723772631,
      "learning_rate": 1.4361570247933883e-06,
      "logits/chosen": -0.1044921875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -374.0,
      "logps/rejected": -344.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.09375,
      "step": 128
    },
    {
      "epoch": 0.15828220858895706,
      "grad_norm": 31.69804344527413,
      "learning_rate": 1.4355371900826447e-06,
      "logits/chosen": -0.00439453125,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -342.0,
      "logps/rejected": -302.0,
      "loss": 0.3139,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 5.5,
      "rewards/rejected": -4.78125,
      "step": 129
    },
    {
      "epoch": 0.15950920245398773,
      "grad_norm": 39.02664413781982,
      "learning_rate": 1.4349173553719008e-06,
      "logits/chosen": -0.11328125,
      "logits/rejected": -0.251953125,
      "logps/chosen": -388.0,
      "logps/rejected": -324.0,
      "loss": 0.3918,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.375,
      "step": 130
    },
    {
      "epoch": 0.1607361963190184,
      "grad_norm": 33.65136979573152,
      "learning_rate": 1.434297520661157e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.2429,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.3671875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -5.59375,
      "step": 131
    },
    {
      "epoch": 0.1619631901840491,
      "grad_norm": 40.70444309722836,
      "learning_rate": 1.4336776859504133e-06,
      "logits/chosen": -0.1845703125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -392.0,
      "logps/rejected": -336.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.15625,
      "step": 132
    },
    {
      "epoch": 0.16319018404907976,
      "grad_norm": 38.39829631153826,
      "learning_rate": 1.4330578512396694e-06,
      "logits/chosen": -0.220703125,
      "logits/rejected": -0.375,
      "logps/chosen": -388.0,
      "logps/rejected": -342.0,
      "loss": 0.3065,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.46875,
      "step": 133
    },
    {
      "epoch": 0.16441717791411042,
      "grad_norm": 54.071557291057594,
      "learning_rate": 1.4324380165289258e-06,
      "logits/chosen": -0.1416015625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -458.0,
      "logps/rejected": -374.0,
      "loss": 0.4193,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -4.625,
      "step": 134
    },
    {
      "epoch": 0.1656441717791411,
      "grad_norm": 42.09400341898127,
      "learning_rate": 1.431818181818182e-06,
      "logits/chosen": -0.0140380859375,
      "logits/rejected": -0.11767578125,
      "logps/chosen": -378.0,
      "logps/rejected": -330.0,
      "loss": 0.4491,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.16015625,
      "rewards/margins": 5.21875,
      "rewards/rejected": -5.0625,
      "step": 135
    },
    {
      "epoch": 0.1668711656441718,
      "grad_norm": 34.790669543589566,
      "learning_rate": 1.431198347107438e-06,
      "logits/chosen": -0.259765625,
      "logits/rejected": -0.3203125,
      "logps/chosen": -392.0,
      "logps/rejected": -322.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.87890625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.46875,
      "step": 136
    },
    {
      "epoch": 0.16809815950920245,
      "grad_norm": 46.051678091055635,
      "learning_rate": 1.4305785123966942e-06,
      "logits/chosen": -0.020751953125,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -398.0,
      "logps/rejected": -320.0,
      "loss": 0.4495,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.609375,
      "rewards/margins": 5.21875,
      "rewards/rejected": -4.625,
      "step": 137
    },
    {
      "epoch": 0.16932515337423312,
      "grad_norm": 39.60892351140069,
      "learning_rate": 1.4299586776859504e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.2421875,
      "logps/chosen": -406.0,
      "logps/rejected": -360.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 5.59375,
      "rewards/rejected": -5.0,
      "step": 138
    },
    {
      "epoch": 0.1705521472392638,
      "grad_norm": 44.067268759137896,
      "learning_rate": 1.4293388429752067e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.2734375,
      "logps/chosen": -418.0,
      "logps/rejected": -380.0,
      "loss": 0.4261,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.59375,
      "step": 139
    },
    {
      "epoch": 0.17177914110429449,
      "grad_norm": 46.60040949853179,
      "learning_rate": 1.4287190082644628e-06,
      "logits/chosen": -0.07177734375,
      "logits/rejected": -0.251953125,
      "logps/chosen": -426.0,
      "logps/rejected": -376.0,
      "loss": 0.4219,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.1875,
      "step": 140
    },
    {
      "epoch": 0.17300613496932515,
      "grad_norm": 34.208928830023055,
      "learning_rate": 1.428099173553719e-06,
      "logits/chosen": -0.05126953125,
      "logits/rejected": -0.173828125,
      "logps/chosen": -358.0,
      "logps/rejected": -338.0,
      "loss": 0.3019,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -5.28125,
      "step": 141
    },
    {
      "epoch": 0.17423312883435582,
      "grad_norm": 41.96948447879753,
      "learning_rate": 1.4274793388429753e-06,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -400.0,
      "logps/rejected": -354.0,
      "loss": 0.4327,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.98828125,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.375,
      "step": 142
    },
    {
      "epoch": 0.1754601226993865,
      "grad_norm": 39.736356252739405,
      "learning_rate": 1.4268595041322315e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.33984375,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.3314,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -5.625,
      "step": 143
    },
    {
      "epoch": 0.17668711656441718,
      "grad_norm": 40.38397393664774,
      "learning_rate": 1.4262396694214876e-06,
      "logits/chosen": -0.224609375,
      "logits/rejected": -0.3984375,
      "logps/chosen": -420.0,
      "logps/rejected": -356.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.1875,
      "step": 144
    },
    {
      "epoch": 0.17791411042944785,
      "grad_norm": 36.78367346174047,
      "learning_rate": 1.425619834710744e-06,
      "logits/chosen": -0.20703125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -380.0,
      "logps/rejected": -348.0,
      "loss": 0.2992,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.21875,
      "step": 145
    },
    {
      "epoch": 0.17914110429447852,
      "grad_norm": 30.78038936070514,
      "learning_rate": 1.425e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.30859375,
      "logps/chosen": -388.0,
      "logps/rejected": -334.0,
      "loss": 0.2674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.53125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.5,
      "step": 146
    },
    {
      "epoch": 0.18036809815950922,
      "grad_norm": 33.88526702598482,
      "learning_rate": 1.4243801652892563e-06,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -388.0,
      "logps/rejected": -332.0,
      "loss": 0.3151,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.3125,
      "step": 147
    },
    {
      "epoch": 0.18159509202453988,
      "grad_norm": 49.54475790087749,
      "learning_rate": 1.4237603305785124e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -416.0,
      "logps/rejected": -368.0,
      "loss": 0.4146,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.34375,
      "step": 148
    },
    {
      "epoch": 0.18282208588957055,
      "grad_norm": 43.24095049009772,
      "learning_rate": 1.4231404958677685e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.244140625,
      "logps/chosen": -406.0,
      "logps/rejected": -384.0,
      "loss": 0.3173,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.03125,
      "step": 149
    },
    {
      "epoch": 0.18404907975460122,
      "grad_norm": 37.7299116363035,
      "learning_rate": 1.422520661157025e-06,
      "logits/chosen": -0.1181640625,
      "logits/rejected": -0.291015625,
      "logps/chosen": -424.0,
      "logps/rejected": -346.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -4.53125,
      "step": 150
    },
    {
      "epoch": 0.18527607361963191,
      "grad_norm": 36.98087652686202,
      "learning_rate": 1.421900826446281e-06,
      "logits/chosen": -0.1376953125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -382.0,
      "logps/rejected": -326.0,
      "loss": 0.3467,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.625,
      "step": 151
    },
    {
      "epoch": 0.18650306748466258,
      "grad_norm": 33.96526048608977,
      "learning_rate": 1.4212809917355372e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.462890625,
      "logps/chosen": -402.0,
      "logps/rejected": -316.0,
      "loss": 0.322,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.5625,
      "rewards/rejected": -5.71875,
      "step": 152
    },
    {
      "epoch": 0.18773006134969325,
      "grad_norm": 34.59925431133825,
      "learning_rate": 1.4206611570247935e-06,
      "logits/chosen": -0.1875,
      "logits/rejected": -0.298828125,
      "logps/chosen": -350.0,
      "logps/rejected": -320.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.390625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -4.875,
      "step": 153
    },
    {
      "epoch": 0.18895705521472392,
      "grad_norm": 45.111496339674666,
      "learning_rate": 1.4200413223140497e-06,
      "logits/chosen": -0.060546875,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -424.0,
      "logps/rejected": -340.0,
      "loss": 0.3836,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.703125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -4.375,
      "step": 154
    },
    {
      "epoch": 0.1901840490797546,
      "grad_norm": 28.559326947209748,
      "learning_rate": 1.4194214876033058e-06,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.41796875,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.2127,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.359375,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.03125,
      "step": 155
    },
    {
      "epoch": 0.19141104294478528,
      "grad_norm": 41.969901464316806,
      "learning_rate": 1.418801652892562e-06,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -370.0,
      "logps/rejected": -350.0,
      "loss": 0.3849,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 5.59375,
      "rewards/rejected": -4.46875,
      "step": 156
    },
    {
      "epoch": 0.19263803680981595,
      "grad_norm": 35.115688822084685,
      "learning_rate": 1.418181818181818e-06,
      "logits/chosen": -0.05810546875,
      "logits/rejected": -0.1796875,
      "logps/chosen": -394.0,
      "logps/rejected": -310.0,
      "loss": 0.3364,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.625,
      "step": 157
    },
    {
      "epoch": 0.19386503067484662,
      "grad_norm": 29.216146857412586,
      "learning_rate": 1.4175619834710745e-06,
      "logits/chosen": -0.2734375,
      "logits/rejected": -0.44921875,
      "logps/chosen": -388.0,
      "logps/rejected": -324.0,
      "loss": 0.2283,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.390625,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.21875,
      "step": 158
    },
    {
      "epoch": 0.1950920245398773,
      "grad_norm": 39.305920966554595,
      "learning_rate": 1.4169421487603306e-06,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.28125,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.3649,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.359375,
      "rewards/margins": 5.71875,
      "rewards/rejected": -4.34375,
      "step": 159
    },
    {
      "epoch": 0.19631901840490798,
      "grad_norm": 26.74081618489141,
      "learning_rate": 1.416322314049587e-06,
      "logits/chosen": -0.047607421875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -354.0,
      "logps/rejected": -310.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.15625,
      "step": 160
    },
    {
      "epoch": 0.19754601226993865,
      "grad_norm": 32.344289601166764,
      "learning_rate": 1.415702479338843e-06,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.4296875,
      "logps/chosen": -386.0,
      "logps/rejected": -302.0,
      "loss": 0.2831,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.9453125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -4.71875,
      "step": 161
    },
    {
      "epoch": 0.19877300613496932,
      "grad_norm": 39.80838608230206,
      "learning_rate": 1.4150826446280992e-06,
      "logits/chosen": -0.1357421875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -350.0,
      "logps/rejected": -340.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.375,
      "step": 162
    },
    {
      "epoch": 0.2,
      "grad_norm": 38.00229232317159,
      "learning_rate": 1.4144628099173554e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.220703125,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.421875,
      "rewards/margins": 6.25,
      "rewards/rejected": -4.8125,
      "step": 163
    },
    {
      "epoch": 0.20122699386503068,
      "grad_norm": 31.02665679742822,
      "learning_rate": 1.4138429752066115e-06,
      "logits/chosen": -0.236328125,
      "logits/rejected": -0.466796875,
      "logps/chosen": -392.0,
      "logps/rejected": -310.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 2.15625,
      "rewards/margins": 7.34375,
      "rewards/rejected": -5.1875,
      "step": 164
    },
    {
      "epoch": 0.20245398773006135,
      "grad_norm": 43.699969414714815,
      "learning_rate": 1.4132231404958679e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.212890625,
      "logps/chosen": -444.0,
      "logps/rejected": -326.0,
      "loss": 0.377,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -4.78125,
      "step": 165
    },
    {
      "epoch": 0.20368098159509201,
      "grad_norm": 38.30377721360019,
      "learning_rate": 1.412603305785124e-06,
      "logits/chosen": -0.09375,
      "logits/rejected": -0.314453125,
      "logps/chosen": -388.0,
      "logps/rejected": -336.0,
      "loss": 0.3851,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.53125,
      "rewards/rejected": -4.625,
      "step": 166
    },
    {
      "epoch": 0.2049079754601227,
      "grad_norm": 38.75636778509963,
      "learning_rate": 1.4119834710743801e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -398.0,
      "logps/rejected": -318.0,
      "loss": 0.3749,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 2.21875,
      "rewards/margins": 6.875,
      "rewards/rejected": -4.65625,
      "step": 167
    },
    {
      "epoch": 0.20613496932515338,
      "grad_norm": 37.779783793801464,
      "learning_rate": 1.4113636363636365e-06,
      "logits/chosen": -0.049560546875,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -348.0,
      "logps/rejected": -292.0,
      "loss": 0.3337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.46875,
      "step": 168
    },
    {
      "epoch": 0.20736196319018405,
      "grad_norm": 34.571770475148426,
      "learning_rate": 1.4107438016528926e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.27734375,
      "logps/chosen": -372.0,
      "logps/rejected": -366.0,
      "loss": 0.3006,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.25,
      "step": 169
    },
    {
      "epoch": 0.2085889570552147,
      "grad_norm": 42.205138483103795,
      "learning_rate": 1.4101239669421488e-06,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.349609375,
      "logps/chosen": -450.0,
      "logps/rejected": -346.0,
      "loss": 0.338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -5.46875,
      "step": 170
    },
    {
      "epoch": 0.2098159509202454,
      "grad_norm": 29.828895289514126,
      "learning_rate": 1.4095041322314051e-06,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -384.0,
      "logps/rejected": -322.0,
      "loss": 0.3086,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -5.90625,
      "step": 171
    },
    {
      "epoch": 0.21104294478527608,
      "grad_norm": 30.980573058674597,
      "learning_rate": 1.408884297520661e-06,
      "logits/chosen": -0.062255859375,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -438.0,
      "logps/rejected": -354.0,
      "loss": 0.2599,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.84375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.0,
      "step": 172
    },
    {
      "epoch": 0.21226993865030674,
      "grad_norm": 25.22446362010358,
      "learning_rate": 1.4082644628099174e-06,
      "logits/chosen": -0.123046875,
      "logits/rejected": -0.353515625,
      "logps/chosen": -428.0,
      "logps/rejected": -362.0,
      "loss": 0.1997,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.703125,
      "rewards/margins": 8.25,
      "rewards/rejected": -6.5625,
      "step": 173
    },
    {
      "epoch": 0.2134969325153374,
      "grad_norm": 43.79333405262764,
      "learning_rate": 1.4076446280991736e-06,
      "logits/chosen": 0.05126953125,
      "logits/rejected": -0.158203125,
      "logps/chosen": -384.0,
      "logps/rejected": -348.0,
      "loss": 0.5204,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.78125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.03125,
      "step": 174
    },
    {
      "epoch": 0.2147239263803681,
      "grad_norm": 45.27680571467687,
      "learning_rate": 1.4070247933884297e-06,
      "logits/chosen": -0.10546875,
      "logits/rejected": -0.220703125,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.09375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.25,
      "step": 175
    },
    {
      "epoch": 0.21595092024539878,
      "grad_norm": 35.16824036294685,
      "learning_rate": 1.406404958677686e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -434.0,
      "logps/rejected": -378.0,
      "loss": 0.3443,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -6.3125,
      "step": 176
    },
    {
      "epoch": 0.21717791411042944,
      "grad_norm": 27.30910780814845,
      "learning_rate": 1.4057851239669422e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.30859375,
      "logps/chosen": -416.0,
      "logps/rejected": -356.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -6.71875,
      "step": 177
    },
    {
      "epoch": 0.2184049079754601,
      "grad_norm": 37.84132418939681,
      "learning_rate": 1.4051652892561983e-06,
      "logits/chosen": -0.07275390625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -330.0,
      "logps/rejected": -318.0,
      "loss": 0.4137,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.1806640625,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.59375,
      "step": 178
    },
    {
      "epoch": 0.2196319018404908,
      "grad_norm": 32.98852515847511,
      "learning_rate": 1.4045454545454547e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.31640625,
      "logps/chosen": -372.0,
      "logps/rejected": -324.0,
      "loss": 0.3252,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.671875,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.1875,
      "step": 179
    },
    {
      "epoch": 0.22085889570552147,
      "grad_norm": 38.78908549807395,
      "learning_rate": 1.4039256198347108e-06,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.21484375,
      "logps/chosen": -356.0,
      "logps/rejected": -314.0,
      "loss": 0.3669,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.234375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.21875,
      "step": 180
    },
    {
      "epoch": 0.22208588957055214,
      "grad_norm": 35.32530991965649,
      "learning_rate": 1.403305785123967e-06,
      "logits/chosen": -0.09423828125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -410.0,
      "logps/rejected": -386.0,
      "loss": 0.3373,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.6171875,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.25,
      "step": 181
    },
    {
      "epoch": 0.2233128834355828,
      "grad_norm": 36.9736807793252,
      "learning_rate": 1.4026859504132231e-06,
      "logits/chosen": -0.12890625,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.3661,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.109375,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.75,
      "step": 182
    },
    {
      "epoch": 0.2245398773006135,
      "grad_norm": 40.8878376560248,
      "learning_rate": 1.4020661157024793e-06,
      "logits/chosen": 0.00762939453125,
      "logits/rejected": -0.1669921875,
      "logps/chosen": -412.0,
      "logps/rejected": -378.0,
      "loss": 0.3866,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.95703125,
      "rewards/margins": 4.90625,
      "rewards/rejected": -3.953125,
      "step": 183
    },
    {
      "epoch": 0.22576687116564417,
      "grad_norm": 42.156241310731595,
      "learning_rate": 1.4014462809917356e-06,
      "logits/chosen": -0.0255126953125,
      "logits/rejected": -0.1328125,
      "logps/chosen": -390.0,
      "logps/rejected": -340.0,
      "loss": 0.4166,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 5.1875,
      "rewards/rejected": -4.3125,
      "step": 184
    },
    {
      "epoch": 0.22699386503067484,
      "grad_norm": 44.943258114663124,
      "learning_rate": 1.4008264462809917e-06,
      "logits/chosen": -0.06787109375,
      "logits/rejected": -0.1220703125,
      "logps/chosen": -382.0,
      "logps/rejected": -350.0,
      "loss": 0.4011,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.9765625,
      "rewards/margins": 5.8125,
      "rewards/rejected": -3.828125,
      "step": 185
    },
    {
      "epoch": 0.2282208588957055,
      "grad_norm": 33.47267300169336,
      "learning_rate": 1.4002066115702479e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -336.0,
      "logps/rejected": -312.0,
      "loss": 0.3163,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.3125,
      "step": 186
    },
    {
      "epoch": 0.2294478527607362,
      "grad_norm": 32.03885108074069,
      "learning_rate": 1.3995867768595042e-06,
      "logits/chosen": -0.07080078125,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -370.0,
      "logps/rejected": -340.0,
      "loss": 0.258,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.6328125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.78125,
      "step": 187
    },
    {
      "epoch": 0.23067484662576687,
      "grad_norm": 35.158897439200416,
      "learning_rate": 1.3989669421487604e-06,
      "logits/chosen": 0.0228271484375,
      "logits/rejected": -0.0615234375,
      "logps/chosen": -330.0,
      "logps/rejected": -292.0,
      "loss": 0.3609,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.40625,
      "rewards/margins": 5.875,
      "rewards/rejected": -4.46875,
      "step": 188
    },
    {
      "epoch": 0.23190184049079754,
      "grad_norm": 32.15934523587848,
      "learning_rate": 1.3983471074380167e-06,
      "logits/chosen": -0.080078125,
      "logits/rejected": -0.263671875,
      "logps/chosen": -396.0,
      "logps/rejected": -322.0,
      "loss": 0.2271,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 2.046875,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.71875,
      "step": 189
    },
    {
      "epoch": 0.2331288343558282,
      "grad_norm": 32.11586724143221,
      "learning_rate": 1.3977272727272727e-06,
      "logits/chosen": 0.041748046875,
      "logits/rejected": -0.10693359375,
      "logps/chosen": -340.0,
      "logps/rejected": -316.0,
      "loss": 0.3271,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.4140625,
      "rewards/margins": 6.0,
      "rewards/rejected": -4.59375,
      "step": 190
    },
    {
      "epoch": 0.2343558282208589,
      "grad_norm": 46.26761231273777,
      "learning_rate": 1.3971074380165288e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.220703125,
      "logps/chosen": -400.0,
      "logps/rejected": -306.0,
      "loss": 0.3906,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.828125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -5.0625,
      "step": 191
    },
    {
      "epoch": 0.23558282208588957,
      "grad_norm": 39.15535099753523,
      "learning_rate": 1.3964876033057852e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -388.0,
      "logps/rejected": -302.0,
      "loss": 0.3379,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 5.65625,
      "rewards/rejected": -4.5,
      "step": 192
    },
    {
      "epoch": 0.23680981595092024,
      "grad_norm": 31.3365855799527,
      "learning_rate": 1.3958677685950413e-06,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.423828125,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.2485,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.7578125,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.625,
      "step": 193
    },
    {
      "epoch": 0.23803680981595093,
      "grad_norm": 38.5490731428379,
      "learning_rate": 1.3952479338842977e-06,
      "logits/chosen": -0.21875,
      "logits/rejected": -0.326171875,
      "logps/chosen": -394.0,
      "logps/rejected": -336.0,
      "loss": 0.296,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.96875,
      "step": 194
    },
    {
      "epoch": 0.2392638036809816,
      "grad_norm": 35.180225705298604,
      "learning_rate": 1.3946280991735538e-06,
      "logits/chosen": -0.031982421875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -408.0,
      "logps/rejected": -348.0,
      "loss": 0.2727,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 2.046875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.125,
      "step": 195
    },
    {
      "epoch": 0.24049079754601227,
      "grad_norm": 37.22166797602668,
      "learning_rate": 1.39400826446281e-06,
      "logits/chosen": 0.0157470703125,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -394.0,
      "logps/rejected": -308.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.53125,
      "step": 196
    },
    {
      "epoch": 0.24171779141104294,
      "grad_norm": 28.562917988218086,
      "learning_rate": 1.3933884297520663e-06,
      "logits/chosen": -0.115234375,
      "logits/rejected": -0.34765625,
      "logps/chosen": -380.0,
      "logps/rejected": -334.0,
      "loss": 0.3009,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.125,
      "rewards/rejected": -5.78125,
      "step": 197
    },
    {
      "epoch": 0.24294478527607363,
      "grad_norm": 36.54466818998776,
      "learning_rate": 1.3927685950413224e-06,
      "logits/chosen": -0.0174560546875,
      "logits/rejected": -0.10205078125,
      "logps/chosen": -346.0,
      "logps/rejected": -276.0,
      "loss": 0.4465,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -5.4375,
      "step": 198
    },
    {
      "epoch": 0.2441717791411043,
      "grad_norm": 25.79094354827764,
      "learning_rate": 1.3921487603305786e-06,
      "logits/chosen": -0.05615234375,
      "logits/rejected": -0.26171875,
      "logps/chosen": -434.0,
      "logps/rejected": -330.0,
      "loss": 0.1969,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.7265625,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.125,
      "step": 199
    },
    {
      "epoch": 0.24539877300613497,
      "grad_norm": 34.674086597448465,
      "learning_rate": 1.3915289256198347e-06,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.25390625,
      "logps/chosen": -442.0,
      "logps/rejected": -392.0,
      "loss": 0.3066,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.2734375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.15625,
      "step": 200
    },
    {
      "epoch": 0.24662576687116564,
      "grad_norm": 29.903922318940893,
      "learning_rate": 1.3909090909090909e-06,
      "logits/chosen": -0.03662109375,
      "logits/rejected": -0.259765625,
      "logps/chosen": -404.0,
      "logps/rejected": -330.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -5.9375,
      "step": 201
    },
    {
      "epoch": 0.24785276073619633,
      "grad_norm": 33.833661458600005,
      "learning_rate": 1.3902892561983472e-06,
      "logits/chosen": -0.02001953125,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -432.0,
      "logps/rejected": -340.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -5.875,
      "step": 202
    },
    {
      "epoch": 0.249079754601227,
      "grad_norm": 37.642944458655066,
      "learning_rate": 1.3896694214876034e-06,
      "logits/chosen": 0.0272216796875,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -392.0,
      "logps/rejected": -354.0,
      "loss": 0.482,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.96875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.0625,
      "step": 203
    },
    {
      "epoch": 0.25030674846625767,
      "grad_norm": 35.44282157848665,
      "learning_rate": 1.3890495867768595e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.1005859375,
      "logps/chosen": -356.0,
      "logps/rejected": -290.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.78125,
      "step": 204
    },
    {
      "epoch": 0.25153374233128833,
      "grad_norm": 38.48812661842802,
      "learning_rate": 1.3884297520661158e-06,
      "logits/chosen": 0.08984375,
      "logits/rejected": -0.04833984375,
      "logps/chosen": -390.0,
      "logps/rejected": -316.0,
      "loss": 0.4435,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.125,
      "step": 205
    },
    {
      "epoch": 0.252760736196319,
      "grad_norm": 26.640868179250116,
      "learning_rate": 1.387809917355372e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.3125,
      "logps/chosen": -434.0,
      "logps/rejected": -350.0,
      "loss": 0.2375,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 1.203125,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.28125,
      "step": 206
    },
    {
      "epoch": 0.25398773006134967,
      "grad_norm": 37.72822731879787,
      "learning_rate": 1.3871900826446281e-06,
      "logits/chosen": -0.16015625,
      "logits/rejected": -0.353515625,
      "logps/chosen": -434.0,
      "logps/rejected": -376.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.21875,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.53125,
      "step": 207
    },
    {
      "epoch": 0.2552147239263804,
      "grad_norm": 39.99822195882057,
      "learning_rate": 1.3865702479338843e-06,
      "logits/chosen": -0.12255859375,
      "logits/rejected": -0.2470703125,
      "logps/chosen": -396.0,
      "logps/rejected": -340.0,
      "loss": 0.4402,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -5.6875,
      "step": 208
    },
    {
      "epoch": 0.25644171779141106,
      "grad_norm": 31.995520167951547,
      "learning_rate": 1.3859504132231404e-06,
      "logits/chosen": -0.125,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -398.0,
      "logps/rejected": -352.0,
      "loss": 0.3203,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.76171875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.5625,
      "step": 209
    },
    {
      "epoch": 0.25766871165644173,
      "grad_norm": 42.71588448177596,
      "learning_rate": 1.3853305785123968e-06,
      "logits/chosen": -0.0732421875,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -400.0,
      "logps/rejected": -382.0,
      "loss": 0.4413,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.314453125,
      "rewards/margins": 5.90625,
      "rewards/rejected": -5.59375,
      "step": 210
    },
    {
      "epoch": 0.2588957055214724,
      "grad_norm": 35.38452290000429,
      "learning_rate": 1.384710743801653e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.375,
      "logps/chosen": -438.0,
      "logps/rejected": -362.0,
      "loss": 0.3649,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.90625,
      "step": 211
    },
    {
      "epoch": 0.26012269938650306,
      "grad_norm": 35.00437697538439,
      "learning_rate": 1.384090909090909e-06,
      "logits/chosen": -0.099609375,
      "logits/rejected": -0.251953125,
      "logps/chosen": -404.0,
      "logps/rejected": -328.0,
      "loss": 0.3606,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.125,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.5625,
      "step": 212
    },
    {
      "epoch": 0.26134969325153373,
      "grad_norm": 40.33706711227177,
      "learning_rate": 1.3834710743801654e-06,
      "logits/chosen": -0.031982421875,
      "logits/rejected": -0.0888671875,
      "logps/chosen": -352.0,
      "logps/rejected": -296.0,
      "loss": 0.3034,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.4765625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -5.3125,
      "step": 213
    },
    {
      "epoch": 0.2625766871165644,
      "grad_norm": 36.278922089821435,
      "learning_rate": 1.3828512396694215e-06,
      "logits/chosen": -0.0152587890625,
      "logits/rejected": -0.171875,
      "logps/chosen": -376.0,
      "logps/rejected": -326.0,
      "loss": 0.3538,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.1328125,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.25,
      "step": 214
    },
    {
      "epoch": 0.26380368098159507,
      "grad_norm": 39.61373791294869,
      "learning_rate": 1.3822314049586777e-06,
      "logits/chosen": 0.0224609375,
      "logits/rejected": -0.06689453125,
      "logps/chosen": -356.0,
      "logps/rejected": -316.0,
      "loss": 0.3393,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.55859375,
      "rewards/margins": 5.5625,
      "rewards/rejected": -5.0,
      "step": 215
    },
    {
      "epoch": 0.2650306748466258,
      "grad_norm": 36.52111248304287,
      "learning_rate": 1.381611570247934e-06,
      "logits/chosen": -0.04443359375,
      "logits/rejected": -0.1572265625,
      "logps/chosen": -334.0,
      "logps/rejected": -302.0,
      "loss": 0.3612,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.42578125,
      "rewards/margins": 5.375,
      "rewards/rejected": -4.96875,
      "step": 216
    },
    {
      "epoch": 0.26625766871165646,
      "grad_norm": 35.893159477690816,
      "learning_rate": 1.38099173553719e-06,
      "logits/chosen": 0.05517578125,
      "logits/rejected": -0.08154296875,
      "logps/chosen": -402.0,
      "logps/rejected": -316.0,
      "loss": 0.4174,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 6.0625,
      "rewards/rejected": -4.78125,
      "step": 217
    },
    {
      "epoch": 0.2674846625766871,
      "grad_norm": 43.28791050351139,
      "learning_rate": 1.3803719008264463e-06,
      "logits/chosen": 0.0108642578125,
      "logits/rejected": -0.0908203125,
      "logps/chosen": -384.0,
      "logps/rejected": -344.0,
      "loss": 0.514,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.79296875,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.8125,
      "step": 218
    },
    {
      "epoch": 0.2687116564417178,
      "grad_norm": 30.96899305966375,
      "learning_rate": 1.3797520661157025e-06,
      "logits/chosen": -0.091796875,
      "logits/rejected": -0.189453125,
      "logps/chosen": -354.0,
      "logps/rejected": -310.0,
      "loss": 0.3339,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.28125,
      "step": 219
    },
    {
      "epoch": 0.26993865030674846,
      "grad_norm": 49.60415252304962,
      "learning_rate": 1.3791322314049586e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": 0.0047607421875,
      "logps/chosen": -366.0,
      "logps/rejected": -372.0,
      "loss": 0.4563,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.416015625,
      "rewards/margins": 5.0625,
      "rewards/rejected": -4.65625,
      "step": 220
    },
    {
      "epoch": 0.27116564417177913,
      "grad_norm": 35.817382292638634,
      "learning_rate": 1.378512396694215e-06,
      "logits/chosen": -0.049560546875,
      "logits/rejected": -0.25390625,
      "logps/chosen": -364.0,
      "logps/rejected": -306.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 5.75,
      "rewards/rejected": -5.28125,
      "step": 221
    },
    {
      "epoch": 0.2723926380368098,
      "grad_norm": 33.842292905676736,
      "learning_rate": 1.377892561983471e-06,
      "logits/chosen": -0.240234375,
      "logits/rejected": -0.3671875,
      "logps/chosen": -388.0,
      "logps/rejected": -318.0,
      "loss": 0.3321,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 7.375,
      "rewards/rejected": -5.9375,
      "step": 222
    },
    {
      "epoch": 0.27361963190184047,
      "grad_norm": 34.24344157812324,
      "learning_rate": 1.3772727272727274e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -390.0,
      "logps/rejected": -338.0,
      "loss": 0.3389,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.58984375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.59375,
      "step": 223
    },
    {
      "epoch": 0.2748466257668712,
      "grad_norm": 36.32910023908209,
      "learning_rate": 1.3766528925619836e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.33203125,
      "logps/chosen": -452.0,
      "logps/rejected": -352.0,
      "loss": 0.3172,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.28125,
      "rewards/margins": 7.1875,
      "rewards/rejected": -5.90625,
      "step": 224
    },
    {
      "epoch": 0.27607361963190186,
      "grad_norm": 33.96451254717418,
      "learning_rate": 1.3760330578512397e-06,
      "logits/chosen": -0.021484375,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -344.0,
      "logps/rejected": -322.0,
      "loss": 0.2991,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4609375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.0625,
      "step": 225
    },
    {
      "epoch": 0.2773006134969325,
      "grad_norm": 37.753965128366424,
      "learning_rate": 1.3754132231404959e-06,
      "logits/chosen": -0.035888671875,
      "logits/rejected": -0.1689453125,
      "logps/chosen": -378.0,
      "logps/rejected": -328.0,
      "loss": 0.4435,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.625,
      "step": 226
    },
    {
      "epoch": 0.2785276073619632,
      "grad_norm": 37.97162675437385,
      "learning_rate": 1.374793388429752e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -362.0,
      "logps/rejected": -332.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -6.0,
      "step": 227
    },
    {
      "epoch": 0.27975460122699386,
      "grad_norm": 42.90390749582614,
      "learning_rate": 1.3741735537190084e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -366.0,
      "logps/rejected": -338.0,
      "loss": 0.3568,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.57421875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.46875,
      "step": 228
    },
    {
      "epoch": 0.2809815950920245,
      "grad_norm": 35.198650610347336,
      "learning_rate": 1.3735537190082645e-06,
      "logits/chosen": 0.0174560546875,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -422.0,
      "logps/rejected": -344.0,
      "loss": 0.3812,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.640625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.375,
      "step": 229
    },
    {
      "epoch": 0.2822085889570552,
      "grad_norm": 31.76904887089288,
      "learning_rate": 1.3729338842975206e-06,
      "logits/chosen": 0.0027618408203125,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.2964,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.375,
      "rewards/rejected": -5.375,
      "step": 230
    },
    {
      "epoch": 0.28343558282208586,
      "grad_norm": 38.2475164063325,
      "learning_rate": 1.372314049586777e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -370.0,
      "logps/rejected": -320.0,
      "loss": 0.4068,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.65625,
      "step": 231
    },
    {
      "epoch": 0.2846625766871166,
      "grad_norm": 34.785350120825576,
      "learning_rate": 1.3716942148760331e-06,
      "logits/chosen": -0.1474609375,
      "logits/rejected": -0.27734375,
      "logps/chosen": -384.0,
      "logps/rejected": -342.0,
      "loss": 0.3319,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 5.5625,
      "rewards/rejected": -4.875,
      "step": 232
    },
    {
      "epoch": 0.28588957055214725,
      "grad_norm": 33.915962784771814,
      "learning_rate": 1.3710743801652893e-06,
      "logits/chosen": -0.04052734375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -372.0,
      "logps/rejected": -296.0,
      "loss": 0.3377,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.4296875,
      "rewards/margins": 6.28125,
      "rewards/rejected": -4.84375,
      "step": 233
    },
    {
      "epoch": 0.2871165644171779,
      "grad_norm": 36.944834789970336,
      "learning_rate": 1.3704545454545456e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.302734375,
      "logps/chosen": -374.0,
      "logps/rejected": -318.0,
      "loss": 0.3474,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.203125,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.40625,
      "step": 234
    },
    {
      "epoch": 0.2883435582822086,
      "grad_norm": 36.36585753307406,
      "learning_rate": 1.3698347107438016e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.34765625,
      "logps/chosen": -362.0,
      "logps/rejected": -326.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -4.78125,
      "step": 235
    },
    {
      "epoch": 0.28957055214723926,
      "grad_norm": 39.75324625491103,
      "learning_rate": 1.369214876033058e-06,
      "logits/chosen": 0.058837890625,
      "logits/rejected": -0.1123046875,
      "logps/chosen": -396.0,
      "logps/rejected": -374.0,
      "loss": 0.4644,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 5.625,
      "rewards/rejected": -3.890625,
      "step": 236
    },
    {
      "epoch": 0.2907975460122699,
      "grad_norm": 33.93837750645907,
      "learning_rate": 1.368595041322314e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.248046875,
      "logps/chosen": -368.0,
      "logps/rejected": -314.0,
      "loss": 0.3597,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.5,
      "rewards/margins": 6.1875,
      "rewards/rejected": -4.6875,
      "step": 237
    },
    {
      "epoch": 0.2920245398773006,
      "grad_norm": 40.82126535762802,
      "learning_rate": 1.3679752066115702e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.5098,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.4453125,
      "rewards/margins": 5.6875,
      "rewards/rejected": -4.25,
      "step": 238
    },
    {
      "epoch": 0.29325153374233126,
      "grad_norm": 37.96435040796285,
      "learning_rate": 1.3673553719008266e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -382.0,
      "logps/rejected": -314.0,
      "loss": 0.371,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 2.078125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.375,
      "step": 239
    },
    {
      "epoch": 0.294478527607362,
      "grad_norm": 34.11140908926358,
      "learning_rate": 1.3667355371900827e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.28515625,
      "logps/chosen": -418.0,
      "logps/rejected": -364.0,
      "loss": 0.2283,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.890625,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.0,
      "step": 240
    },
    {
      "epoch": 0.29570552147239265,
      "grad_norm": 33.461868421672946,
      "learning_rate": 1.3661157024793388e-06,
      "logits/chosen": -0.076171875,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -366.0,
      "logps/rejected": -336.0,
      "loss": 0.2828,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.8359375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.59375,
      "step": 241
    },
    {
      "epoch": 0.2969325153374233,
      "grad_norm": 32.77289894542164,
      "learning_rate": 1.3654958677685952e-06,
      "logits/chosen": -0.10009765625,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -350.0,
      "logps/rejected": -324.0,
      "loss": 0.2975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.2578125,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.34375,
      "step": 242
    },
    {
      "epoch": 0.298159509202454,
      "grad_norm": 38.07813065801456,
      "learning_rate": 1.3648760330578511e-06,
      "logits/chosen": -0.1923828125,
      "logits/rejected": -0.328125,
      "logps/chosen": -390.0,
      "logps/rejected": -364.0,
      "loss": 0.4242,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.328125,
      "rewards/margins": 5.625,
      "rewards/rejected": -4.3125,
      "step": 243
    },
    {
      "epoch": 0.29938650306748466,
      "grad_norm": 39.871497711059654,
      "learning_rate": 1.3642561983471075e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.39453125,
      "logps/chosen": -386.0,
      "logps/rejected": -346.0,
      "loss": 0.5523,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.265625,
      "rewards/margins": 5.46875,
      "rewards/rejected": -4.1875,
      "step": 244
    },
    {
      "epoch": 0.3006134969325153,
      "grad_norm": 32.221505039188294,
      "learning_rate": 1.3636363636363636e-06,
      "logits/chosen": -0.15625,
      "logits/rejected": -0.2890625,
      "logps/chosen": -398.0,
      "logps/rejected": -354.0,
      "loss": 0.2967,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.53125,
      "step": 245
    },
    {
      "epoch": 0.301840490797546,
      "grad_norm": 34.1015902082894,
      "learning_rate": 1.3630165289256198e-06,
      "logits/chosen": -0.166015625,
      "logits/rejected": -0.31640625,
      "logps/chosen": -374.0,
      "logps/rejected": -346.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.171875,
      "rewards/margins": 6.46875,
      "rewards/rejected": -4.3125,
      "step": 246
    },
    {
      "epoch": 0.3030674846625767,
      "grad_norm": 32.83309359401401,
      "learning_rate": 1.3623966942148761e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.337890625,
      "logps/chosen": -372.0,
      "logps/rejected": -308.0,
      "loss": 0.3534,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -4.78125,
      "step": 247
    },
    {
      "epoch": 0.3042944785276074,
      "grad_norm": 33.089465845458236,
      "learning_rate": 1.3617768595041322e-06,
      "logits/chosen": -0.076171875,
      "logits/rejected": -0.3203125,
      "logps/chosen": -388.0,
      "logps/rejected": -332.0,
      "loss": 0.2682,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.703125,
      "rewards/margins": 6.03125,
      "rewards/rejected": -4.3125,
      "step": 248
    },
    {
      "epoch": 0.30552147239263805,
      "grad_norm": 39.03476607944447,
      "learning_rate": 1.3611570247933884e-06,
      "logits/chosen": -0.20703125,
      "logits/rejected": -0.396484375,
      "logps/chosen": -474.0,
      "logps/rejected": -368.0,
      "loss": 0.3241,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.640625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.15625,
      "step": 249
    },
    {
      "epoch": 0.3067484662576687,
      "grad_norm": 38.29235099515543,
      "learning_rate": 1.3605371900826447e-06,
      "logits/chosen": -0.30078125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -340.0,
      "logps/rejected": -324.0,
      "loss": 0.3804,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.34375,
      "rewards/margins": 5.78125,
      "rewards/rejected": -4.4375,
      "step": 250
    },
    {
      "epoch": 0.3079754601226994,
      "grad_norm": 37.34357400280288,
      "learning_rate": 1.3599173553719009e-06,
      "logits/chosen": 0.0289306640625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -376.0,
      "logps/rejected": -308.0,
      "loss": 0.4118,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 1.421875,
      "rewards/margins": 5.375,
      "rewards/rejected": -3.9375,
      "step": 251
    },
    {
      "epoch": 0.30920245398773005,
      "grad_norm": 37.841749969537275,
      "learning_rate": 1.359297520661157e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.29296875,
      "logps/chosen": -392.0,
      "logps/rejected": -310.0,
      "loss": 0.4506,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.890625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -3.875,
      "step": 252
    },
    {
      "epoch": 0.3104294478527607,
      "grad_norm": 25.74895782175542,
      "learning_rate": 1.3586776859504132e-06,
      "logits/chosen": -0.1630859375,
      "logits/rejected": -0.392578125,
      "logps/chosen": -336.0,
      "logps/rejected": -298.0,
      "loss": 0.2694,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.46875,
      "rewards/margins": 6.0625,
      "rewards/rejected": -4.59375,
      "step": 253
    },
    {
      "epoch": 0.3116564417177914,
      "grad_norm": 36.56021325822385,
      "learning_rate": 1.3580578512396693e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.25,
      "logps/chosen": -352.0,
      "logps/rejected": -320.0,
      "loss": 0.4183,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.8671875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -4.3125,
      "step": 254
    },
    {
      "epoch": 0.3128834355828221,
      "grad_norm": 37.45936986772829,
      "learning_rate": 1.3574380165289257e-06,
      "logits/chosen": -0.07958984375,
      "logits/rejected": -0.228515625,
      "logps/chosen": -384.0,
      "logps/rejected": -312.0,
      "loss": 0.439,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.8515625,
      "rewards/margins": 5.71875,
      "rewards/rejected": -3.859375,
      "step": 255
    },
    {
      "epoch": 0.3141104294478528,
      "grad_norm": 33.113238785557265,
      "learning_rate": 1.3568181818181818e-06,
      "logits/chosen": -0.1640625,
      "logits/rejected": -0.328125,
      "logps/chosen": -348.0,
      "logps/rejected": -316.0,
      "loss": 0.3148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3125,
      "rewards/margins": 7.0,
      "rewards/rejected": -4.6875,
      "step": 256
    },
    {
      "epoch": 0.31533742331288345,
      "grad_norm": 34.825667619250744,
      "learning_rate": 1.3561983471074382e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.2890625,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.3455,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 2.25,
      "rewards/margins": 6.5625,
      "rewards/rejected": -4.3125,
      "step": 257
    },
    {
      "epoch": 0.3165644171779141,
      "grad_norm": 35.08263037322888,
      "learning_rate": 1.3555785123966943e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.28125,
      "logps/chosen": -396.0,
      "logps/rejected": -342.0,
      "loss": 0.319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.75,
      "step": 258
    },
    {
      "epoch": 0.3177914110429448,
      "grad_norm": 33.547677568046986,
      "learning_rate": 1.3549586776859504e-06,
      "logits/chosen": -0.1689453125,
      "logits/rejected": -0.349609375,
      "logps/chosen": -344.0,
      "logps/rejected": -356.0,
      "loss": 0.3553,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.5,
      "rewards/rejected": -5.1875,
      "step": 259
    },
    {
      "epoch": 0.31901840490797545,
      "grad_norm": 35.38305104419636,
      "learning_rate": 1.3543388429752068e-06,
      "logits/chosen": -0.1904296875,
      "logits/rejected": -0.26953125,
      "logps/chosen": -390.0,
      "logps/rejected": -326.0,
      "loss": 0.3657,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.15625,
      "step": 260
    },
    {
      "epoch": 0.3202453987730061,
      "grad_norm": 37.62925536561278,
      "learning_rate": 1.3537190082644627e-06,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.263671875,
      "logps/chosen": -380.0,
      "logps/rejected": -330.0,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.296875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.53125,
      "step": 261
    },
    {
      "epoch": 0.3214723926380368,
      "grad_norm": 45.47620412654006,
      "learning_rate": 1.353099173553719e-06,
      "logits/chosen": -0.1591796875,
      "logits/rejected": -0.3359375,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.4019,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -5.0625,
      "step": 262
    },
    {
      "epoch": 0.3226993865030675,
      "grad_norm": 43.246204937096174,
      "learning_rate": 1.3524793388429752e-06,
      "logits/chosen": -0.1494140625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.4521,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0,
      "step": 263
    },
    {
      "epoch": 0.3239263803680982,
      "grad_norm": 37.571005698421835,
      "learning_rate": 1.3518595041322314e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.20703125,
      "logps/chosen": -354.0,
      "logps/rejected": -308.0,
      "loss": 0.3748,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.15625,
      "step": 264
    },
    {
      "epoch": 0.32515337423312884,
      "grad_norm": 45.351424976255885,
      "learning_rate": 1.3512396694214877e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.265625,
      "logps/chosen": -360.0,
      "logps/rejected": -288.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.875,
      "step": 265
    },
    {
      "epoch": 0.3263803680981595,
      "grad_norm": 44.31098246879602,
      "learning_rate": 1.3506198347107439e-06,
      "logits/chosen": -0.07861328125,
      "logits/rejected": -0.228515625,
      "logps/chosen": -326.0,
      "logps/rejected": -316.0,
      "loss": 0.5312,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.27734375,
      "rewards/margins": 5.25,
      "rewards/rejected": -5.53125,
      "step": 266
    },
    {
      "epoch": 0.3276073619631902,
      "grad_norm": 43.337206726154484,
      "learning_rate": 1.35e-06,
      "logits/chosen": -0.0966796875,
      "logits/rejected": -0.31640625,
      "logps/chosen": -406.0,
      "logps/rejected": -322.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.2421875,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.625,
      "step": 267
    },
    {
      "epoch": 0.32883435582822085,
      "grad_norm": 28.822383795146568,
      "learning_rate": 1.3493801652892563e-06,
      "logits/chosen": -0.25390625,
      "logits/rejected": -0.365234375,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.209,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.65625,
      "step": 268
    },
    {
      "epoch": 0.3300613496932515,
      "grad_norm": 30.291051549986957,
      "learning_rate": 1.3487603305785125e-06,
      "logits/chosen": -0.1240234375,
      "logits/rejected": -0.43359375,
      "logps/chosen": -412.0,
      "logps/rejected": -322.0,
      "loss": 0.237,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.3125,
      "step": 269
    },
    {
      "epoch": 0.3312883435582822,
      "grad_norm": 38.1594801225088,
      "learning_rate": 1.3481404958677686e-06,
      "logits/chosen": -0.09912109375,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -334.0,
      "logps/rejected": -308.0,
      "loss": 0.4766,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 5.625,
      "rewards/rejected": -5.15625,
      "step": 270
    },
    {
      "epoch": 0.3325153374233129,
      "grad_norm": 28.374821367974555,
      "learning_rate": 1.3475206611570248e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.240234375,
      "logps/chosen": -390.0,
      "logps/rejected": -344.0,
      "loss": 0.3004,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3984375,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.09375,
      "step": 271
    },
    {
      "epoch": 0.3337423312883436,
      "grad_norm": 34.17489484053231,
      "learning_rate": 1.346900826446281e-06,
      "logits/chosen": -0.00823974609375,
      "logits/rejected": -0.171875,
      "logps/chosen": -366.0,
      "logps/rejected": -318.0,
      "loss": 0.2861,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.423828125,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.34375,
      "step": 272
    },
    {
      "epoch": 0.33496932515337424,
      "grad_norm": 41.21053534408477,
      "learning_rate": 1.3462809917355373e-06,
      "logits/chosen": 0.06640625,
      "logits/rejected": -0.007110595703125,
      "logps/chosen": -344.0,
      "logps/rejected": -314.0,
      "loss": 0.5165,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 0.166015625,
      "rewards/margins": 5.75,
      "rewards/rejected": -5.5625,
      "step": 273
    },
    {
      "epoch": 0.3361963190184049,
      "grad_norm": 34.571787806623156,
      "learning_rate": 1.3456611570247934e-06,
      "logits/chosen": 0.0419921875,
      "logits/rejected": -0.1025390625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3839,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.5625,
      "step": 274
    },
    {
      "epoch": 0.3374233128834356,
      "grad_norm": 31.979565052575825,
      "learning_rate": 1.3450413223140495e-06,
      "logits/chosen": -0.0302734375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -326.0,
      "logps/rejected": -326.0,
      "loss": 0.4175,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.412109375,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.6875,
      "step": 275
    },
    {
      "epoch": 0.33865030674846625,
      "grad_norm": 28.189301031425032,
      "learning_rate": 1.344421487603306e-06,
      "logits/chosen": 0.0498046875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -390.0,
      "logps/rejected": -330.0,
      "loss": 0.3505,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.96875,
      "step": 276
    },
    {
      "epoch": 0.3398773006134969,
      "grad_norm": 29.004541701793432,
      "learning_rate": 1.343801652892562e-06,
      "logits/chosen": -0.15234375,
      "logits/rejected": -0.337890625,
      "logps/chosen": -414.0,
      "logps/rejected": -344.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.375,
      "step": 277
    },
    {
      "epoch": 0.3411042944785276,
      "grad_norm": 40.163399091060114,
      "learning_rate": 1.3431818181818184e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -384.0,
      "logps/rejected": -356.0,
      "loss": 0.4008,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.75,
      "step": 278
    },
    {
      "epoch": 0.3423312883435583,
      "grad_norm": 21.8515947233153,
      "learning_rate": 1.3425619834710743e-06,
      "logits/chosen": -0.10888671875,
      "logits/rejected": -0.2890625,
      "logps/chosen": -392.0,
      "logps/rejected": -326.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.8125,
      "step": 279
    },
    {
      "epoch": 0.34355828220858897,
      "grad_norm": 34.753521492334585,
      "learning_rate": 1.3419421487603305e-06,
      "logits/chosen": -0.09228515625,
      "logits/rejected": -0.234375,
      "logps/chosen": -408.0,
      "logps/rejected": -372.0,
      "loss": 0.4097,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.15625,
      "step": 280
    },
    {
      "epoch": 0.34478527607361964,
      "grad_norm": 35.589195433139196,
      "learning_rate": 1.3413223140495868e-06,
      "logits/chosen": -0.00921630859375,
      "logits/rejected": -0.201171875,
      "logps/chosen": -382.0,
      "logps/rejected": -330.0,
      "loss": 0.3917,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.0625,
      "step": 281
    },
    {
      "epoch": 0.3460122699386503,
      "grad_norm": 29.302596470256102,
      "learning_rate": 1.340702479338843e-06,
      "logits/chosen": -0.2021484375,
      "logits/rejected": -0.341796875,
      "logps/chosen": -344.0,
      "logps/rejected": -324.0,
      "loss": 0.2481,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.09375,
      "step": 282
    },
    {
      "epoch": 0.347239263803681,
      "grad_norm": 33.7375126879184,
      "learning_rate": 1.3400826446280993e-06,
      "logits/chosen": -0.2265625,
      "logits/rejected": -0.3125,
      "logps/chosen": -354.0,
      "logps/rejected": -298.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.59765625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.5625,
      "step": 283
    },
    {
      "epoch": 0.34846625766871164,
      "grad_norm": 35.97047400633379,
      "learning_rate": 1.3394628099173555e-06,
      "logits/chosen": -0.034423828125,
      "logits/rejected": -0.166015625,
      "logps/chosen": -370.0,
      "logps/rejected": -320.0,
      "loss": 0.2954,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.859375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.84375,
      "step": 284
    },
    {
      "epoch": 0.3496932515337423,
      "grad_norm": 29.423070534885003,
      "learning_rate": 1.3388429752066116e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.322265625,
      "logps/chosen": -360.0,
      "logps/rejected": -328.0,
      "loss": 0.2845,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.80078125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.875,
      "step": 285
    },
    {
      "epoch": 0.350920245398773,
      "grad_norm": 35.376791661357736,
      "learning_rate": 1.338223140495868e-06,
      "logits/chosen": -0.04736328125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -422.0,
      "logps/rejected": -356.0,
      "loss": 0.3799,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.0625,
      "step": 286
    },
    {
      "epoch": 0.3521472392638037,
      "grad_norm": 38.51049635159653,
      "learning_rate": 1.337603305785124e-06,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.283203125,
      "logps/chosen": -382.0,
      "logps/rejected": -330.0,
      "loss": 0.3715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.8125,
      "step": 287
    },
    {
      "epoch": 0.35337423312883437,
      "grad_norm": 31.395092155644413,
      "learning_rate": 1.3369834710743802e-06,
      "logits/chosen": 0.057373046875,
      "logits/rejected": -0.0888671875,
      "logps/chosen": -386.0,
      "logps/rejected": -336.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.298828125,
      "rewards/margins": 5.9375,
      "rewards/rejected": -5.625,
      "step": 288
    },
    {
      "epoch": 0.35460122699386504,
      "grad_norm": 25.699158986974325,
      "learning_rate": 1.3363636363636364e-06,
      "logits/chosen": -0.11083984375,
      "logits/rejected": -0.248046875,
      "logps/chosen": -404.0,
      "logps/rejected": -330.0,
      "loss": 0.1963,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -6.71875,
      "step": 289
    },
    {
      "epoch": 0.3558282208588957,
      "grad_norm": 26.172712857085628,
      "learning_rate": 1.3357438016528925e-06,
      "logits/chosen": -0.283203125,
      "logits/rejected": -0.41796875,
      "logps/chosen": -380.0,
      "logps/rejected": -350.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.0,
      "step": 290
    },
    {
      "epoch": 0.3570552147239264,
      "grad_norm": 35.706855965144605,
      "learning_rate": 1.3351239669421489e-06,
      "logits/chosen": -0.2275390625,
      "logits/rejected": -0.3359375,
      "logps/chosen": -394.0,
      "logps/rejected": -340.0,
      "loss": 0.4375,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.5625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.53125,
      "step": 291
    },
    {
      "epoch": 0.35828220858895704,
      "grad_norm": 32.40631116792141,
      "learning_rate": 1.334504132231405e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.390625,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2821,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.7734375,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.40625,
      "step": 292
    },
    {
      "epoch": 0.3595092024539877,
      "grad_norm": 27.724555696310663,
      "learning_rate": 1.3338842975206611e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -382.0,
      "logps/rejected": -332.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.033203125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.71875,
      "step": 293
    },
    {
      "epoch": 0.36073619631901843,
      "grad_norm": 26.756162333522596,
      "learning_rate": 1.3332644628099175e-06,
      "logits/chosen": -0.359375,
      "logits/rejected": -0.4921875,
      "logps/chosen": -472.0,
      "logps/rejected": -368.0,
      "loss": 0.3094,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.0,
      "step": 294
    },
    {
      "epoch": 0.3619631901840491,
      "grad_norm": 27.599547481570436,
      "learning_rate": 1.3326446280991736e-06,
      "logits/chosen": -0.04345703125,
      "logits/rejected": -0.1884765625,
      "logps/chosen": -356.0,
      "logps/rejected": -332.0,
      "loss": 0.2495,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.2734375,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.59375,
      "step": 295
    },
    {
      "epoch": 0.36319018404907977,
      "grad_norm": 35.1081345543023,
      "learning_rate": 1.3320247933884298e-06,
      "logits/chosen": -0.007598876953125,
      "logits/rejected": -0.154296875,
      "logps/chosen": -420.0,
      "logps/rejected": -338.0,
      "loss": 0.3852,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -5.5,
      "step": 296
    },
    {
      "epoch": 0.36441717791411044,
      "grad_norm": 31.495501539039623,
      "learning_rate": 1.331404958677686e-06,
      "logits/chosen": -0.154296875,
      "logits/rejected": -0.30859375,
      "logps/chosen": -360.0,
      "logps/rejected": -322.0,
      "loss": 0.2553,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.79296875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.25,
      "step": 297
    },
    {
      "epoch": 0.3656441717791411,
      "grad_norm": 37.56929706779854,
      "learning_rate": 1.330785123966942e-06,
      "logits/chosen": -0.009521484375,
      "logits/rejected": -0.09716796875,
      "logps/chosen": -384.0,
      "logps/rejected": -336.0,
      "loss": 0.3993,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.498046875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.53125,
      "step": 298
    },
    {
      "epoch": 0.36687116564417177,
      "grad_norm": 36.46646666012725,
      "learning_rate": 1.3301652892561984e-06,
      "logits/chosen": -0.058837890625,
      "logits/rejected": -0.228515625,
      "logps/chosen": -400.0,
      "logps/rejected": -334.0,
      "loss": 0.3895,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1650390625,
      "rewards/margins": 5.78125,
      "rewards/rejected": -5.9375,
      "step": 299
    },
    {
      "epoch": 0.36809815950920244,
      "grad_norm": 37.01196046043553,
      "learning_rate": 1.3295454545454546e-06,
      "logits/chosen": -0.173828125,
      "logits/rejected": -0.3203125,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.4644,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.482421875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.15625,
      "step": 300
    },
    {
      "epoch": 0.3693251533742331,
      "grad_norm": 32.17417313364543,
      "learning_rate": 1.3289256198347107e-06,
      "logits/chosen": -0.11865234375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -460.0,
      "logps/rejected": -386.0,
      "loss": 0.3417,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.34375,
      "step": 301
    },
    {
      "epoch": 0.37055214723926383,
      "grad_norm": 25.798891470824255,
      "learning_rate": 1.328305785123967e-06,
      "logits/chosen": -0.150390625,
      "logits/rejected": -0.34765625,
      "logps/chosen": -382.0,
      "logps/rejected": -316.0,
      "loss": 0.2452,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.287109375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.40625,
      "step": 302
    },
    {
      "epoch": 0.3717791411042945,
      "grad_norm": 40.25697610802204,
      "learning_rate": 1.3276859504132232e-06,
      "logits/chosen": -0.1474609375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -388.0,
      "logps/rejected": -376.0,
      "loss": 0.4569,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.466796875,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.65625,
      "step": 303
    },
    {
      "epoch": 0.37300613496932516,
      "grad_norm": 30.538056754282714,
      "learning_rate": 1.3270661157024793e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -396.0,
      "logps/rejected": -334.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.140625,
      "rewards/margins": 6.875,
      "rewards/rejected": -7.0,
      "step": 304
    },
    {
      "epoch": 0.37423312883435583,
      "grad_norm": 27.228440712710828,
      "learning_rate": 1.3264462809917357e-06,
      "logits/chosen": -0.109375,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -416.0,
      "logps/rejected": -344.0,
      "loss": 0.2623,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.341796875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.375,
      "step": 305
    },
    {
      "epoch": 0.3754601226993865,
      "grad_norm": 34.02132219179046,
      "learning_rate": 1.3258264462809916e-06,
      "logits/chosen": -0.2177734375,
      "logits/rejected": -0.37890625,
      "logps/chosen": -430.0,
      "logps/rejected": -364.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.33203125,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.0625,
      "step": 306
    },
    {
      "epoch": 0.37668711656441717,
      "grad_norm": 34.60907155238812,
      "learning_rate": 1.325206611570248e-06,
      "logits/chosen": -0.06103515625,
      "logits/rejected": -0.19140625,
      "logps/chosen": -382.0,
      "logps/rejected": -344.0,
      "loss": 0.4791,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.18359375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -7.09375,
      "step": 307
    },
    {
      "epoch": 0.37791411042944784,
      "grad_norm": 34.523248358542865,
      "learning_rate": 1.3245867768595041e-06,
      "logits/chosen": -0.00148773193359375,
      "logits/rejected": -0.162109375,
      "logps/chosen": -398.0,
      "logps/rejected": -360.0,
      "loss": 0.3857,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.5625,
      "step": 308
    },
    {
      "epoch": 0.3791411042944785,
      "grad_norm": 29.962650665710807,
      "learning_rate": 1.3239669421487603e-06,
      "logits/chosen": -0.208984375,
      "logits/rejected": -0.31640625,
      "logps/chosen": -388.0,
      "logps/rejected": -354.0,
      "loss": 0.2543,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.609375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.9375,
      "step": 309
    },
    {
      "epoch": 0.3803680981595092,
      "grad_norm": 33.20481159292772,
      "learning_rate": 1.3233471074380166e-06,
      "logits/chosen": 0.051513671875,
      "logits/rejected": -0.037353515625,
      "logps/chosen": -322.0,
      "logps/rejected": -312.0,
      "loss": 0.4174,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.875,
      "rewards/margins": 5.5,
      "rewards/rejected": -6.375,
      "step": 310
    },
    {
      "epoch": 0.3815950920245399,
      "grad_norm": 32.36453885140526,
      "learning_rate": 1.3227272727272727e-06,
      "logits/chosen": -0.158203125,
      "logits/rejected": -0.3359375,
      "logps/chosen": -392.0,
      "logps/rejected": -348.0,
      "loss": 0.3525,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.486328125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.28125,
      "step": 311
    },
    {
      "epoch": 0.38282208588957056,
      "grad_norm": 39.08052846369179,
      "learning_rate": 1.322107438016529e-06,
      "logits/chosen": 0.0162353515625,
      "logits/rejected": -0.1142578125,
      "logps/chosen": -392.0,
      "logps/rejected": -378.0,
      "loss": 0.5608,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": 0.0712890625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.28125,
      "step": 312
    },
    {
      "epoch": 0.38404907975460123,
      "grad_norm": 35.15009453427041,
      "learning_rate": 1.3214876033057852e-06,
      "logits/chosen": -0.0235595703125,
      "logits/rejected": -0.298828125,
      "logps/chosen": -410.0,
      "logps/rejected": -372.0,
      "loss": 0.383,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -6.9375,
      "step": 313
    },
    {
      "epoch": 0.3852760736196319,
      "grad_norm": 28.038472021341548,
      "learning_rate": 1.3208677685950414e-06,
      "logits/chosen": -0.05517578125,
      "logits/rejected": -0.2421875,
      "logps/chosen": -458.0,
      "logps/rejected": -360.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 2.03125,
      "rewards/margins": 9.375,
      "rewards/rejected": -7.34375,
      "step": 314
    },
    {
      "epoch": 0.38650306748466257,
      "grad_norm": 31.702508477036716,
      "learning_rate": 1.3202479338842975e-06,
      "logits/chosen": 0.076171875,
      "logits/rejected": -0.11865234375,
      "logps/chosen": -350.0,
      "logps/rejected": -300.0,
      "loss": 0.3293,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.2431640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.59375,
      "step": 315
    },
    {
      "epoch": 0.38773006134969323,
      "grad_norm": 27.443273044054905,
      "learning_rate": 1.3196280991735537e-06,
      "logits/chosen": 0.05419921875,
      "logits/rejected": -0.08544921875,
      "logps/chosen": -360.0,
      "logps/rejected": -308.0,
      "loss": 0.3129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.8125,
      "step": 316
    },
    {
      "epoch": 0.3889570552147239,
      "grad_norm": 28.339536052474916,
      "learning_rate": 1.31900826446281e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": -0.3046875,
      "logps/chosen": -400.0,
      "logps/rejected": -368.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.15625,
      "step": 317
    },
    {
      "epoch": 0.3901840490797546,
      "grad_norm": 38.792002326692455,
      "learning_rate": 1.3183884297520662e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.296875,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4379,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.91796875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.125,
      "step": 318
    },
    {
      "epoch": 0.3914110429447853,
      "grad_norm": 17.992013049840406,
      "learning_rate": 1.3177685950413223e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -414.0,
      "logps/rejected": -320.0,
      "loss": 0.1471,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.65625,
      "step": 319
    },
    {
      "epoch": 0.39263803680981596,
      "grad_norm": 31.193757820598883,
      "learning_rate": 1.3171487603305787e-06,
      "logits/chosen": -0.056884765625,
      "logits/rejected": -0.166015625,
      "logps/chosen": -320.0,
      "logps/rejected": -286.0,
      "loss": 0.325,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.53125,
      "step": 320
    },
    {
      "epoch": 0.39386503067484663,
      "grad_norm": 39.71075166734815,
      "learning_rate": 1.3165289256198348e-06,
      "logits/chosen": -0.201171875,
      "logits/rejected": -0.35546875,
      "logps/chosen": -410.0,
      "logps/rejected": -398.0,
      "loss": 0.3784,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.375,
      "step": 321
    },
    {
      "epoch": 0.3950920245398773,
      "grad_norm": 26.7824487712344,
      "learning_rate": 1.315909090909091e-06,
      "logits/chosen": -0.107421875,
      "logits/rejected": -0.2578125,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.203125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.65625,
      "step": 322
    },
    {
      "epoch": 0.39631901840490796,
      "grad_norm": 36.45438287117724,
      "learning_rate": 1.3152892561983473e-06,
      "logits/chosen": -0.1318359375,
      "logits/rejected": -0.265625,
      "logps/chosen": -354.0,
      "logps/rejected": -334.0,
      "loss": 0.4474,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.6875,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.46875,
      "step": 323
    },
    {
      "epoch": 0.39754601226993863,
      "grad_norm": 31.15880722522094,
      "learning_rate": 1.3146694214876032e-06,
      "logits/chosen": -0.193359375,
      "logits/rejected": -0.361328125,
      "logps/chosen": -356.0,
      "logps/rejected": -328.0,
      "loss": 0.3486,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.8125,
      "step": 324
    },
    {
      "epoch": 0.3987730061349693,
      "grad_norm": 36.47850348671778,
      "learning_rate": 1.3140495867768596e-06,
      "logits/chosen": -0.0272216796875,
      "logits/rejected": -0.09326171875,
      "logps/chosen": -330.0,
      "logps/rejected": -304.0,
      "loss": 0.5542,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.33203125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.84375,
      "step": 325
    },
    {
      "epoch": 0.4,
      "grad_norm": 26.62936328977598,
      "learning_rate": 1.3134297520661157e-06,
      "logits/chosen": -0.2294921875,
      "logits/rejected": -0.369140625,
      "logps/chosen": -398.0,
      "logps/rejected": -374.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.5546875,
      "rewards/margins": 8.375,
      "rewards/rejected": -6.84375,
      "step": 326
    },
    {
      "epoch": 0.4012269938650307,
      "grad_norm": 39.76264342514329,
      "learning_rate": 1.3128099173553719e-06,
      "logits/chosen": -0.1015625,
      "logits/rejected": -0.271484375,
      "logps/chosen": -430.0,
      "logps/rejected": -352.0,
      "loss": 0.4557,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.921875,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.1875,
      "step": 327
    },
    {
      "epoch": 0.40245398773006136,
      "grad_norm": 33.62855666124103,
      "learning_rate": 1.3121900826446282e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -362.0,
      "logps/rejected": -288.0,
      "loss": 0.363,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.228515625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.9375,
      "step": 328
    },
    {
      "epoch": 0.403680981595092,
      "grad_norm": 36.1359556726495,
      "learning_rate": 1.3115702479338844e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -466.0,
      "logps/rejected": -386.0,
      "loss": 0.3754,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.078125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.25,
      "step": 329
    },
    {
      "epoch": 0.4049079754601227,
      "grad_norm": 27.601952208856858,
      "learning_rate": 1.3109504132231405e-06,
      "logits/chosen": -0.22265625,
      "logits/rejected": -0.423828125,
      "logps/chosen": -418.0,
      "logps/rejected": -352.0,
      "loss": 0.2524,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.25,
      "step": 330
    },
    {
      "epoch": 0.40613496932515336,
      "grad_norm": 26.526762211128467,
      "learning_rate": 1.3103305785123968e-06,
      "logits/chosen": -0.1767578125,
      "logits/rejected": -0.283203125,
      "logps/chosen": -408.0,
      "logps/rejected": -356.0,
      "loss": 0.2755,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.09375,
      "rewards/margins": 7.625,
      "rewards/rejected": -6.53125,
      "step": 331
    },
    {
      "epoch": 0.40736196319018403,
      "grad_norm": 24.05844405102659,
      "learning_rate": 1.3097107438016528e-06,
      "logits/chosen": -0.1826171875,
      "logits/rejected": -0.37109375,
      "logps/chosen": -430.0,
      "logps/rejected": -332.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -6.84375,
      "step": 332
    },
    {
      "epoch": 0.4085889570552147,
      "grad_norm": 36.41468851723906,
      "learning_rate": 1.3090909090909091e-06,
      "logits/chosen": 0.017822265625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -372.0,
      "logps/rejected": -346.0,
      "loss": 0.4229,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 0.48046875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.90625,
      "step": 333
    },
    {
      "epoch": 0.4098159509202454,
      "grad_norm": 30.33603786358444,
      "learning_rate": 1.3084710743801653e-06,
      "logits/chosen": -0.1181640625,
      "logits/rejected": -0.349609375,
      "logps/chosen": -382.0,
      "logps/rejected": -366.0,
      "loss": 0.3109,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.458984375,
      "rewards/margins": 7.75,
      "rewards/rejected": -7.28125,
      "step": 334
    },
    {
      "epoch": 0.4110429447852761,
      "grad_norm": 33.01078113652063,
      "learning_rate": 1.3078512396694214e-06,
      "logits/chosen": -0.0213623046875,
      "logits/rejected": -0.28125,
      "logps/chosen": -432.0,
      "logps/rejected": -332.0,
      "loss": 0.2319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.34375,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.71875,
      "step": 335
    },
    {
      "epoch": 0.41226993865030676,
      "grad_norm": 31.965240510538873,
      "learning_rate": 1.3072314049586778e-06,
      "logits/chosen": -0.177734375,
      "logits/rejected": -0.326171875,
      "logps/chosen": -402.0,
      "logps/rejected": -352.0,
      "loss": 0.3352,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.8125,
      "step": 336
    },
    {
      "epoch": 0.4134969325153374,
      "grad_norm": 36.98349242868308,
      "learning_rate": 1.306611570247934e-06,
      "logits/chosen": 0.0264892578125,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -358.0,
      "logps/rejected": -352.0,
      "loss": 0.4393,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.75,
      "step": 337
    },
    {
      "epoch": 0.4147239263803681,
      "grad_norm": 31.08894150655132,
      "learning_rate": 1.30599173553719e-06,
      "logits/chosen": -0.005157470703125,
      "logits/rejected": -0.1484375,
      "logps/chosen": -392.0,
      "logps/rejected": -354.0,
      "loss": 0.3686,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.75,
      "step": 338
    },
    {
      "epoch": 0.41595092024539876,
      "grad_norm": 34.40525022311832,
      "learning_rate": 1.3053719008264464e-06,
      "logits/chosen": 0.0654296875,
      "logits/rejected": -0.06103515625,
      "logps/chosen": -378.0,
      "logps/rejected": -346.0,
      "loss": 0.3699,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.02880859375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.5625,
      "step": 339
    },
    {
      "epoch": 0.4171779141104294,
      "grad_norm": 35.90111385254676,
      "learning_rate": 1.3047520661157025e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.267578125,
      "logps/chosen": -366.0,
      "logps/rejected": -350.0,
      "loss": 0.3448,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.058349609375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.1875,
      "step": 340
    },
    {
      "epoch": 0.41840490797546015,
      "grad_norm": 27.984626902771293,
      "learning_rate": 1.3041322314049587e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.3125,
      "logps/chosen": -398.0,
      "logps/rejected": -334.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.75,
      "step": 341
    },
    {
      "epoch": 0.4196319018404908,
      "grad_norm": 34.66561631382139,
      "learning_rate": 1.3035123966942148e-06,
      "logits/chosen": -0.021240234375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -340.0,
      "logps/rejected": -318.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.049072265625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.03125,
      "step": 342
    },
    {
      "epoch": 0.4208588957055215,
      "grad_norm": 34.35806991693024,
      "learning_rate": 1.302892561983471e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.1513671875,
      "logps/chosen": -386.0,
      "logps/rejected": -356.0,
      "loss": 0.4138,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.251953125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -5.8125,
      "step": 343
    },
    {
      "epoch": 0.42208588957055215,
      "grad_norm": 36.57762286698119,
      "learning_rate": 1.3022727272727273e-06,
      "logits/chosen": -0.0654296875,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -408.0,
      "logps/rejected": -340.0,
      "loss": 0.4341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.036376953125,
      "rewards/margins": 5.875,
      "rewards/rejected": -5.84375,
      "step": 344
    },
    {
      "epoch": 0.4233128834355828,
      "grad_norm": 32.698706365667604,
      "learning_rate": 1.3016528925619835e-06,
      "logits/chosen": -0.04931640625,
      "logits/rejected": -0.2265625,
      "logps/chosen": -414.0,
      "logps/rejected": -340.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2177734375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.21875,
      "step": 345
    },
    {
      "epoch": 0.4245398773006135,
      "grad_norm": 34.95248105852957,
      "learning_rate": 1.3010330578512398e-06,
      "logits/chosen": -0.036376953125,
      "logits/rejected": -0.23046875,
      "logps/chosen": -344.0,
      "logps/rejected": -306.0,
      "loss": 0.4481,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.19921875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -6.03125,
      "step": 346
    },
    {
      "epoch": 0.42576687116564416,
      "grad_norm": 27.628512071923325,
      "learning_rate": 1.300413223140496e-06,
      "logits/chosen": -0.06884765625,
      "logits/rejected": -0.1875,
      "logps/chosen": -370.0,
      "logps/rejected": -356.0,
      "loss": 0.317,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.20703125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.90625,
      "step": 347
    },
    {
      "epoch": 0.4269938650306748,
      "grad_norm": 24.80952384589789,
      "learning_rate": 1.299793388429752e-06,
      "logits/chosen": -0.12451171875,
      "logits/rejected": -0.318359375,
      "logps/chosen": -418.0,
      "logps/rejected": -328.0,
      "loss": 0.2563,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 7.4375,
      "rewards/rejected": -7.3125,
      "step": 348
    },
    {
      "epoch": 0.42822085889570555,
      "grad_norm": 38.377577445863466,
      "learning_rate": 1.2991735537190084e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.2412109375,
      "logps/chosen": -406.0,
      "logps/rejected": -322.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.046875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.84375,
      "step": 349
    },
    {
      "epoch": 0.4294478527607362,
      "grad_norm": 37.75213306544608,
      "learning_rate": 1.2985537190082644e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": -0.12890625,
      "logps/chosen": -412.0,
      "logps/rejected": -318.0,
      "loss": 0.3697,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.045654296875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.0,
      "step": 350
    },
    {
      "epoch": 0.4306748466257669,
      "grad_norm": 35.53586893463626,
      "learning_rate": 1.2979338842975207e-06,
      "logits/chosen": -0.11279296875,
      "logits/rejected": -0.251953125,
      "logps/chosen": -342.0,
      "logps/rejected": -326.0,
      "loss": 0.4086,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1337890625,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.1875,
      "step": 351
    },
    {
      "epoch": 0.43190184049079755,
      "grad_norm": 30.59296136721169,
      "learning_rate": 1.2973140495867769e-06,
      "logits/chosen": -0.0230712890625,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -386.0,
      "logps/rejected": -334.0,
      "loss": 0.4358,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.0625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.28125,
      "step": 352
    },
    {
      "epoch": 0.4331288343558282,
      "grad_norm": 24.76896622328251,
      "learning_rate": 1.296694214876033e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.244140625,
      "logps/chosen": -360.0,
      "logps/rejected": -330.0,
      "loss": 0.2411,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.921875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.5625,
      "step": 353
    },
    {
      "epoch": 0.4343558282208589,
      "grad_norm": 31.01854459387497,
      "learning_rate": 1.2960743801652894e-06,
      "logits/chosen": 0.01043701171875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -444.0,
      "logps/rejected": -378.0,
      "loss": 0.3525,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -5.53125,
      "step": 354
    },
    {
      "epoch": 0.43558282208588955,
      "grad_norm": 32.456178590454805,
      "learning_rate": 1.2954545454545455e-06,
      "logits/chosen": -0.1572265625,
      "logits/rejected": -0.24609375,
      "logps/chosen": -348.0,
      "logps/rejected": -356.0,
      "loss": 0.4542,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.296875,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.0,
      "step": 355
    },
    {
      "epoch": 0.4368098159509202,
      "grad_norm": 31.595656958351967,
      "learning_rate": 1.2948347107438016e-06,
      "logits/chosen": 0.06396484375,
      "logits/rejected": -0.0859375,
      "logps/chosen": -396.0,
      "logps/rejected": -332.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -5.59375,
      "step": 356
    },
    {
      "epoch": 0.43803680981595094,
      "grad_norm": 28.72400215925028,
      "learning_rate": 1.294214876033058e-06,
      "logits/chosen": -0.23046875,
      "logits/rejected": -0.384765625,
      "logps/chosen": -402.0,
      "logps/rejected": -340.0,
      "loss": 0.3662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.78125,
      "step": 357
    },
    {
      "epoch": 0.4392638036809816,
      "grad_norm": 29.5841283121426,
      "learning_rate": 1.2935950413223141e-06,
      "logits/chosen": -0.1708984375,
      "logits/rejected": -0.328125,
      "logps/chosen": -446.0,
      "logps/rejected": -330.0,
      "loss": 0.3109,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 7.125,
      "rewards/rejected": -5.90625,
      "step": 358
    },
    {
      "epoch": 0.4404907975460123,
      "grad_norm": 33.37778751879116,
      "learning_rate": 1.2929752066115703e-06,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.154296875,
      "logps/chosen": -424.0,
      "logps/rejected": -334.0,
      "loss": 0.3631,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.89453125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.8125,
      "step": 359
    },
    {
      "epoch": 0.44171779141104295,
      "grad_norm": 32.72250588219352,
      "learning_rate": 1.2923553719008264e-06,
      "logits/chosen": -0.02587890625,
      "logits/rejected": -0.1484375,
      "logps/chosen": -396.0,
      "logps/rejected": -358.0,
      "loss": 0.2778,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.33984375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.59375,
      "step": 360
    },
    {
      "epoch": 0.4429447852760736,
      "grad_norm": 29.99311979639803,
      "learning_rate": 1.2917355371900826e-06,
      "logits/chosen": 0.010498046875,
      "logits/rejected": -0.177734375,
      "logps/chosen": -438.0,
      "logps/rejected": -382.0,
      "loss": 0.3143,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.875,
      "step": 361
    },
    {
      "epoch": 0.4441717791411043,
      "grad_norm": 30.82537911112007,
      "learning_rate": 1.291115702479339e-06,
      "logits/chosen": -0.11083984375,
      "logits/rejected": -0.267578125,
      "logps/chosen": -418.0,
      "logps/rejected": -344.0,
      "loss": 0.3606,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.62109375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.59375,
      "step": 362
    },
    {
      "epoch": 0.44539877300613495,
      "grad_norm": 26.541431319509446,
      "learning_rate": 1.290495867768595e-06,
      "logits/chosen": -0.018798828125,
      "logits/rejected": -0.171875,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.2712,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.77734375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.03125,
      "step": 363
    },
    {
      "epoch": 0.4466257668711656,
      "grad_norm": 36.761947246861894,
      "learning_rate": 1.2898760330578512e-06,
      "logits/chosen": 0.06591796875,
      "logits/rejected": -0.109375,
      "logps/chosen": -392.0,
      "logps/rejected": -314.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.765625,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.0,
      "step": 364
    },
    {
      "epoch": 0.44785276073619634,
      "grad_norm": 27.87437703363369,
      "learning_rate": 1.2892561983471076e-06,
      "logits/chosen": -0.0018463134765625,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -352.0,
      "logps/rejected": -314.0,
      "loss": 0.3086,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.53125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.125,
      "step": 365
    },
    {
      "epoch": 0.449079754601227,
      "grad_norm": 32.787747807253844,
      "learning_rate": 1.2886363636363637e-06,
      "logits/chosen": 0.0291748046875,
      "logits/rejected": -0.06787109375,
      "logps/chosen": -366.0,
      "logps/rejected": -332.0,
      "loss": 0.447,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.87890625,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.125,
      "step": 366
    },
    {
      "epoch": 0.4503067484662577,
      "grad_norm": 26.662012319210124,
      "learning_rate": 1.2880165289256198e-06,
      "logits/chosen": -0.142578125,
      "logits/rejected": -0.326171875,
      "logps/chosen": -394.0,
      "logps/rejected": -336.0,
      "loss": 0.2357,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.9453125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.71875,
      "step": 367
    },
    {
      "epoch": 0.45153374233128835,
      "grad_norm": 36.83996177643189,
      "learning_rate": 1.287396694214876e-06,
      "logits/chosen": 0.06787109375,
      "logits/rejected": 0.0234375,
      "logps/chosen": -368.0,
      "logps/rejected": -328.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.3125,
      "step": 368
    },
    {
      "epoch": 0.452760736196319,
      "grad_norm": 35.726856908761434,
      "learning_rate": 1.2867768595041321e-06,
      "logits/chosen": 0.05322265625,
      "logits/rejected": -0.06591796875,
      "logps/chosen": -344.0,
      "logps/rejected": -298.0,
      "loss": 0.3738,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.169921875,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.5625,
      "step": 369
    },
    {
      "epoch": 0.4539877300613497,
      "grad_norm": 31.108323501043053,
      "learning_rate": 1.2861570247933885e-06,
      "logits/chosen": -0.1484375,
      "logits/rejected": -0.328125,
      "logps/chosen": -402.0,
      "logps/rejected": -348.0,
      "loss": 0.369,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.75,
      "rewards/margins": 6.65625,
      "rewards/rejected": -4.90625,
      "step": 370
    },
    {
      "epoch": 0.45521472392638035,
      "grad_norm": 31.06340590217051,
      "learning_rate": 1.2855371900826446e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -460.0,
      "logps/rejected": -364.0,
      "loss": 0.3181,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.0,
      "rewards/rejected": -4.375,
      "step": 371
    },
    {
      "epoch": 0.456441717791411,
      "grad_norm": 28.189166139003635,
      "learning_rate": 1.2849173553719008e-06,
      "logits/chosen": -0.0301513671875,
      "logits/rejected": -0.1875,
      "logps/chosen": -410.0,
      "logps/rejected": -324.0,
      "loss": 0.2419,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.4375,
      "rewards/rejected": -4.9375,
      "step": 372
    },
    {
      "epoch": 0.45766871165644174,
      "grad_norm": 33.21287873989905,
      "learning_rate": 1.2842975206611571e-06,
      "logits/chosen": -0.053466796875,
      "logits/rejected": -0.1591796875,
      "logps/chosen": -362.0,
      "logps/rejected": -322.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.58203125,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.125,
      "step": 373
    },
    {
      "epoch": 0.4588957055214724,
      "grad_norm": 26.151862774833457,
      "learning_rate": 1.2836776859504132e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -382.0,
      "logps/rejected": -316.0,
      "loss": 0.2534,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 2.0,
      "rewards/margins": 6.78125,
      "rewards/rejected": -4.78125,
      "step": 374
    },
    {
      "epoch": 0.4601226993865031,
      "grad_norm": 32.35488575008133,
      "learning_rate": 1.2830578512396696e-06,
      "logits/chosen": -0.14453125,
      "logits/rejected": -0.2177734375,
      "logps/chosen": -392.0,
      "logps/rejected": -350.0,
      "loss": 0.2771,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.7109375,
      "rewards/margins": 5.8125,
      "rewards/rejected": -4.09375,
      "step": 375
    },
    {
      "epoch": 0.46134969325153374,
      "grad_norm": 34.001922340864844,
      "learning_rate": 1.2824380165289257e-06,
      "logits/chosen": -0.00946044921875,
      "logits/rejected": -0.15625,
      "logps/chosen": -376.0,
      "logps/rejected": -344.0,
      "loss": 0.3408,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.6953125,
      "rewards/margins": 5.75,
      "rewards/rejected": -4.03125,
      "step": 376
    },
    {
      "epoch": 0.4625766871165644,
      "grad_norm": 35.73955039729585,
      "learning_rate": 1.2818181818181817e-06,
      "logits/chosen": -0.0927734375,
      "logits/rejected": -0.150390625,
      "logps/chosen": -352.0,
      "logps/rejected": -318.0,
      "loss": 0.4247,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": 1.5625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.0625,
      "step": 377
    },
    {
      "epoch": 0.4638036809815951,
      "grad_norm": 33.82458737973786,
      "learning_rate": 1.281198347107438e-06,
      "logits/chosen": -0.1396484375,
      "logits/rejected": -0.376953125,
      "logps/chosen": -376.0,
      "logps/rejected": -332.0,
      "loss": 0.3987,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.625,
      "rewards/rejected": -5.0,
      "step": 378
    },
    {
      "epoch": 0.46503067484662575,
      "grad_norm": 31.86429872362659,
      "learning_rate": 1.2805785123966942e-06,
      "logits/chosen": -0.1953125,
      "logits/rejected": -0.359375,
      "logps/chosen": -408.0,
      "logps/rejected": -344.0,
      "loss": 0.3028,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.71875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -5.46875,
      "step": 379
    },
    {
      "epoch": 0.4662576687116564,
      "grad_norm": 31.446849732786838,
      "learning_rate": 1.2799586776859505e-06,
      "logits/chosen": -0.09326171875,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -382.0,
      "logps/rejected": -326.0,
      "loss": 0.3068,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.15625,
      "step": 380
    },
    {
      "epoch": 0.46748466257668714,
      "grad_norm": 31.679856008810177,
      "learning_rate": 1.2793388429752067e-06,
      "logits/chosen": -0.189453125,
      "logits/rejected": -0.3671875,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.374,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.046875,
      "rewards/margins": 7.0,
      "rewards/rejected": -5.9375,
      "step": 381
    },
    {
      "epoch": 0.4687116564417178,
      "grad_norm": 31.436122523167693,
      "learning_rate": 1.2787190082644628e-06,
      "logits/chosen": -0.185546875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.3158,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.078125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.8125,
      "step": 382
    },
    {
      "epoch": 0.4699386503067485,
      "grad_norm": 35.065632595795535,
      "learning_rate": 1.2780991735537192e-06,
      "logits/chosen": -0.09912109375,
      "logits/rejected": -0.2578125,
      "logps/chosen": -402.0,
      "logps/rejected": -378.0,
      "loss": 0.4456,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.1015625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -5.6875,
      "step": 383
    },
    {
      "epoch": 0.47116564417177914,
      "grad_norm": 33.79101177267741,
      "learning_rate": 1.2774793388429753e-06,
      "logits/chosen": -0.2001953125,
      "logits/rejected": -0.435546875,
      "logps/chosen": -398.0,
      "logps/rejected": -342.0,
      "loss": 0.2961,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.640625,
      "rewards/margins": 7.6875,
      "rewards/rejected": -7.03125,
      "step": 384
    },
    {
      "epoch": 0.4723926380368098,
      "grad_norm": 36.518171637968166,
      "learning_rate": 1.2768595041322314e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.283203125,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.3771,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.283203125,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.0625,
      "step": 385
    },
    {
      "epoch": 0.4736196319018405,
      "grad_norm": 34.95682432591222,
      "learning_rate": 1.2762396694214876e-06,
      "logits/chosen": -0.043701171875,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -342.0,
      "logps/rejected": -310.0,
      "loss": 0.369,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3203125,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.40625,
      "step": 386
    },
    {
      "epoch": 0.47484662576687114,
      "grad_norm": 64.7195376952873,
      "learning_rate": 1.2756198347107437e-06,
      "logits/chosen": -0.068359375,
      "logits/rejected": -0.271484375,
      "logps/chosen": -422.0,
      "logps/rejected": -350.0,
      "loss": 0.4035,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.6171875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.75,
      "step": 387
    },
    {
      "epoch": 0.47607361963190187,
      "grad_norm": 42.357455920572896,
      "learning_rate": 1.275e-06,
      "logits/chosen": -0.08349609375,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -406.0,
      "logps/rejected": -326.0,
      "loss": 0.5163,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.04736328125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -5.59375,
      "step": 388
    },
    {
      "epoch": 0.47730061349693254,
      "grad_norm": 38.953026072404185,
      "learning_rate": 1.2743801652892562e-06,
      "logits/chosen": -0.1435546875,
      "logits/rejected": -0.2236328125,
      "logps/chosen": -412.0,
      "logps/rejected": -340.0,
      "loss": 0.4455,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.099609375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.1875,
      "step": 389
    },
    {
      "epoch": 0.4785276073619632,
      "grad_norm": 38.5573431700193,
      "learning_rate": 1.2737603305785124e-06,
      "logits/chosen": -0.2216796875,
      "logits/rejected": -0.34375,
      "logps/chosen": -372.0,
      "logps/rejected": -344.0,
      "loss": 0.492,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.62890625,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.65625,
      "step": 390
    },
    {
      "epoch": 0.47975460122699387,
      "grad_norm": 37.845045364551986,
      "learning_rate": 1.2731404958677687e-06,
      "logits/chosen": -0.1201171875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -390.0,
      "logps/rejected": -344.0,
      "loss": 0.4807,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.181640625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.625,
      "step": 391
    },
    {
      "epoch": 0.48098159509202454,
      "grad_norm": 29.781954844950388,
      "learning_rate": 1.2725206611570249e-06,
      "logits/chosen": -0.060791015625,
      "logits/rejected": -0.208984375,
      "logps/chosen": -362.0,
      "logps/rejected": -330.0,
      "loss": 0.3103,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06982421875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -6.65625,
      "step": 392
    },
    {
      "epoch": 0.4822085889570552,
      "grad_norm": 33.5926238632541,
      "learning_rate": 1.271900826446281e-06,
      "logits/chosen": -0.0286865234375,
      "logits/rejected": -0.166015625,
      "logps/chosen": -416.0,
      "logps/rejected": -374.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.263671875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.6875,
      "step": 393
    },
    {
      "epoch": 0.4834355828220859,
      "grad_norm": 32.087849602750985,
      "learning_rate": 1.2712809917355373e-06,
      "logits/chosen": -0.09521484375,
      "logits/rejected": -0.298828125,
      "logps/chosen": -398.0,
      "logps/rejected": -342.0,
      "loss": 0.262,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.166015625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.375,
      "step": 394
    },
    {
      "epoch": 0.48466257668711654,
      "grad_norm": 34.842476262528464,
      "learning_rate": 1.2706611570247933e-06,
      "logits/chosen": -0.1025390625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -448.0,
      "logps/rejected": -356.0,
      "loss": 0.3764,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.09375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.21875,
      "step": 395
    },
    {
      "epoch": 0.48588957055214727,
      "grad_norm": 22.42844686295793,
      "learning_rate": 1.2700413223140496e-06,
      "logits/chosen": -0.1806640625,
      "logits/rejected": -0.388671875,
      "logps/chosen": -348.0,
      "logps/rejected": -332.0,
      "loss": 0.2277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.54296875,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.875,
      "step": 396
    },
    {
      "epoch": 0.48711656441717793,
      "grad_norm": 33.97555781703141,
      "learning_rate": 1.2694214876033058e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.314453125,
      "logps/chosen": -388.0,
      "logps/rejected": -350.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.40625,
      "step": 397
    },
    {
      "epoch": 0.4883435582822086,
      "grad_norm": 39.08030474072834,
      "learning_rate": 1.268801652892562e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.27734375,
      "logps/chosen": -428.0,
      "logps/rejected": -346.0,
      "loss": 0.4427,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.08740234375,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.46875,
      "step": 398
    },
    {
      "epoch": 0.48957055214723927,
      "grad_norm": 33.83840388728663,
      "learning_rate": 1.2681818181818183e-06,
      "logits/chosen": -0.1611328125,
      "logits/rejected": -0.291015625,
      "logps/chosen": -394.0,
      "logps/rejected": -346.0,
      "loss": 0.2963,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.0035858154296875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.53125,
      "step": 399
    },
    {
      "epoch": 0.49079754601226994,
      "grad_norm": 41.49480537946823,
      "learning_rate": 1.2675619834710744e-06,
      "logits/chosen": -0.09130859375,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -384.0,
      "logps/rejected": -368.0,
      "loss": 0.5062,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 5.59375,
      "rewards/rejected": -6.15625,
      "step": 400
    },
    {
      "epoch": 0.4920245398773006,
      "grad_norm": 27.800803034272004,
      "learning_rate": 1.2669421487603308e-06,
      "logits/chosen": -0.23828125,
      "logits/rejected": -0.373046875,
      "logps/chosen": -456.0,
      "logps/rejected": -390.0,
      "loss": 0.2938,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.40625,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.1875,
      "step": 401
    },
    {
      "epoch": 0.49325153374233127,
      "grad_norm": 32.09882908595564,
      "learning_rate": 1.266322314049587e-06,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.1796875,
      "logps/chosen": -394.0,
      "logps/rejected": -350.0,
      "loss": 0.349,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.04443359375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.53125,
      "step": 402
    },
    {
      "epoch": 0.49447852760736194,
      "grad_norm": 29.930227321008882,
      "learning_rate": 1.265702479338843e-06,
      "logits/chosen": -0.07958984375,
      "logits/rejected": -0.15625,
      "logps/chosen": -396.0,
      "logps/rejected": -374.0,
      "loss": 0.341,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.033447265625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.8125,
      "step": 403
    },
    {
      "epoch": 0.49570552147239266,
      "grad_norm": 25.987833840695732,
      "learning_rate": 1.2650826446280992e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.2606,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.1669921875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.90625,
      "step": 404
    },
    {
      "epoch": 0.49693251533742333,
      "grad_norm": 38.14679307536266,
      "learning_rate": 1.2644628099173553e-06,
      "logits/chosen": -0.1064453125,
      "logits/rejected": -0.328125,
      "logps/chosen": -370.0,
      "logps/rejected": -322.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.03125,
      "step": 405
    },
    {
      "epoch": 0.498159509202454,
      "grad_norm": 37.82975021877776,
      "learning_rate": 1.2638429752066117e-06,
      "logits/chosen": 0.0166015625,
      "logits/rejected": -0.0625,
      "logps/chosen": -378.0,
      "logps/rejected": -340.0,
      "loss": 0.5427,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 6.09375,
      "rewards/rejected": -5.75,
      "step": 406
    },
    {
      "epoch": 0.49938650306748467,
      "grad_norm": 38.332851373691135,
      "learning_rate": 1.2632231404958678e-06,
      "logits/chosen": 0.0115966796875,
      "logits/rejected": -0.134765625,
      "logps/chosen": -408.0,
      "logps/rejected": -344.0,
      "loss": 0.4382,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.34375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.0625,
      "step": 407
    },
    {
      "epoch": 0.5006134969325153,
      "grad_norm": 41.6208959235071,
      "learning_rate": 1.262603305785124e-06,
      "logits/chosen": -0.083984375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -426.0,
      "logps/rejected": -360.0,
      "loss": 0.5081,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.091796875,
      "rewards/margins": 5.1875,
      "rewards/rejected": -5.28125,
      "step": 408
    },
    {
      "epoch": 0.501840490797546,
      "grad_norm": 42.88666645295333,
      "learning_rate": 1.2619834710743803e-06,
      "logits/chosen": -0.00860595703125,
      "logits/rejected": -0.205078125,
      "logps/chosen": -426.0,
      "logps/rejected": -342.0,
      "loss": 0.4119,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.56640625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -6.15625,
      "step": 409
    },
    {
      "epoch": 0.5030674846625767,
      "grad_norm": 36.428858153344706,
      "learning_rate": 1.2613636363636365e-06,
      "logits/chosen": -0.00095367431640625,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -388.0,
      "logps/rejected": -360.0,
      "loss": 0.4038,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.3828125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.75,
      "step": 410
    },
    {
      "epoch": 0.5042944785276073,
      "grad_norm": 33.21094894864319,
      "learning_rate": 1.2607438016528926e-06,
      "logits/chosen": 0.0140380859375,
      "logits/rejected": -0.12890625,
      "logps/chosen": -342.0,
      "logps/rejected": -320.0,
      "loss": 0.3058,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.1298828125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.34375,
      "step": 411
    },
    {
      "epoch": 0.505521472392638,
      "grad_norm": 42.15288062503197,
      "learning_rate": 1.260123966942149e-06,
      "logits/chosen": 0.007080078125,
      "logits/rejected": -0.1259765625,
      "logps/chosen": -428.0,
      "logps/rejected": -368.0,
      "loss": 0.4444,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.30078125,
      "rewards/margins": 5.96875,
      "rewards/rejected": -5.6875,
      "step": 412
    },
    {
      "epoch": 0.5067484662576687,
      "grad_norm": 37.795734491735246,
      "learning_rate": 1.2595041322314049e-06,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.26171875,
      "logps/chosen": -408.0,
      "logps/rejected": -332.0,
      "loss": 0.3878,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.0625,
      "step": 413
    },
    {
      "epoch": 0.5079754601226993,
      "grad_norm": 37.27154344231617,
      "learning_rate": 1.2588842975206612e-06,
      "logits/chosen": -0.0859375,
      "logits/rejected": -0.2265625,
      "logps/chosen": -406.0,
      "logps/rejected": -354.0,
      "loss": 0.3559,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.85546875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.84375,
      "step": 414
    },
    {
      "epoch": 0.50920245398773,
      "grad_norm": 30.678885373774655,
      "learning_rate": 1.2582644628099174e-06,
      "logits/chosen": -0.041748046875,
      "logits/rejected": -0.212890625,
      "logps/chosen": -350.0,
      "logps/rejected": -332.0,
      "loss": 0.3048,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.21875,
      "step": 415
    },
    {
      "epoch": 0.5104294478527608,
      "grad_norm": 26.882010464050307,
      "learning_rate": 1.2576446280991735e-06,
      "logits/chosen": -0.004730224609375,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -360.0,
      "logps/rejected": -324.0,
      "loss": 0.1899,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.458984375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.28125,
      "step": 416
    },
    {
      "epoch": 0.5116564417177915,
      "grad_norm": 30.354406082685717,
      "learning_rate": 1.2570247933884299e-06,
      "logits/chosen": -0.2197265625,
      "logits/rejected": -0.384765625,
      "logps/chosen": -438.0,
      "logps/rejected": -382.0,
      "loss": 0.3453,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.4765625,
      "rewards/margins": 8.125,
      "rewards/rejected": -6.65625,
      "step": 417
    },
    {
      "epoch": 0.5128834355828221,
      "grad_norm": 33.53763392215037,
      "learning_rate": 1.256404958677686e-06,
      "logits/chosen": -0.126953125,
      "logits/rejected": -0.2099609375,
      "logps/chosen": -368.0,
      "logps/rejected": -306.0,
      "loss": 0.3744,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.7890625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.375,
      "step": 418
    },
    {
      "epoch": 0.5141104294478528,
      "grad_norm": 28.10440726715766,
      "learning_rate": 1.2557851239669421e-06,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.21875,
      "logps/chosen": -380.0,
      "logps/rejected": -342.0,
      "loss": 0.3206,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.78125,
      "step": 419
    },
    {
      "epoch": 0.5153374233128835,
      "grad_norm": 30.627621824355007,
      "learning_rate": 1.2551652892561985e-06,
      "logits/chosen": 0.0262451171875,
      "logits/rejected": -0.134765625,
      "logps/chosen": -346.0,
      "logps/rejected": -312.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.796875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.65625,
      "step": 420
    },
    {
      "epoch": 0.5165644171779141,
      "grad_norm": 31.844076287237254,
      "learning_rate": 1.2545454545454544e-06,
      "logits/chosen": -0.01544189453125,
      "logits/rejected": -0.2158203125,
      "logps/chosen": -364.0,
      "logps/rejected": -320.0,
      "loss": 0.4123,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.1875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.625,
      "step": 421
    },
    {
      "epoch": 0.5177914110429448,
      "grad_norm": 33.522822368907185,
      "learning_rate": 1.2539256198347108e-06,
      "logits/chosen": 0.0595703125,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -360.0,
      "logps/rejected": -302.0,
      "loss": 0.3724,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.52734375,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.15625,
      "step": 422
    },
    {
      "epoch": 0.5190184049079755,
      "grad_norm": 37.40841671544669,
      "learning_rate": 1.253305785123967e-06,
      "logits/chosen": -0.0595703125,
      "logits/rejected": -0.20703125,
      "logps/chosen": -398.0,
      "logps/rejected": -362.0,
      "loss": 0.4749,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.578125,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.4375,
      "step": 423
    },
    {
      "epoch": 0.5202453987730061,
      "grad_norm": 34.546118881756456,
      "learning_rate": 1.252685950413223e-06,
      "logits/chosen": 0.0235595703125,
      "logits/rejected": -0.166015625,
      "logps/chosen": -378.0,
      "logps/rejected": -342.0,
      "loss": 0.4503,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.875,
      "step": 424
    },
    {
      "epoch": 0.5214723926380368,
      "grad_norm": 37.84213132068162,
      "learning_rate": 1.2520661157024794e-06,
      "logits/chosen": -0.0123291015625,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -396.0,
      "logps/rejected": -348.0,
      "loss": 0.5144,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.3203125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.53125,
      "step": 425
    },
    {
      "epoch": 0.5226993865030675,
      "grad_norm": 36.174280199739684,
      "learning_rate": 1.2514462809917356e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.166015625,
      "logps/chosen": -370.0,
      "logps/rejected": -348.0,
      "loss": 0.4214,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.015625,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.0625,
      "step": 426
    },
    {
      "epoch": 0.5239263803680981,
      "grad_norm": 45.37106746020955,
      "learning_rate": 1.2508264462809917e-06,
      "logits/chosen": -0.01483154296875,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -378.0,
      "logps/rejected": -334.0,
      "loss": 0.4318,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.625,
      "step": 427
    },
    {
      "epoch": 0.5251533742331288,
      "grad_norm": 33.589023647082186,
      "learning_rate": 1.250206611570248e-06,
      "logits/chosen": -0.0419921875,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -364.0,
      "logps/rejected": -324.0,
      "loss": 0.4488,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.462890625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.0,
      "step": 428
    },
    {
      "epoch": 0.5263803680981595,
      "grad_norm": 30.11424815064351,
      "learning_rate": 1.2495867768595042e-06,
      "logits/chosen": -0.044677734375,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -360.0,
      "logps/rejected": -376.0,
      "loss": 0.2639,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.373046875,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.59375,
      "step": 429
    },
    {
      "epoch": 0.5276073619631901,
      "grad_norm": 23.36575464672381,
      "learning_rate": 1.2489669421487603e-06,
      "logits/chosen": -0.11767578125,
      "logits/rejected": -0.28125,
      "logps/chosen": -394.0,
      "logps/rejected": -350.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.921875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.625,
      "step": 430
    },
    {
      "epoch": 0.5288343558282208,
      "grad_norm": 39.63938191210347,
      "learning_rate": 1.2483471074380165e-06,
      "logits/chosen": 0.01806640625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -386.0,
      "logps/rejected": -346.0,
      "loss": 0.4684,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.12255859375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.3125,
      "step": 431
    },
    {
      "epoch": 0.5300613496932516,
      "grad_norm": 35.90106246867024,
      "learning_rate": 1.2477272727272726e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": 0.035888671875,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.4356,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.78515625,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.9375,
      "step": 432
    },
    {
      "epoch": 0.5312883435582823,
      "grad_norm": 43.12860297142818,
      "learning_rate": 1.247107438016529e-06,
      "logits/chosen": 0.08740234375,
      "logits/rejected": -0.1328125,
      "logps/chosen": -388.0,
      "logps/rejected": -320.0,
      "loss": 0.4803,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.12353515625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.5625,
      "step": 433
    },
    {
      "epoch": 0.5325153374233129,
      "grad_norm": 37.083326074229944,
      "learning_rate": 1.2464876033057851e-06,
      "logits/chosen": -0.0869140625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -390.0,
      "logps/rejected": -356.0,
      "loss": 0.3206,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.07666015625,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.28125,
      "step": 434
    },
    {
      "epoch": 0.5337423312883436,
      "grad_norm": 37.88064967580844,
      "learning_rate": 1.2458677685950415e-06,
      "logits/chosen": -0.01458740234375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -394.0,
      "logps/rejected": -362.0,
      "loss": 0.3939,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.51953125,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.65625,
      "step": 435
    },
    {
      "epoch": 0.5349693251533743,
      "grad_norm": 30.102357096787667,
      "learning_rate": 1.2452479338842976e-06,
      "logits/chosen": -0.0250244140625,
      "logits/rejected": -0.2255859375,
      "logps/chosen": -422.0,
      "logps/rejected": -358.0,
      "loss": 0.2562,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 7.75,
      "rewards/rejected": -6.625,
      "step": 436
    },
    {
      "epoch": 0.5361963190184049,
      "grad_norm": 25.03840429478704,
      "learning_rate": 1.2446280991735538e-06,
      "logits/chosen": -0.080078125,
      "logits/rejected": -0.25,
      "logps/chosen": -388.0,
      "logps/rejected": -350.0,
      "loss": 0.241,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.875,
      "step": 437
    },
    {
      "epoch": 0.5374233128834356,
      "grad_norm": 36.15225304248156,
      "learning_rate": 1.24400826446281e-06,
      "logits/chosen": 0.02001953125,
      "logits/rejected": -0.06982421875,
      "logps/chosen": -366.0,
      "logps/rejected": -334.0,
      "loss": 0.5276,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0250244140625,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.6875,
      "step": 438
    },
    {
      "epoch": 0.5386503067484663,
      "grad_norm": 34.58997013752512,
      "learning_rate": 1.243388429752066e-06,
      "logits/chosen": 0.0625,
      "logits/rejected": -0.126953125,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.3671,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0703125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.71875,
      "step": 439
    },
    {
      "epoch": 0.5398773006134969,
      "grad_norm": 43.86154514789445,
      "learning_rate": 1.2427685950413224e-06,
      "logits/chosen": 0.0218505859375,
      "logits/rejected": -0.0751953125,
      "logps/chosen": -392.0,
      "logps/rejected": -344.0,
      "loss": 0.4825,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.515625,
      "rewards/margins": 5.4375,
      "rewards/rejected": -5.96875,
      "step": 440
    },
    {
      "epoch": 0.5411042944785276,
      "grad_norm": 32.02976017157544,
      "learning_rate": 1.2421487603305785e-06,
      "logits/chosen": 0.09619140625,
      "logits/rejected": -0.0108642578125,
      "logps/chosen": -340.0,
      "logps/rejected": -314.0,
      "loss": 0.4408,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.0869140625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -6.21875,
      "step": 441
    },
    {
      "epoch": 0.5423312883435583,
      "grad_norm": 31.122024927447573,
      "learning_rate": 1.2415289256198347e-06,
      "logits/chosen": -0.064453125,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -390.0,
      "logps/rejected": -352.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.51171875,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.0625,
      "step": 442
    },
    {
      "epoch": 0.5435582822085889,
      "grad_norm": 35.35206800809496,
      "learning_rate": 1.240909090909091e-06,
      "logits/chosen": -0.024169921875,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -372.0,
      "logps/rejected": -332.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.69140625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.125,
      "step": 443
    },
    {
      "epoch": 0.5447852760736196,
      "grad_norm": 35.692369145811284,
      "learning_rate": 1.2402892561983472e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.298828125,
      "logps/chosen": -454.0,
      "logps/rejected": -382.0,
      "loss": 0.3655,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2890625,
      "rewards/margins": 8.0,
      "rewards/rejected": -6.71875,
      "step": 444
    },
    {
      "epoch": 0.5460122699386503,
      "grad_norm": 32.62912831625137,
      "learning_rate": 1.2396694214876033e-06,
      "logits/chosen": -0.016357421875,
      "logits/rejected": -0.208984375,
      "logps/chosen": -358.0,
      "logps/rejected": -314.0,
      "loss": 0.4199,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.66796875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.5625,
      "step": 445
    },
    {
      "epoch": 0.5472392638036809,
      "grad_norm": 25.78605234898617,
      "learning_rate": 1.2390495867768597e-06,
      "logits/chosen": -0.040283203125,
      "logits/rejected": -0.203125,
      "logps/chosen": -438.0,
      "logps/rejected": -370.0,
      "loss": 0.2432,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.81640625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -5.65625,
      "step": 446
    },
    {
      "epoch": 0.5484662576687117,
      "grad_norm": 28.526549681854004,
      "learning_rate": 1.2384297520661158e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.23828125,
      "logps/chosen": -414.0,
      "logps/rejected": -360.0,
      "loss": 0.3087,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.359375,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.84375,
      "step": 447
    },
    {
      "epoch": 0.5496932515337424,
      "grad_norm": 33.26832004918433,
      "learning_rate": 1.237809917355372e-06,
      "logits/chosen": 0.042724609375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -346.0,
      "logps/rejected": -314.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1796875,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.84375,
      "step": 448
    },
    {
      "epoch": 0.550920245398773,
      "grad_norm": 29.824788916298026,
      "learning_rate": 1.237190082644628e-06,
      "logits/chosen": -0.0341796875,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -400.0,
      "logps/rejected": -346.0,
      "loss": 0.3203,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.234375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -6.09375,
      "step": 449
    },
    {
      "epoch": 0.5521472392638037,
      "grad_norm": 29.646416826102993,
      "learning_rate": 1.2365702479338842e-06,
      "logits/chosen": -0.1103515625,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -350.0,
      "logps/rejected": -302.0,
      "loss": 0.2642,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.66015625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.125,
      "step": 450
    },
    {
      "epoch": 0.5533742331288344,
      "grad_norm": 31.583737211583365,
      "learning_rate": 1.2359504132231406e-06,
      "logits/chosen": 0.03173828125,
      "logits/rejected": -0.1396484375,
      "logps/chosen": -348.0,
      "logps/rejected": -294.0,
      "loss": 0.3883,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.875,
      "step": 451
    },
    {
      "epoch": 0.554601226993865,
      "grad_norm": 27.43380959875228,
      "learning_rate": 1.2353305785123967e-06,
      "logits/chosen": -0.0439453125,
      "logits/rejected": -0.28125,
      "logps/chosen": -460.0,
      "logps/rejected": -342.0,
      "loss": 0.2556,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.9453125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -5.59375,
      "step": 452
    },
    {
      "epoch": 0.5558282208588957,
      "grad_norm": 24.613642812642805,
      "learning_rate": 1.2347107438016529e-06,
      "logits/chosen": -0.130859375,
      "logits/rejected": -0.33984375,
      "logps/chosen": -440.0,
      "logps/rejected": -378.0,
      "loss": 0.2702,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -6.0625,
      "step": 453
    },
    {
      "epoch": 0.5570552147239264,
      "grad_norm": 31.985648649098213,
      "learning_rate": 1.2340909090909092e-06,
      "logits/chosen": 0.046875,
      "logits/rejected": -0.0206298828125,
      "logps/chosen": -346.0,
      "logps/rejected": -320.0,
      "loss": 0.3305,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0859375,
      "rewards/margins": 6.0625,
      "rewards/rejected": -6.0,
      "step": 454
    },
    {
      "epoch": 0.558282208588957,
      "grad_norm": 26.277923249806037,
      "learning_rate": 1.2334710743801654e-06,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.201171875,
      "logps/chosen": -372.0,
      "logps/rejected": -362.0,
      "loss": 0.2299,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.5,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.09375,
      "step": 455
    },
    {
      "epoch": 0.5595092024539877,
      "grad_norm": 35.777155080443045,
      "learning_rate": 1.2328512396694215e-06,
      "logits/chosen": 0.0419921875,
      "logits/rejected": -0.0576171875,
      "logps/chosen": -318.0,
      "logps/rejected": -314.0,
      "loss": 0.4054,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.412109375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.375,
      "step": 456
    },
    {
      "epoch": 0.5607361963190184,
      "grad_norm": 33.41411882209408,
      "learning_rate": 1.2322314049586776e-06,
      "logits/chosen": 0.0126953125,
      "logits/rejected": -0.1142578125,
      "logps/chosen": -360.0,
      "logps/rejected": -330.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.63671875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.59375,
      "step": 457
    },
    {
      "epoch": 0.561963190184049,
      "grad_norm": 27.510163645115238,
      "learning_rate": 1.2316115702479338e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -376.0,
      "logps/rejected": -302.0,
      "loss": 0.2733,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.109375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.3125,
      "step": 458
    },
    {
      "epoch": 0.5631901840490797,
      "grad_norm": 32.20829218573613,
      "learning_rate": 1.2309917355371901e-06,
      "logits/chosen": -0.0172119140625,
      "logits/rejected": -0.1796875,
      "logps/chosen": -396.0,
      "logps/rejected": -356.0,
      "loss": 0.3698,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.171875,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.1875,
      "step": 459
    },
    {
      "epoch": 0.5644171779141104,
      "grad_norm": 28.463767397560563,
      "learning_rate": 1.2303719008264463e-06,
      "logits/chosen": -0.0159912109375,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -354.0,
      "logps/rejected": -342.0,
      "loss": 0.2835,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.146484375,
      "rewards/margins": 7.46875,
      "rewards/rejected": -7.3125,
      "step": 460
    },
    {
      "epoch": 0.5656441717791411,
      "grad_norm": 30.885477551125014,
      "learning_rate": 1.2297520661157024e-06,
      "logits/chosen": -0.2158203125,
      "logits/rejected": -0.330078125,
      "logps/chosen": -380.0,
      "logps/rejected": -352.0,
      "loss": 0.2845,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.1884765625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.375,
      "step": 461
    },
    {
      "epoch": 0.5668711656441717,
      "grad_norm": 44.21195903099332,
      "learning_rate": 1.2291322314049588e-06,
      "logits/chosen": 0.047607421875,
      "logits/rejected": -0.1552734375,
      "logps/chosen": -370.0,
      "logps/rejected": -310.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.1337890625,
      "rewards/margins": 5.875,
      "rewards/rejected": -6.03125,
      "step": 462
    },
    {
      "epoch": 0.5680981595092025,
      "grad_norm": 33.208183287178784,
      "learning_rate": 1.228512396694215e-06,
      "logits/chosen": -0.0849609375,
      "logits/rejected": -0.18359375,
      "logps/chosen": -370.0,
      "logps/rejected": -372.0,
      "loss": 0.3252,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.4140625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.6875,
      "step": 463
    },
    {
      "epoch": 0.5693251533742332,
      "grad_norm": 37.13310236043479,
      "learning_rate": 1.2278925619834713e-06,
      "logits/chosen": 0.05224609375,
      "logits/rejected": -0.119140625,
      "logps/chosen": -398.0,
      "logps/rejected": -334.0,
      "loss": 0.3492,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.3359375,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.59375,
      "step": 464
    },
    {
      "epoch": 0.5705521472392638,
      "grad_norm": 21.80723319666503,
      "learning_rate": 1.2272727272727274e-06,
      "logits/chosen": -0.06396484375,
      "logits/rejected": -0.302734375,
      "logps/chosen": -404.0,
      "logps/rejected": -362.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.6171875,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.625,
      "step": 465
    },
    {
      "epoch": 0.5717791411042945,
      "grad_norm": 32.116293843093004,
      "learning_rate": 1.2266528925619833e-06,
      "logits/chosen": -0.016845703125,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -424.0,
      "logps/rejected": -356.0,
      "loss": 0.3118,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.0458984375,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.46875,
      "step": 466
    },
    {
      "epoch": 0.5730061349693252,
      "grad_norm": 37.63110345633159,
      "learning_rate": 1.2260330578512397e-06,
      "logits/chosen": -0.0279541015625,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -388.0,
      "logps/rejected": -364.0,
      "loss": 0.3871,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.8046875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.40625,
      "step": 467
    },
    {
      "epoch": 0.5742331288343558,
      "grad_norm": 33.08460200253717,
      "learning_rate": 1.2254132231404958e-06,
      "logits/chosen": -0.0419921875,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -372.0,
      "logps/rejected": -358.0,
      "loss": 0.3099,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.09375,
      "step": 468
    },
    {
      "epoch": 0.5754601226993865,
      "grad_norm": 22.648898471514507,
      "learning_rate": 1.2247933884297522e-06,
      "logits/chosen": 0.0230712890625,
      "logits/rejected": -0.15234375,
      "logps/chosen": -390.0,
      "logps/rejected": -320.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.796875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.0625,
      "step": 469
    },
    {
      "epoch": 0.5766871165644172,
      "grad_norm": 33.108021332496286,
      "learning_rate": 1.2241735537190083e-06,
      "logits/chosen": -0.010498046875,
      "logits/rejected": -0.07373046875,
      "logps/chosen": -356.0,
      "logps/rejected": -338.0,
      "loss": 0.367,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.69921875,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.8125,
      "step": 470
    },
    {
      "epoch": 0.5779141104294478,
      "grad_norm": 30.281427499200404,
      "learning_rate": 1.2235537190082645e-06,
      "logits/chosen": 0.1171875,
      "logits/rejected": -0.0458984375,
      "logps/chosen": -408.0,
      "logps/rejected": -350.0,
      "loss": 0.3469,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.328125,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.5625,
      "step": 471
    },
    {
      "epoch": 0.5791411042944785,
      "grad_norm": 30.766178072522326,
      "learning_rate": 1.2229338842975208e-06,
      "logits/chosen": 0.0654296875,
      "logits/rejected": -0.15234375,
      "logps/chosen": -414.0,
      "logps/rejected": -336.0,
      "loss": 0.341,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.71875,
      "step": 472
    },
    {
      "epoch": 0.5803680981595092,
      "grad_norm": 35.82872232913366,
      "learning_rate": 1.222314049586777e-06,
      "logits/chosen": -0.10986328125,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -406.0,
      "logps/rejected": -394.0,
      "loss": 0.3929,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.5,
      "step": 473
    },
    {
      "epoch": 0.5815950920245399,
      "grad_norm": 38.235476934077774,
      "learning_rate": 1.221694214876033e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.271484375,
      "logps/chosen": -440.0,
      "logps/rejected": -362.0,
      "loss": 0.3842,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0673828125,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.8125,
      "step": 474
    },
    {
      "epoch": 0.5828220858895705,
      "grad_norm": 28.310340512331347,
      "learning_rate": 1.2210743801652892e-06,
      "logits/chosen": -0.06689453125,
      "logits/rejected": -0.2734375,
      "logps/chosen": -434.0,
      "logps/rejected": -362.0,
      "loss": 0.2596,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.140625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.3125,
      "step": 475
    },
    {
      "epoch": 0.5840490797546012,
      "grad_norm": 31.21556050331037,
      "learning_rate": 1.2204545454545454e-06,
      "logits/chosen": 0.00927734375,
      "logits/rejected": -0.1806640625,
      "logps/chosen": -420.0,
      "logps/rejected": -360.0,
      "loss": 0.3346,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -1.078125,
      "rewards/margins": 6.53125,
      "rewards/rejected": -7.625,
      "step": 476
    },
    {
      "epoch": 0.5852760736196319,
      "grad_norm": 31.773096981894017,
      "learning_rate": 1.2198347107438017e-06,
      "logits/chosen": -0.11474609375,
      "logits/rejected": -0.255859375,
      "logps/chosen": -432.0,
      "logps/rejected": -372.0,
      "loss": 0.3232,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.1484375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.65625,
      "step": 477
    },
    {
      "epoch": 0.5865030674846625,
      "grad_norm": 30.507836762025494,
      "learning_rate": 1.2192148760330579e-06,
      "logits/chosen": -0.04150390625,
      "logits/rejected": -0.2080078125,
      "logps/chosen": -430.0,
      "logps/rejected": -368.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.024658203125,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.65625,
      "step": 478
    },
    {
      "epoch": 0.5877300613496933,
      "grad_norm": 32.261887607989564,
      "learning_rate": 1.218595041322314e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -370.0,
      "logps/rejected": -352.0,
      "loss": 0.322,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 6.625,
      "rewards/rejected": -7.78125,
      "step": 479
    },
    {
      "epoch": 0.588957055214724,
      "grad_norm": 28.188949742159306,
      "learning_rate": 1.2179752066115704e-06,
      "logits/chosen": -0.0791015625,
      "logits/rejected": -0.208984375,
      "logps/chosen": -372.0,
      "logps/rejected": -348.0,
      "loss": 0.2558,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.8125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -8.0,
      "step": 480
    },
    {
      "epoch": 0.5901840490797546,
      "grad_norm": 37.17733184183342,
      "learning_rate": 1.2173553719008265e-06,
      "logits/chosen": -0.07568359375,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -438.0,
      "logps/rejected": -398.0,
      "loss": 0.4973,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.578125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.3125,
      "step": 481
    },
    {
      "epoch": 0.5914110429447853,
      "grad_norm": 34.97683920834337,
      "learning_rate": 1.2167355371900826e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.244140625,
      "logps/chosen": -424.0,
      "logps/rejected": -352.0,
      "loss": 0.3731,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 6.875,
      "rewards/rejected": -7.125,
      "step": 482
    },
    {
      "epoch": 0.592638036809816,
      "grad_norm": 32.698615963886375,
      "learning_rate": 1.216115702479339e-06,
      "logits/chosen": 0.07763671875,
      "logits/rejected": -0.0556640625,
      "logps/chosen": -358.0,
      "logps/rejected": -330.0,
      "loss": 0.3304,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.55859375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.34375,
      "step": 483
    },
    {
      "epoch": 0.5938650306748466,
      "grad_norm": 31.73235234166197,
      "learning_rate": 1.215495867768595e-06,
      "logits/chosen": 0.00390625,
      "logits/rejected": -0.142578125,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.3937,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.1611328125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.3125,
      "step": 484
    },
    {
      "epoch": 0.5950920245398773,
      "grad_norm": 28.55956265398308,
      "learning_rate": 1.2148760330578513e-06,
      "logits/chosen": 0.023681640625,
      "logits/rejected": -0.12109375,
      "logps/chosen": -370.0,
      "logps/rejected": -324.0,
      "loss": 0.3173,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.36328125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.3125,
      "step": 485
    },
    {
      "epoch": 0.596319018404908,
      "grad_norm": 33.81986931883939,
      "learning_rate": 1.2142561983471074e-06,
      "logits/chosen": -0.0050048828125,
      "logits/rejected": -0.14453125,
      "logps/chosen": -414.0,
      "logps/rejected": -352.0,
      "loss": 0.3778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.455078125,
      "rewards/margins": 7.0,
      "rewards/rejected": -6.53125,
      "step": 486
    },
    {
      "epoch": 0.5975460122699386,
      "grad_norm": 30.028596310368773,
      "learning_rate": 1.2136363636363636e-06,
      "logits/chosen": -0.08251953125,
      "logits/rejected": -0.27734375,
      "logps/chosen": -410.0,
      "logps/rejected": -366.0,
      "loss": 0.2781,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.33984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.03125,
      "step": 487
    },
    {
      "epoch": 0.5987730061349693,
      "grad_norm": 35.71061050092003,
      "learning_rate": 1.21301652892562e-06,
      "logits/chosen": -0.0693359375,
      "logits/rejected": -0.138671875,
      "logps/chosen": -424.0,
      "logps/rejected": -370.0,
      "loss": 0.3544,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5078125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.46875,
      "step": 488
    },
    {
      "epoch": 0.6,
      "grad_norm": 26.138379124419743,
      "learning_rate": 1.212396694214876e-06,
      "logits/chosen": 0.01324462890625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -384.0,
      "logps/rejected": -340.0,
      "loss": 0.3008,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 7.875,
      "rewards/rejected": -6.9375,
      "step": 489
    },
    {
      "epoch": 0.6012269938650306,
      "grad_norm": 38.68817736099095,
      "learning_rate": 1.2117768595041322e-06,
      "logits/chosen": 0.055419921875,
      "logits/rejected": -0.1689453125,
      "logps/chosen": -432.0,
      "logps/rejected": -364.0,
      "loss": 0.404,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.263671875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.5625,
      "step": 490
    },
    {
      "epoch": 0.6024539877300613,
      "grad_norm": 30.239394074669093,
      "learning_rate": 1.2111570247933886e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -372.0,
      "logps/rejected": -328.0,
      "loss": 0.3068,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.53125,
      "step": 491
    },
    {
      "epoch": 0.603680981595092,
      "grad_norm": 33.80787533439265,
      "learning_rate": 1.2105371900826447e-06,
      "logits/chosen": -0.013427734375,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -398.0,
      "logps/rejected": -390.0,
      "loss": 0.3711,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.125,
      "step": 492
    },
    {
      "epoch": 0.6049079754601226,
      "grad_norm": 41.137878058973136,
      "learning_rate": 1.2099173553719008e-06,
      "logits/chosen": 0.14453125,
      "logits/rejected": 0.043701171875,
      "logps/chosen": -392.0,
      "logps/rejected": -340.0,
      "loss": 0.5049,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.19921875,
      "rewards/margins": 5.65625,
      "rewards/rejected": -5.4375,
      "step": 493
    },
    {
      "epoch": 0.6061349693251534,
      "grad_norm": 28.550910744803765,
      "learning_rate": 1.209297520661157e-06,
      "logits/chosen": 0.044921875,
      "logits/rejected": -0.1435546875,
      "logps/chosen": -428.0,
      "logps/rejected": -340.0,
      "loss": 0.3041,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.0625,
      "step": 494
    },
    {
      "epoch": 0.6073619631901841,
      "grad_norm": 32.18887165372733,
      "learning_rate": 1.2086776859504131e-06,
      "logits/chosen": 0.031494140625,
      "logits/rejected": -0.1064453125,
      "logps/chosen": -402.0,
      "logps/rejected": -350.0,
      "loss": 0.3067,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.0859375,
      "rewards/margins": 7.25,
      "rewards/rejected": -6.15625,
      "step": 495
    },
    {
      "epoch": 0.6085889570552148,
      "grad_norm": 34.7968597037722,
      "learning_rate": 1.2080578512396695e-06,
      "logits/chosen": -0.0025634765625,
      "logits/rejected": -0.21484375,
      "logps/chosen": -382.0,
      "logps/rejected": -322.0,
      "loss": 0.3605,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1708984375,
      "rewards/margins": 6.25,
      "rewards/rejected": -6.09375,
      "step": 496
    },
    {
      "epoch": 0.6098159509202454,
      "grad_norm": 36.914820937247434,
      "learning_rate": 1.2074380165289256e-06,
      "logits/chosen": 0.0361328125,
      "logits/rejected": -0.125,
      "logps/chosen": -404.0,
      "logps/rejected": -366.0,
      "loss": 0.4718,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1337890625,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.375,
      "step": 497
    },
    {
      "epoch": 0.6110429447852761,
      "grad_norm": 27.898760612078714,
      "learning_rate": 1.206818181818182e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.265625,
      "logps/chosen": -400.0,
      "logps/rejected": -366.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.9765625,
      "rewards/margins": 7.46875,
      "rewards/rejected": -6.5,
      "step": 498
    },
    {
      "epoch": 0.6122699386503068,
      "grad_norm": 33.514362873417944,
      "learning_rate": 1.2061983471074381e-06,
      "logits/chosen": 0.041015625,
      "logits/rejected": -0.177734375,
      "logps/chosen": -426.0,
      "logps/rejected": -350.0,
      "loss": 0.3147,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.0,
      "step": 499
    },
    {
      "epoch": 0.6134969325153374,
      "grad_norm": 30.177881260758163,
      "learning_rate": 1.2055785123966943e-06,
      "logits/chosen": 0.0224609375,
      "logits/rejected": -0.09375,
      "logps/chosen": -418.0,
      "logps/rejected": -338.0,
      "loss": 0.3575,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.28125,
      "step": 500
    },
    {
      "epoch": 0.6147239263803681,
      "grad_norm": 34.255808614960706,
      "learning_rate": 1.2049586776859504e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.171875,
      "logps/chosen": -410.0,
      "logps/rejected": -352.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.234375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.34375,
      "step": 501
    },
    {
      "epoch": 0.6159509202453988,
      "grad_norm": 28.29113723996875,
      "learning_rate": 1.2043388429752065e-06,
      "logits/chosen": -0.01165771484375,
      "logits/rejected": -0.28125,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.3467,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.6796875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.03125,
      "step": 502
    },
    {
      "epoch": 0.6171779141104294,
      "grad_norm": 29.752413229458206,
      "learning_rate": 1.2037190082644629e-06,
      "logits/chosen": 0.1904296875,
      "logits/rejected": -0.0013275146484375,
      "logps/chosen": -334.0,
      "logps/rejected": -324.0,
      "loss": 0.2996,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.28515625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.40625,
      "step": 503
    },
    {
      "epoch": 0.6184049079754601,
      "grad_norm": 24.506669229979327,
      "learning_rate": 1.203099173553719e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.234375,
      "logps/chosen": -388.0,
      "logps/rejected": -340.0,
      "loss": 0.2718,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.1025390625,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.3125,
      "step": 504
    },
    {
      "epoch": 0.6196319018404908,
      "grad_norm": 38.20086418972538,
      "learning_rate": 1.2024793388429752e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.298828125,
      "logps/chosen": -434.0,
      "logps/rejected": -406.0,
      "loss": 0.4423,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.609375,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.375,
      "step": 505
    },
    {
      "epoch": 0.6208588957055214,
      "grad_norm": 67.15816433272404,
      "learning_rate": 1.2018595041322315e-06,
      "logits/chosen": 0.06982421875,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -428.0,
      "logps/rejected": -340.0,
      "loss": 0.3369,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.0081787109375,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.03125,
      "step": 506
    },
    {
      "epoch": 0.6220858895705521,
      "grad_norm": 25.2127178538785,
      "learning_rate": 1.2012396694214877e-06,
      "logits/chosen": -0.04296875,
      "logits/rejected": -0.150390625,
      "logps/chosen": -394.0,
      "logps/rejected": -318.0,
      "loss": 0.1989,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.11474609375,
      "rewards/margins": 7.34375,
      "rewards/rejected": -7.21875,
      "step": 507
    },
    {
      "epoch": 0.6233128834355828,
      "grad_norm": 31.70942946416519,
      "learning_rate": 1.2006198347107438e-06,
      "logits/chosen": 0.14453125,
      "logits/rejected": -0.033935546875,
      "logps/chosen": -396.0,
      "logps/rejected": -320.0,
      "loss": 0.3214,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1494140625,
      "rewards/margins": 6.40625,
      "rewards/rejected": -6.5625,
      "step": 508
    },
    {
      "epoch": 0.6245398773006134,
      "grad_norm": 39.83330773441419,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -0.0225830078125,
      "logits/rejected": -0.2578125,
      "logps/chosen": -474.0,
      "logps/rejected": -396.0,
      "loss": 0.569,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.21875,
      "step": 509
    },
    {
      "epoch": 0.6257668711656442,
      "grad_norm": 29.863730506340346,
      "learning_rate": 1.199380165289256e-06,
      "logits/chosen": -0.11181640625,
      "logits/rejected": -0.291015625,
      "logps/chosen": -368.0,
      "logps/rejected": -314.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.6875,
      "step": 510
    },
    {
      "epoch": 0.6269938650306749,
      "grad_norm": 35.188221718693796,
      "learning_rate": 1.1987603305785124e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.259765625,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0859375,
      "rewards/margins": 7.3125,
      "rewards/rejected": -7.21875,
      "step": 511
    },
    {
      "epoch": 0.6282208588957056,
      "grad_norm": 35.23659404136466,
      "learning_rate": 1.1981404958677686e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.0654296875,
      "logps/chosen": -388.0,
      "logps/rejected": -398.0,
      "loss": 0.4354,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.388671875,
      "rewards/margins": 5.90625,
      "rewards/rejected": -6.28125,
      "step": 512
    },
    {
      "epoch": 0.6294478527607362,
      "grad_norm": 33.94396537230905,
      "learning_rate": 1.1975206611570247e-06,
      "logits/chosen": 0.07958984375,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -384.0,
      "logps/rejected": -326.0,
      "loss": 0.4151,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1328125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.8125,
      "step": 513
    },
    {
      "epoch": 0.6306748466257669,
      "grad_norm": 27.33280439239978,
      "learning_rate": 1.196900826446281e-06,
      "logits/chosen": 0.09765625,
      "logits/rejected": -0.0537109375,
      "logps/chosen": -340.0,
      "logps/rejected": -302.0,
      "loss": 0.2677,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -1.015625,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.53125,
      "step": 514
    },
    {
      "epoch": 0.6319018404907976,
      "grad_norm": 32.16724360242484,
      "learning_rate": 1.1962809917355372e-06,
      "logits/chosen": 0.07275390625,
      "logits/rejected": -0.07763671875,
      "logps/chosen": -358.0,
      "logps/rejected": -362.0,
      "loss": 0.3029,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -1.0703125,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.5625,
      "step": 515
    },
    {
      "epoch": 0.6331288343558282,
      "grad_norm": 26.23429258203521,
      "learning_rate": 1.1956611570247934e-06,
      "logits/chosen": 0.0279541015625,
      "logits/rejected": -0.07861328125,
      "logps/chosen": -378.0,
      "logps/rejected": -346.0,
      "loss": 0.2424,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.8828125,
      "rewards/margins": 6.65625,
      "rewards/rejected": -7.53125,
      "step": 516
    },
    {
      "epoch": 0.6343558282208589,
      "grad_norm": 37.00771401511282,
      "learning_rate": 1.1950413223140497e-06,
      "logits/chosen": 0.0703125,
      "logits/rejected": -0.08642578125,
      "logps/chosen": -400.0,
      "logps/rejected": -352.0,
      "loss": 0.4735,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.48828125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.78125,
      "step": 517
    },
    {
      "epoch": 0.6355828220858896,
      "grad_norm": 28.05157133110184,
      "learning_rate": 1.1944214876033059e-06,
      "logits/chosen": -0.04541015625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -424.0,
      "logps/rejected": -360.0,
      "loss": 0.2475,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.21875,
      "step": 518
    },
    {
      "epoch": 0.6368098159509202,
      "grad_norm": 38.286039724245256,
      "learning_rate": 1.193801652892562e-06,
      "logits/chosen": -0.026123046875,
      "logits/rejected": -0.1630859375,
      "logps/chosen": -396.0,
      "logps/rejected": -354.0,
      "loss": 0.3562,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.60546875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.53125,
      "step": 519
    },
    {
      "epoch": 0.6380368098159509,
      "grad_norm": 34.57537616564543,
      "learning_rate": 1.1931818181818181e-06,
      "logits/chosen": -0.0517578125,
      "logits/rejected": -0.279296875,
      "logps/chosen": -380.0,
      "logps/rejected": -350.0,
      "loss": 0.37,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 7.71875,
      "rewards/rejected": -8.0,
      "step": 520
    },
    {
      "epoch": 0.6392638036809816,
      "grad_norm": 29.132838683315903,
      "learning_rate": 1.1925619834710743e-06,
      "logits/chosen": 0.158203125,
      "logits/rejected": 0.01312255859375,
      "logps/chosen": -428.0,
      "logps/rejected": -378.0,
      "loss": 0.2975,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.01263427734375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.8125,
      "step": 521
    },
    {
      "epoch": 0.6404907975460122,
      "grad_norm": 30.38859952385869,
      "learning_rate": 1.1919421487603306e-06,
      "logits/chosen": -0.0546875,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -438.0,
      "logps/rejected": -374.0,
      "loss": 0.3466,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.439453125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -8.625,
      "step": 522
    },
    {
      "epoch": 0.6417177914110429,
      "grad_norm": 36.440630501558566,
      "learning_rate": 1.1913223140495868e-06,
      "logits/chosen": 0.0238037109375,
      "logits/rejected": -0.12890625,
      "logps/chosen": -380.0,
      "logps/rejected": -346.0,
      "loss": 0.4467,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.09375,
      "rewards/margins": 6.3125,
      "rewards/rejected": -7.40625,
      "step": 523
    },
    {
      "epoch": 0.6429447852760736,
      "grad_norm": 39.231543678462224,
      "learning_rate": 1.1907024793388431e-06,
      "logits/chosen": 0.05615234375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -440.0,
      "logps/rejected": -380.0,
      "loss": 0.4136,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -1.1953125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -7.59375,
      "step": 524
    },
    {
      "epoch": 0.6441717791411042,
      "grad_norm": 47.6608831234207,
      "learning_rate": 1.1900826446280993e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": 0.02587890625,
      "logps/chosen": -406.0,
      "logps/rejected": -382.0,
      "loss": 0.5676,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -1.3125,
      "rewards/margins": 5.53125,
      "rewards/rejected": -6.84375,
      "step": 525
    },
    {
      "epoch": 0.645398773006135,
      "grad_norm": 31.7891594700196,
      "learning_rate": 1.1894628099173554e-06,
      "logits/chosen": 0.0458984375,
      "logits/rejected": -0.10107421875,
      "logps/chosen": -402.0,
      "logps/rejected": -356.0,
      "loss": 0.3316,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.6171875,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.84375,
      "step": 526
    },
    {
      "epoch": 0.6466257668711657,
      "grad_norm": 32.21728261594982,
      "learning_rate": 1.1888429752066118e-06,
      "logits/chosen": 0.1201171875,
      "logits/rejected": -0.103515625,
      "logps/chosen": -438.0,
      "logps/rejected": -348.0,
      "loss": 0.3529,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.7265625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.75,
      "step": 527
    },
    {
      "epoch": 0.6478527607361964,
      "grad_norm": 26.60182420517136,
      "learning_rate": 1.1882231404958677e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.220703125,
      "logps/chosen": -368.0,
      "logps/rejected": -344.0,
      "loss": 0.2928,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.671875,
      "rewards/margins": 7.375,
      "rewards/rejected": -8.0625,
      "step": 528
    },
    {
      "epoch": 0.649079754601227,
      "grad_norm": 32.109755070015275,
      "learning_rate": 1.187603305785124e-06,
      "logits/chosen": 0.030029296875,
      "logits/rejected": -0.14453125,
      "logps/chosen": -352.0,
      "logps/rejected": -352.0,
      "loss": 0.418,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.3984375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.34375,
      "step": 529
    },
    {
      "epoch": 0.6503067484662577,
      "grad_norm": 30.441331665820965,
      "learning_rate": 1.1869834710743802e-06,
      "logits/chosen": 0.07373046875,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -370.0,
      "logps/rejected": -366.0,
      "loss": 0.2962,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.1103515625,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.59375,
      "step": 530
    },
    {
      "epoch": 0.6515337423312884,
      "grad_norm": 25.97427159310545,
      "learning_rate": 1.1863636363636363e-06,
      "logits/chosen": -0.0005950927734375,
      "logits/rejected": -0.1767578125,
      "logps/chosen": -394.0,
      "logps/rejected": -338.0,
      "loss": 0.3013,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.2001953125,
      "rewards/margins": 7.0625,
      "rewards/rejected": -7.25,
      "step": 531
    },
    {
      "epoch": 0.652760736196319,
      "grad_norm": 29.45586726470446,
      "learning_rate": 1.1857438016528927e-06,
      "logits/chosen": 0.12060546875,
      "logits/rejected": -0.01611328125,
      "logps/chosen": -394.0,
      "logps/rejected": -342.0,
      "loss": 0.3256,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.1650390625,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.71875,
      "step": 532
    },
    {
      "epoch": 0.6539877300613497,
      "grad_norm": 31.510396061786174,
      "learning_rate": 1.1851239669421488e-06,
      "logits/chosen": -0.0634765625,
      "logits/rejected": -0.265625,
      "logps/chosen": -414.0,
      "logps/rejected": -360.0,
      "loss": 0.322,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.09375,
      "step": 533
    },
    {
      "epoch": 0.6552147239263804,
      "grad_norm": 29.028681159770606,
      "learning_rate": 1.184504132231405e-06,
      "logits/chosen": 0.0908203125,
      "logits/rejected": -0.02685546875,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.3243,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.0341796875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -6.375,
      "step": 534
    },
    {
      "epoch": 0.656441717791411,
      "grad_norm": 33.385986566673,
      "learning_rate": 1.1838842975206613e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.125,
      "logps/chosen": -378.0,
      "logps/rejected": -348.0,
      "loss": 0.3502,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7421875,
      "rewards/margins": 6.40625,
      "rewards/rejected": -7.125,
      "step": 535
    },
    {
      "epoch": 0.6576687116564417,
      "grad_norm": 27.982276879832543,
      "learning_rate": 1.1832644628099175e-06,
      "logits/chosen": 0.0201416015625,
      "logits/rejected": -0.07568359375,
      "logps/chosen": -394.0,
      "logps/rejected": -330.0,
      "loss": 0.2519,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.6015625,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.78125,
      "step": 536
    },
    {
      "epoch": 0.6588957055214724,
      "grad_norm": 40.68442319965527,
      "learning_rate": 1.1826446280991736e-06,
      "logits/chosen": 0.06201171875,
      "logits/rejected": -0.0927734375,
      "logps/chosen": -386.0,
      "logps/rejected": -318.0,
      "loss": 0.5685,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 5.28125,
      "rewards/rejected": -5.78125,
      "step": 537
    },
    {
      "epoch": 0.660122699386503,
      "grad_norm": 35.32326255751808,
      "learning_rate": 1.1820247933884297e-06,
      "logits/chosen": 0.068359375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -402.0,
      "logps/rejected": -360.0,
      "loss": 0.4164,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.78515625,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.96875,
      "step": 538
    },
    {
      "epoch": 0.6613496932515337,
      "grad_norm": 32.525023600229474,
      "learning_rate": 1.1814049586776859e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.04541015625,
      "logps/chosen": -400.0,
      "logps/rejected": -346.0,
      "loss": 0.4519,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 5.84375,
      "rewards/rejected": -5.03125,
      "step": 539
    },
    {
      "epoch": 0.6625766871165644,
      "grad_norm": 35.156155067546834,
      "learning_rate": 1.1807851239669422e-06,
      "logits/chosen": 0.0986328125,
      "logits/rejected": -0.06787109375,
      "logps/chosen": -396.0,
      "logps/rejected": -332.0,
      "loss": 0.432,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.9375,
      "step": 540
    },
    {
      "epoch": 0.6638036809815951,
      "grad_norm": 45.099896848619814,
      "learning_rate": 1.1801652892561984e-06,
      "logits/chosen": 0.08447265625,
      "logits/rejected": -0.0064697265625,
      "logps/chosen": -390.0,
      "logps/rejected": -354.0,
      "loss": 0.3808,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.279296875,
      "rewards/margins": 5.84375,
      "rewards/rejected": -5.5625,
      "step": 541
    },
    {
      "epoch": 0.6650306748466258,
      "grad_norm": 30.118145811004524,
      "learning_rate": 1.1795454545454545e-06,
      "logits/chosen": 0.031494140625,
      "logits/rejected": -0.053466796875,
      "logps/chosen": -356.0,
      "logps/rejected": -320.0,
      "loss": 0.3659,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -5.15625,
      "step": 542
    },
    {
      "epoch": 0.6662576687116565,
      "grad_norm": 32.83770736751292,
      "learning_rate": 1.1789256198347109e-06,
      "logits/chosen": -0.02294921875,
      "logits/rejected": -0.201171875,
      "logps/chosen": -450.0,
      "logps/rejected": -354.0,
      "loss": 0.3396,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.5078125,
      "rewards/margins": 6.875,
      "rewards/rejected": -5.375,
      "step": 543
    },
    {
      "epoch": 0.6674846625766871,
      "grad_norm": 35.490916209773005,
      "learning_rate": 1.178305785123967e-06,
      "logits/chosen": 0.007598876953125,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -378.0,
      "logps/rejected": -318.0,
      "loss": 0.434,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.376953125,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.78125,
      "step": 544
    },
    {
      "epoch": 0.6687116564417178,
      "grad_norm": 33.60719342971152,
      "learning_rate": 1.1776859504132231e-06,
      "logits/chosen": -0.054443359375,
      "logits/rejected": -0.21875,
      "logps/chosen": -458.0,
      "logps/rejected": -368.0,
      "loss": 0.3855,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.94140625,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.25,
      "step": 545
    },
    {
      "epoch": 0.6699386503067485,
      "grad_norm": 33.972456048977826,
      "learning_rate": 1.1770661157024793e-06,
      "logits/chosen": 0.0693359375,
      "logits/rejected": -0.028564453125,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.3815,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.6953125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.71875,
      "step": 546
    },
    {
      "epoch": 0.6711656441717792,
      "grad_norm": 28.303815223441166,
      "learning_rate": 1.1764462809917354e-06,
      "logits/chosen": 0.0133056640625,
      "logits/rejected": -0.166015625,
      "logps/chosen": -386.0,
      "logps/rejected": -312.0,
      "loss": 0.2992,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.88671875,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.53125,
      "step": 547
    },
    {
      "epoch": 0.6723926380368098,
      "grad_norm": 38.31133772907115,
      "learning_rate": 1.1758264462809918e-06,
      "logits/chosen": -0.06494140625,
      "logits/rejected": -0.265625,
      "logps/chosen": -402.0,
      "logps/rejected": -358.0,
      "loss": 0.3532,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.015625,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.125,
      "step": 548
    },
    {
      "epoch": 0.6736196319018405,
      "grad_norm": 33.292222723297044,
      "learning_rate": 1.175206611570248e-06,
      "logits/chosen": 0.047607421875,
      "logits/rejected": -0.1572265625,
      "logps/chosen": -414.0,
      "logps/rejected": -344.0,
      "loss": 0.3292,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.5625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -5.40625,
      "step": 549
    },
    {
      "epoch": 0.6748466257668712,
      "grad_norm": 30.807870745155437,
      "learning_rate": 1.174586776859504e-06,
      "logits/chosen": -0.024658203125,
      "logits/rejected": -0.08642578125,
      "logps/chosen": -402.0,
      "logps/rejected": -330.0,
      "loss": 0.3926,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12890625,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.5625,
      "step": 550
    },
    {
      "epoch": 0.6760736196319018,
      "grad_norm": 28.384977078060157,
      "learning_rate": 1.1739669421487604e-06,
      "logits/chosen": 0.06787109375,
      "logits/rejected": -0.04541015625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 6.75,
      "rewards/rejected": -5.53125,
      "step": 551
    },
    {
      "epoch": 0.6773006134969325,
      "grad_norm": 29.63861597355486,
      "learning_rate": 1.1733471074380166e-06,
      "logits/chosen": -0.022705078125,
      "logits/rejected": -0.158203125,
      "logps/chosen": -416.0,
      "logps/rejected": -360.0,
      "loss": 0.3331,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.71875,
      "rewards/margins": 6.25,
      "rewards/rejected": -5.53125,
      "step": 552
    },
    {
      "epoch": 0.6785276073619632,
      "grad_norm": 32.698792932077886,
      "learning_rate": 1.172727272727273e-06,
      "logits/chosen": 0.01806640625,
      "logits/rejected": -0.1259765625,
      "logps/chosen": -366.0,
      "logps/rejected": -342.0,
      "loss": 0.4129,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.1474609375,
      "rewards/margins": 5.625,
      "rewards/rejected": -5.78125,
      "step": 553
    },
    {
      "epoch": 0.6797546012269938,
      "grad_norm": 32.72020008420759,
      "learning_rate": 1.172107438016529e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.1171875,
      "logps/chosen": -434.0,
      "logps/rejected": -334.0,
      "loss": 0.3464,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.83984375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -5.59375,
      "step": 554
    },
    {
      "epoch": 0.6809815950920245,
      "grad_norm": 30.608371906092913,
      "learning_rate": 1.171487603305785e-06,
      "logits/chosen": 0.2021484375,
      "logits/rejected": 0.09130859375,
      "logps/chosen": -364.0,
      "logps/rejected": -318.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 6.34375,
      "rewards/rejected": -6.0,
      "step": 555
    },
    {
      "epoch": 0.6822085889570552,
      "grad_norm": 26.18493705591653,
      "learning_rate": 1.1708677685950413e-06,
      "logits/chosen": 0.053466796875,
      "logits/rejected": -0.162109375,
      "logps/chosen": -424.0,
      "logps/rejected": -360.0,
      "loss": 0.2487,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.96875,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.59375,
      "step": 556
    },
    {
      "epoch": 0.6834355828220859,
      "grad_norm": 27.830299010779843,
      "learning_rate": 1.1702479338842975e-06,
      "logits/chosen": 0.05029296875,
      "logits/rejected": -0.16796875,
      "logps/chosen": -398.0,
      "logps/rejected": -346.0,
      "loss": 0.3343,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.53515625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.15625,
      "step": 557
    },
    {
      "epoch": 0.6846625766871166,
      "grad_norm": 25.854408165941038,
      "learning_rate": 1.1696280991735538e-06,
      "logits/chosen": 0.087890625,
      "logits/rejected": -0.078125,
      "logps/chosen": -378.0,
      "logps/rejected": -340.0,
      "loss": 0.2782,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.3125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.25,
      "step": 558
    },
    {
      "epoch": 0.6858895705521473,
      "grad_norm": 33.3339739767785,
      "learning_rate": 1.16900826446281e-06,
      "logits/chosen": -0.072265625,
      "logits/rejected": -0.1591796875,
      "logps/chosen": -388.0,
      "logps/rejected": -354.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.78125,
      "step": 559
    },
    {
      "epoch": 0.6871165644171779,
      "grad_norm": 21.74474553758103,
      "learning_rate": 1.1683884297520661e-06,
      "logits/chosen": -0.1669921875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -394.0,
      "logps/rejected": -344.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.6875,
      "step": 560
    },
    {
      "epoch": 0.6883435582822086,
      "grad_norm": 24.604934270788146,
      "learning_rate": 1.1677685950413225e-06,
      "logits/chosen": 0.00164031982421875,
      "logits/rejected": -0.177734375,
      "logps/chosen": -360.0,
      "logps/rejected": -314.0,
      "loss": 0.2132,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.171875,
      "rewards/margins": 7.25,
      "rewards/rejected": -7.09375,
      "step": 561
    },
    {
      "epoch": 0.6895705521472393,
      "grad_norm": 25.33619721263671,
      "learning_rate": 1.1671487603305786e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": -0.064453125,
      "logps/chosen": -382.0,
      "logps/rejected": -336.0,
      "loss": 0.2353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.55078125,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.3125,
      "step": 562
    },
    {
      "epoch": 0.69079754601227,
      "grad_norm": 28.2123146430669,
      "learning_rate": 1.1665289256198348e-06,
      "logits/chosen": -0.055908203125,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -398.0,
      "logps/rejected": -386.0,
      "loss": 0.3157,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.3125,
      "step": 563
    },
    {
      "epoch": 0.6920245398773006,
      "grad_norm": 24.610651917154996,
      "learning_rate": 1.1659090909090909e-06,
      "logits/chosen": -0.040283203125,
      "logits/rejected": -0.1552734375,
      "logps/chosen": -396.0,
      "logps/rejected": -350.0,
      "loss": 0.3066,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.21875,
      "rewards/margins": 6.625,
      "rewards/rejected": -6.84375,
      "step": 564
    },
    {
      "epoch": 0.6932515337423313,
      "grad_norm": 30.241252760906917,
      "learning_rate": 1.165289256198347e-06,
      "logits/chosen": 0.0081787109375,
      "logits/rejected": -0.1796875,
      "logps/chosen": -404.0,
      "logps/rejected": -354.0,
      "loss": 0.4933,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.396484375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.40625,
      "step": 565
    },
    {
      "epoch": 0.694478527607362,
      "grad_norm": 30.063355062368128,
      "learning_rate": 1.1646694214876034e-06,
      "logits/chosen": 0.004852294921875,
      "logits/rejected": -0.08544921875,
      "logps/chosen": -420.0,
      "logps/rejected": -348.0,
      "loss": 0.2711,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.765625,
      "rewards/margins": 7.3125,
      "rewards/rejected": -6.53125,
      "step": 566
    },
    {
      "epoch": 0.6957055214723926,
      "grad_norm": 23.27581640569879,
      "learning_rate": 1.1640495867768595e-06,
      "logits/chosen": 0.11865234375,
      "logits/rejected": -0.0096435546875,
      "logps/chosen": -322.0,
      "logps/rejected": -288.0,
      "loss": 0.2144,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.30859375,
      "rewards/margins": 6.03125,
      "rewards/rejected": -6.34375,
      "step": 567
    },
    {
      "epoch": 0.6969325153374233,
      "grad_norm": 30.55160170642105,
      "learning_rate": 1.1634297520661157e-06,
      "logits/chosen": -0.0390625,
      "logits/rejected": -0.0869140625,
      "logps/chosen": -388.0,
      "logps/rejected": -334.0,
      "loss": 0.3414,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.357421875,
      "rewards/margins": 6.71875,
      "rewards/rejected": -6.375,
      "step": 568
    },
    {
      "epoch": 0.698159509202454,
      "grad_norm": 29.293298679377934,
      "learning_rate": 1.162809917355372e-06,
      "logits/chosen": 0.04736328125,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -388.0,
      "logps/rejected": -316.0,
      "loss": 0.2723,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0361328125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.3125,
      "step": 569
    },
    {
      "epoch": 0.6993865030674846,
      "grad_norm": 29.217886094805987,
      "learning_rate": 1.1621900826446282e-06,
      "logits/chosen": -0.1298828125,
      "logits/rejected": -0.2490234375,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2078,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.9375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.46875,
      "step": 570
    },
    {
      "epoch": 0.7006134969325153,
      "grad_norm": 29.496729411168463,
      "learning_rate": 1.1615702479338843e-06,
      "logits/chosen": 0.0179443359375,
      "logits/rejected": -0.10205078125,
      "logps/chosen": -332.0,
      "logps/rejected": -314.0,
      "loss": 0.3188,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0294189453125,
      "rewards/margins": 5.875,
      "rewards/rejected": -5.84375,
      "step": 571
    },
    {
      "epoch": 0.701840490797546,
      "grad_norm": 28.740254642158334,
      "learning_rate": 1.1609504132231407e-06,
      "logits/chosen": 0.03173828125,
      "logits/rejected": -0.11669921875,
      "logps/chosen": -388.0,
      "logps/rejected": -378.0,
      "loss": 0.2732,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.1484375,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.25,
      "step": 572
    },
    {
      "epoch": 0.7030674846625767,
      "grad_norm": 35.19506105279461,
      "learning_rate": 1.1603305785123966e-06,
      "logits/chosen": 0.078125,
      "logits/rejected": -0.050048828125,
      "logps/chosen": -394.0,
      "logps/rejected": -346.0,
      "loss": 0.5289,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.150390625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.59375,
      "step": 573
    },
    {
      "epoch": 0.7042944785276074,
      "grad_norm": 30.319920980400575,
      "learning_rate": 1.159710743801653e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -328.0,
      "logps/rejected": -320.0,
      "loss": 0.34,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.28125,
      "rewards/margins": 6.78125,
      "rewards/rejected": -6.5,
      "step": 574
    },
    {
      "epoch": 0.7055214723926381,
      "grad_norm": 29.615184087034994,
      "learning_rate": 1.159090909090909e-06,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.2734375,
      "logps/chosen": -412.0,
      "logps/rejected": -388.0,
      "loss": 0.2626,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.03125,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.34375,
      "step": 575
    },
    {
      "epoch": 0.7067484662576687,
      "grad_norm": 22.847983936498952,
      "learning_rate": 1.1584710743801652e-06,
      "logits/chosen": 0.03271484375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -386.0,
      "logps/rejected": -344.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 1.0625,
      "rewards/margins": 8.5,
      "rewards/rejected": -7.46875,
      "step": 576
    },
    {
      "epoch": 0.7079754601226994,
      "grad_norm": 33.54348915562207,
      "learning_rate": 1.1578512396694216e-06,
      "logits/chosen": 0.08935546875,
      "logits/rejected": -0.0296630859375,
      "logps/chosen": -368.0,
      "logps/rejected": -334.0,
      "loss": 0.4071,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.828125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.34375,
      "step": 577
    },
    {
      "epoch": 0.7092024539877301,
      "grad_norm": 31.17463611360663,
      "learning_rate": 1.1572314049586777e-06,
      "logits/chosen": -0.06591796875,
      "logits/rejected": -0.208984375,
      "logps/chosen": -420.0,
      "logps/rejected": -354.0,
      "loss": 0.3672,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.921875,
      "rewards/margins": 8.6875,
      "rewards/rejected": -7.75,
      "step": 578
    },
    {
      "epoch": 0.7104294478527607,
      "grad_norm": 37.58239380773895,
      "learning_rate": 1.1566115702479339e-06,
      "logits/chosen": -0.025390625,
      "logits/rejected": -0.1181640625,
      "logps/chosen": -440.0,
      "logps/rejected": -352.0,
      "loss": 0.4655,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.28125,
      "rewards/rejected": -7.0,
      "step": 579
    },
    {
      "epoch": 0.7116564417177914,
      "grad_norm": 34.56148267033386,
      "learning_rate": 1.1559917355371902e-06,
      "logits/chosen": 0.0673828125,
      "logits/rejected": -0.10791015625,
      "logps/chosen": -390.0,
      "logps/rejected": -374.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.21484375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -8.3125,
      "step": 580
    },
    {
      "epoch": 0.7128834355828221,
      "grad_norm": 26.418081399921064,
      "learning_rate": 1.1553719008264464e-06,
      "logits/chosen": -0.0498046875,
      "logits/rejected": -0.244140625,
      "logps/chosen": -380.0,
      "logps/rejected": -338.0,
      "loss": 0.2267,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -8.0625,
      "step": 581
    },
    {
      "epoch": 0.7141104294478527,
      "grad_norm": 30.496748663730113,
      "learning_rate": 1.1547520661157025e-06,
      "logits/chosen": 0.197265625,
      "logits/rejected": -0.02587890625,
      "logps/chosen": -402.0,
      "logps/rejected": -350.0,
      "loss": 0.3169,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.01220703125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -7.0,
      "step": 582
    },
    {
      "epoch": 0.7153374233128834,
      "grad_norm": 31.96559757214927,
      "learning_rate": 1.1541322314049586e-06,
      "logits/chosen": -0.02880859375,
      "logits/rejected": -0.1748046875,
      "logps/chosen": -392.0,
      "logps/rejected": -364.0,
      "loss": 0.3362,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.3671875,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.125,
      "step": 583
    },
    {
      "epoch": 0.7165644171779141,
      "grad_norm": 36.407577582473756,
      "learning_rate": 1.1535123966942148e-06,
      "logits/chosen": 0.08154296875,
      "logits/rejected": 0.0218505859375,
      "logps/chosen": -380.0,
      "logps/rejected": -392.0,
      "loss": 0.431,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 5.8125,
      "rewards/rejected": -6.375,
      "step": 584
    },
    {
      "epoch": 0.7177914110429447,
      "grad_norm": 41.799396220975545,
      "learning_rate": 1.1528925619834711e-06,
      "logits/chosen": 0.1796875,
      "logits/rejected": 0.0634765625,
      "logps/chosen": -410.0,
      "logps/rejected": -344.0,
      "loss": 0.5833,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": 0.2255859375,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.21875,
      "step": 585
    },
    {
      "epoch": 0.7190184049079754,
      "grad_norm": 32.85377702391839,
      "learning_rate": 1.1522727272727273e-06,
      "logits/chosen": 0.035888671875,
      "logits/rejected": -0.07373046875,
      "logps/chosen": -356.0,
      "logps/rejected": -324.0,
      "loss": 0.3732,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.00927734375,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.90625,
      "step": 586
    },
    {
      "epoch": 0.7202453987730061,
      "grad_norm": 28.585914695021586,
      "learning_rate": 1.1516528925619836e-06,
      "logits/chosen": 0.01116943359375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -342.0,
      "logps/rejected": -332.0,
      "loss": 0.3594,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.08935546875,
      "rewards/margins": 7.0625,
      "rewards/rejected": -6.96875,
      "step": 587
    },
    {
      "epoch": 0.7214723926380369,
      "grad_norm": 24.10336561473994,
      "learning_rate": 1.1510330578512398e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.1826171875,
      "logps/chosen": -388.0,
      "logps/rejected": -342.0,
      "loss": 0.2892,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.2109375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -6.875,
      "step": 588
    },
    {
      "epoch": 0.7226993865030675,
      "grad_norm": 28.27312313115512,
      "learning_rate": 1.150413223140496e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.2041015625,
      "logps/chosen": -372.0,
      "logps/rejected": -354.0,
      "loss": 0.3598,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.37890625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.65625,
      "step": 589
    },
    {
      "epoch": 0.7239263803680982,
      "grad_norm": 30.343626428171632,
      "learning_rate": 1.149793388429752e-06,
      "logits/chosen": 0.00927734375,
      "logits/rejected": -0.103515625,
      "logps/chosen": -374.0,
      "logps/rejected": -330.0,
      "loss": 0.3324,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0986328125,
      "rewards/margins": 6.875,
      "rewards/rejected": -6.78125,
      "step": 590
    },
    {
      "epoch": 0.7251533742331289,
      "grad_norm": 36.86660943740023,
      "learning_rate": 1.1491735537190082e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.373046875,
      "logps/chosen": -414.0,
      "logps/rejected": -354.0,
      "loss": 0.28,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.71484375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.3125,
      "step": 591
    },
    {
      "epoch": 0.7263803680981595,
      "grad_norm": 27.76314433981488,
      "learning_rate": 1.1485537190082645e-06,
      "logits/chosen": -0.02490234375,
      "logits/rejected": -0.2138671875,
      "logps/chosen": -378.0,
      "logps/rejected": -328.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.78125,
      "step": 592
    },
    {
      "epoch": 0.7276073619631902,
      "grad_norm": 36.82046387528082,
      "learning_rate": 1.1479338842975207e-06,
      "logits/chosen": 0.022705078125,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -454.0,
      "logps/rejected": -392.0,
      "loss": 0.4117,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.76953125,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.09375,
      "step": 593
    },
    {
      "epoch": 0.7288343558282209,
      "grad_norm": 30.96499084941041,
      "learning_rate": 1.1473140495867768e-06,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.197265625,
      "logps/chosen": -366.0,
      "logps/rejected": -334.0,
      "loss": 0.3336,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.65625,
      "step": 594
    },
    {
      "epoch": 0.7300613496932515,
      "grad_norm": 33.881983550723454,
      "learning_rate": 1.1466942148760332e-06,
      "logits/chosen": -0.059814453125,
      "logits/rejected": -0.26953125,
      "logps/chosen": -454.0,
      "logps/rejected": -372.0,
      "loss": 0.2673,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.71875,
      "step": 595
    },
    {
      "epoch": 0.7312883435582822,
      "grad_norm": 34.4461650357885,
      "learning_rate": 1.1460743801652893e-06,
      "logits/chosen": -0.0286865234375,
      "logits/rejected": -0.18359375,
      "logps/chosen": -422.0,
      "logps/rejected": -368.0,
      "loss": 0.3909,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.004608154296875,
      "rewards/margins": 7.25,
      "rewards/rejected": -7.25,
      "step": 596
    },
    {
      "epoch": 0.7325153374233129,
      "grad_norm": 32.96451484117692,
      "learning_rate": 1.1454545454545455e-06,
      "logits/chosen": 0.0830078125,
      "logits/rejected": -0.09228515625,
      "logps/chosen": -388.0,
      "logps/rejected": -306.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.53125,
      "step": 597
    },
    {
      "epoch": 0.7337423312883435,
      "grad_norm": 25.225306353625232,
      "learning_rate": 1.1448347107438018e-06,
      "logits/chosen": -0.0211181640625,
      "logits/rejected": -0.216796875,
      "logps/chosen": -446.0,
      "logps/rejected": -338.0,
      "loss": 0.2285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.296875,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.09375,
      "step": 598
    },
    {
      "epoch": 0.7349693251533742,
      "grad_norm": 30.51091893610521,
      "learning_rate": 1.1442148760330577e-06,
      "logits/chosen": -0.169921875,
      "logits/rejected": -0.3125,
      "logps/chosen": -456.0,
      "logps/rejected": -386.0,
      "loss": 0.3423,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.53125,
      "step": 599
    },
    {
      "epoch": 0.7361963190184049,
      "grad_norm": 31.65374536927,
      "learning_rate": 1.143595041322314e-06,
      "logits/chosen": -0.1796875,
      "logits/rejected": -0.25390625,
      "logps/chosen": -364.0,
      "logps/rejected": -334.0,
      "loss": 0.437,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.3515625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.1875,
      "step": 600
    },
    {
      "epoch": 0.7374233128834355,
      "grad_norm": 29.257508369320092,
      "learning_rate": 1.1429752066115702e-06,
      "logits/chosen": -0.03759765625,
      "logits/rejected": -0.224609375,
      "logps/chosen": -348.0,
      "logps/rejected": -324.0,
      "loss": 0.3337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.51953125,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.71875,
      "step": 601
    },
    {
      "epoch": 0.7386503067484662,
      "grad_norm": 23.637703488224467,
      "learning_rate": 1.1423553719008264e-06,
      "logits/chosen": 0.1103515625,
      "logits/rejected": -0.0517578125,
      "logps/chosen": -406.0,
      "logps/rejected": -350.0,
      "loss": 0.2192,
      "rewards/accuracies": 0.9296875,
      "rewards/chosen": 0.64453125,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.78125,
      "step": 602
    },
    {
      "epoch": 0.7398773006134969,
      "grad_norm": 24.798455930230393,
      "learning_rate": 1.1417355371900827e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.1943359375,
      "logps/chosen": -382.0,
      "logps/rejected": -336.0,
      "loss": 0.3279,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.82421875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.5,
      "step": 603
    },
    {
      "epoch": 0.7411042944785277,
      "grad_norm": 28.661443673376716,
      "learning_rate": 1.1411157024793389e-06,
      "logits/chosen": -0.11376953125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -420.0,
      "logps/rejected": -374.0,
      "loss": 0.2818,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.267578125,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.8125,
      "step": 604
    },
    {
      "epoch": 0.7423312883435583,
      "grad_norm": 28.682789953669307,
      "learning_rate": 1.140495867768595e-06,
      "logits/chosen": -0.057373046875,
      "logits/rejected": -0.24609375,
      "logps/chosen": -380.0,
      "logps/rejected": -348.0,
      "loss": 0.2877,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.349609375,
      "rewards/margins": 8.5,
      "rewards/rejected": -8.125,
      "step": 605
    },
    {
      "epoch": 0.743558282208589,
      "grad_norm": 31.866662459492442,
      "learning_rate": 1.1398760330578514e-06,
      "logits/chosen": -0.138671875,
      "logits/rejected": -0.263671875,
      "logps/chosen": -390.0,
      "logps/rejected": -348.0,
      "loss": 0.3751,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.443359375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.21875,
      "step": 606
    },
    {
      "epoch": 0.7447852760736197,
      "grad_norm": 29.660350580126263,
      "learning_rate": 1.1392561983471075e-06,
      "logits/chosen": -0.0289306640625,
      "logits/rejected": -0.25,
      "logps/chosen": -404.0,
      "logps/rejected": -386.0,
      "loss": 0.3196,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.265625,
      "rewards/margins": 8.0625,
      "rewards/rejected": -8.3125,
      "step": 607
    },
    {
      "epoch": 0.7460122699386503,
      "grad_norm": 31.21931912524627,
      "learning_rate": 1.1386363636363636e-06,
      "logits/chosen": 0.10498046875,
      "logits/rejected": -0.1416015625,
      "logps/chosen": -404.0,
      "logps/rejected": -360.0,
      "loss": 0.3793,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.5,
      "step": 608
    },
    {
      "epoch": 0.747239263803681,
      "grad_norm": 35.278030303126314,
      "learning_rate": 1.1380165289256198e-06,
      "logits/chosen": -0.02978515625,
      "logits/rejected": -0.1953125,
      "logps/chosen": -412.0,
      "logps/rejected": -342.0,
      "loss": 0.4025,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.255859375,
      "rewards/margins": 7.9375,
      "rewards/rejected": -8.1875,
      "step": 609
    },
    {
      "epoch": 0.7484662576687117,
      "grad_norm": 29.748859331814412,
      "learning_rate": 1.137396694214876e-06,
      "logits/chosen": -0.10302734375,
      "logits/rejected": -0.3203125,
      "logps/chosen": -452.0,
      "logps/rejected": -396.0,
      "loss": 0.2827,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.2216796875,
      "rewards/margins": 8.625,
      "rewards/rejected": -8.375,
      "step": 610
    },
    {
      "epoch": 0.7496932515337423,
      "grad_norm": 42.36265371910942,
      "learning_rate": 1.1367768595041323e-06,
      "logits/chosen": -0.050537109375,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -442.0,
      "logps/rejected": -384.0,
      "loss": 0.493,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.234375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -8.0625,
      "step": 611
    },
    {
      "epoch": 0.750920245398773,
      "grad_norm": 36.03030744089288,
      "learning_rate": 1.1361570247933884e-06,
      "logits/chosen": -0.042236328125,
      "logits/rejected": -0.1796875,
      "logps/chosen": -402.0,
      "logps/rejected": -376.0,
      "loss": 0.4916,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.462890625,
      "rewards/margins": 6.78125,
      "rewards/rejected": -7.25,
      "step": 612
    },
    {
      "epoch": 0.7521472392638037,
      "grad_norm": 35.23308877108843,
      "learning_rate": 1.1355371900826446e-06,
      "logits/chosen": -0.04052734375,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -388.0,
      "logps/rejected": -348.0,
      "loss": 0.4598,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -7.875,
      "step": 613
    },
    {
      "epoch": 0.7533742331288343,
      "grad_norm": 35.827377930375896,
      "learning_rate": 1.134917355371901e-06,
      "logits/chosen": -0.095703125,
      "logits/rejected": -0.2216796875,
      "logps/chosen": -430.0,
      "logps/rejected": -364.0,
      "loss": 0.3803,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.333984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.71875,
      "step": 614
    },
    {
      "epoch": 0.754601226993865,
      "grad_norm": 31.70255072714647,
      "learning_rate": 1.134297520661157e-06,
      "logits/chosen": -0.07373046875,
      "logits/rejected": -0.166015625,
      "logps/chosen": -376.0,
      "logps/rejected": -364.0,
      "loss": 0.3435,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.037841796875,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.8125,
      "step": 615
    },
    {
      "epoch": 0.7558282208588957,
      "grad_norm": 30.125086244839487,
      "learning_rate": 1.1336776859504134e-06,
      "logits/chosen": -0.004241943359375,
      "logits/rejected": -0.15625,
      "logps/chosen": -332.0,
      "logps/rejected": -310.0,
      "loss": 0.3151,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.6328125,
      "rewards/margins": 6.53125,
      "rewards/rejected": -7.15625,
      "step": 616
    },
    {
      "epoch": 0.7570552147239263,
      "grad_norm": 32.7474160937551,
      "learning_rate": 1.1330578512396693e-06,
      "logits/chosen": 0.030517578125,
      "logits/rejected": -0.0390625,
      "logps/chosen": -378.0,
      "logps/rejected": -336.0,
      "loss": 0.3705,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 6.75,
      "rewards/rejected": -7.15625,
      "step": 617
    },
    {
      "epoch": 0.758282208588957,
      "grad_norm": 45.056199177526004,
      "learning_rate": 1.1324380165289255e-06,
      "logits/chosen": 0.109375,
      "logits/rejected": -0.056396484375,
      "logps/chosen": -426.0,
      "logps/rejected": -332.0,
      "loss": 0.5324,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.625,
      "step": 618
    },
    {
      "epoch": 0.7595092024539877,
      "grad_norm": 29.95096958444537,
      "learning_rate": 1.1318181818181818e-06,
      "logits/chosen": 0.08056640625,
      "logits/rejected": -0.12060546875,
      "logps/chosen": -432.0,
      "logps/rejected": -350.0,
      "loss": 0.2823,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.36328125,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.34375,
      "step": 619
    },
    {
      "epoch": 0.7607361963190185,
      "grad_norm": 37.035466575479155,
      "learning_rate": 1.131198347107438e-06,
      "logits/chosen": 0.08349609375,
      "logits/rejected": -0.11181640625,
      "logps/chosen": -410.0,
      "logps/rejected": -346.0,
      "loss": 0.3202,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.03271484375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -6.71875,
      "step": 620
    },
    {
      "epoch": 0.7619631901840491,
      "grad_norm": 30.546031931386054,
      "learning_rate": 1.1305785123966943e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -482.0,
      "logps/rejected": -380.0,
      "loss": 0.3488,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.7109375,
      "rewards/margins": 8.125,
      "rewards/rejected": -7.40625,
      "step": 621
    },
    {
      "epoch": 0.7631901840490798,
      "grad_norm": 30.102626290429743,
      "learning_rate": 1.1299586776859505e-06,
      "logits/chosen": 0.07666015625,
      "logits/rejected": -0.0791015625,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.3683,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.2060546875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.15625,
      "step": 622
    },
    {
      "epoch": 0.7644171779141105,
      "grad_norm": 31.272882227536257,
      "learning_rate": 1.1293388429752066e-06,
      "logits/chosen": 0.041259765625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -456.0,
      "logps/rejected": -390.0,
      "loss": 0.338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.0078125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.78125,
      "step": 623
    },
    {
      "epoch": 0.7656441717791411,
      "grad_norm": 29.25638333606671,
      "learning_rate": 1.128719008264463e-06,
      "logits/chosen": -0.036865234375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -382.0,
      "logps/rejected": -334.0,
      "loss": 0.302,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.8671875,
      "rewards/margins": 8.5,
      "rewards/rejected": -7.59375,
      "step": 624
    },
    {
      "epoch": 0.7668711656441718,
      "grad_norm": 27.76458750020888,
      "learning_rate": 1.1280991735537191e-06,
      "logits/chosen": 0.01214599609375,
      "logits/rejected": -0.123046875,
      "logps/chosen": -408.0,
      "logps/rejected": -334.0,
      "loss": 0.2518,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.69140625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.46875,
      "step": 625
    },
    {
      "epoch": 0.7680981595092025,
      "grad_norm": 34.03667353357896,
      "learning_rate": 1.1274793388429753e-06,
      "logits/chosen": -0.119140625,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -416.0,
      "logps/rejected": -358.0,
      "loss": 0.4042,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.140625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -5.875,
      "step": 626
    },
    {
      "epoch": 0.7693251533742331,
      "grad_norm": 30.346756983376576,
      "learning_rate": 1.1268595041322314e-06,
      "logits/chosen": 0.095703125,
      "logits/rejected": 0.0223388671875,
      "logps/chosen": -314.0,
      "logps/rejected": -332.0,
      "loss": 0.3854,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.65625,
      "rewards/margins": 5.28125,
      "rewards/rejected": -5.9375,
      "step": 627
    },
    {
      "epoch": 0.7705521472392638,
      "grad_norm": 31.196230115400443,
      "learning_rate": 1.1262396694214875e-06,
      "logits/chosen": 0.0732421875,
      "logits/rejected": -0.06884765625,
      "logps/chosen": -380.0,
      "logps/rejected": -304.0,
      "loss": 0.2959,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.70703125,
      "rewards/margins": 6.40625,
      "rewards/rejected": -5.71875,
      "step": 628
    },
    {
      "epoch": 0.7717791411042945,
      "grad_norm": 27.109929558619076,
      "learning_rate": 1.1256198347107439e-06,
      "logits/chosen": 0.0262451171875,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -352.0,
      "logps/rejected": -292.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.353515625,
      "rewards/margins": 6.5,
      "rewards/rejected": -6.15625,
      "step": 629
    },
    {
      "epoch": 0.7730061349693251,
      "grad_norm": 27.79699333558513,
      "learning_rate": 1.125e-06,
      "logits/chosen": 0.0947265625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -374.0,
      "logps/rejected": -372.0,
      "loss": 0.3309,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 1.21875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.875,
      "step": 630
    },
    {
      "epoch": 0.7742331288343558,
      "grad_norm": 32.55842929304895,
      "learning_rate": 1.1243801652892562e-06,
      "logits/chosen": -0.11474609375,
      "logits/rejected": -0.2001953125,
      "logps/chosen": -394.0,
      "logps/rejected": -326.0,
      "loss": 0.4115,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 1.03125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -5.6875,
      "step": 631
    },
    {
      "epoch": 0.7754601226993865,
      "grad_norm": 30.882292248466843,
      "learning_rate": 1.1237603305785125e-06,
      "logits/chosen": -0.0111083984375,
      "logits/rejected": -0.134765625,
      "logps/chosen": -406.0,
      "logps/rejected": -344.0,
      "loss": 0.3282,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.125,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.09375,
      "step": 632
    },
    {
      "epoch": 0.7766871165644171,
      "grad_norm": 32.30019011289937,
      "learning_rate": 1.1231404958677687e-06,
      "logits/chosen": 0.072265625,
      "logits/rejected": -0.03857421875,
      "logps/chosen": -342.0,
      "logps/rejected": -338.0,
      "loss": 0.3515,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.39453125,
      "rewards/margins": 5.71875,
      "rewards/rejected": -5.34375,
      "step": 633
    },
    {
      "epoch": 0.7779141104294478,
      "grad_norm": 26.84260857948927,
      "learning_rate": 1.1225206611570248e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.248046875,
      "logps/chosen": -416.0,
      "logps/rejected": -328.0,
      "loss": 0.2366,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 2.546875,
      "rewards/margins": 8.3125,
      "rewards/rejected": -5.78125,
      "step": 634
    },
    {
      "epoch": 0.7791411042944786,
      "grad_norm": 23.744218806065337,
      "learning_rate": 1.121900826446281e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.09912109375,
      "logps/chosen": -362.0,
      "logps/rejected": -302.0,
      "loss": 0.2194,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -5.5625,
      "step": 635
    },
    {
      "epoch": 0.7803680981595092,
      "grad_norm": 34.66381046216169,
      "learning_rate": 1.121280991735537e-06,
      "logits/chosen": -0.06201171875,
      "logits/rejected": -0.166015625,
      "logps/chosen": -414.0,
      "logps/rejected": -368.0,
      "loss": 0.4205,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.46875,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.1875,
      "step": 636
    },
    {
      "epoch": 0.7815950920245399,
      "grad_norm": 41.24072249858344,
      "learning_rate": 1.1206611570247934e-06,
      "logits/chosen": 0.11962890625,
      "logits/rejected": -0.0615234375,
      "logps/chosen": -382.0,
      "logps/rejected": -306.0,
      "loss": 0.4805,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 6.125,
      "rewards/rejected": -5.6875,
      "step": 637
    },
    {
      "epoch": 0.7828220858895706,
      "grad_norm": 31.604533876936294,
      "learning_rate": 1.1200413223140496e-06,
      "logits/chosen": -0.044677734375,
      "logits/rejected": -0.2451171875,
      "logps/chosen": -406.0,
      "logps/rejected": -340.0,
      "loss": 0.2858,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3046875,
      "rewards/margins": 6.9375,
      "rewards/rejected": -5.625,
      "step": 638
    },
    {
      "epoch": 0.7840490797546013,
      "grad_norm": 34.862589865848754,
      "learning_rate": 1.1194214876033057e-06,
      "logits/chosen": 0.0869140625,
      "logits/rejected": -0.080078125,
      "logps/chosen": -408.0,
      "logps/rejected": -342.0,
      "loss": 0.4829,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 1.046875,
      "rewards/margins": 6.34375,
      "rewards/rejected": -5.28125,
      "step": 639
    },
    {
      "epoch": 0.7852760736196319,
      "grad_norm": 29.526720756728196,
      "learning_rate": 1.118801652892562e-06,
      "logits/chosen": -0.08837890625,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -446.0,
      "logps/rejected": -394.0,
      "loss": 0.2982,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.6328125,
      "rewards/margins": 7.5,
      "rewards/rejected": -5.875,
      "step": 640
    },
    {
      "epoch": 0.7865030674846626,
      "grad_norm": 38.16790195191018,
      "learning_rate": 1.1181818181818182e-06,
      "logits/chosen": -0.1259765625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -398.0,
      "logps/rejected": -360.0,
      "loss": 0.5882,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.90625,
      "rewards/margins": 6.15625,
      "rewards/rejected": -5.25,
      "step": 641
    },
    {
      "epoch": 0.7877300613496933,
      "grad_norm": 25.77138922359556,
      "learning_rate": 1.1175619834710746e-06,
      "logits/chosen": -0.01422119140625,
      "logits/rejected": -0.2333984375,
      "logps/chosen": -352.0,
      "logps/rejected": -304.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 6.65625,
      "rewards/rejected": -5.75,
      "step": 642
    },
    {
      "epoch": 0.7889570552147239,
      "grad_norm": 35.220692662985385,
      "learning_rate": 1.1169421487603307e-06,
      "logits/chosen": -0.1748046875,
      "logits/rejected": -0.333984375,
      "logps/chosen": -406.0,
      "logps/rejected": -386.0,
      "loss": 0.327,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.9375,
      "step": 643
    },
    {
      "epoch": 0.7901840490797546,
      "grad_norm": 30.676171080381682,
      "learning_rate": 1.1163223140495866e-06,
      "logits/chosen": -0.07666015625,
      "logits/rejected": -0.21875,
      "logps/chosen": -392.0,
      "logps/rejected": -338.0,
      "loss": 0.3619,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.75390625,
      "rewards/margins": 7.25,
      "rewards/rejected": -6.5,
      "step": 644
    },
    {
      "epoch": 0.7914110429447853,
      "grad_norm": 29.15888390878358,
      "learning_rate": 1.115702479338843e-06,
      "logits/chosen": -0.03955078125,
      "logits/rejected": -0.234375,
      "logps/chosen": -336.0,
      "logps/rejected": -346.0,
      "loss": 0.2816,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.98046875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.5625,
      "step": 645
    },
    {
      "epoch": 0.7926380368098159,
      "grad_norm": 28.593454046054227,
      "learning_rate": 1.1150826446280991e-06,
      "logits/chosen": 0.047119140625,
      "logits/rejected": -0.134765625,
      "logps/chosen": -386.0,
      "logps/rejected": -340.0,
      "loss": 0.2765,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.59375,
      "step": 646
    },
    {
      "epoch": 0.7938650306748466,
      "grad_norm": 32.656529890149606,
      "learning_rate": 1.1144628099173555e-06,
      "logits/chosen": -0.033935546875,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -386.0,
      "logps/rejected": -334.0,
      "loss": 0.3573,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.9296875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -6.875,
      "step": 647
    },
    {
      "epoch": 0.7950920245398773,
      "grad_norm": 36.18303049965862,
      "learning_rate": 1.1138429752066116e-06,
      "logits/chosen": -0.0908203125,
      "logits/rejected": -0.08837890625,
      "logps/chosen": -334.0,
      "logps/rejected": -336.0,
      "loss": 0.438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0986328125,
      "rewards/margins": 6.375,
      "rewards/rejected": -6.46875,
      "step": 648
    },
    {
      "epoch": 0.7963190184049079,
      "grad_norm": 33.937125436202074,
      "learning_rate": 1.1132231404958678e-06,
      "logits/chosen": 0.061279296875,
      "logits/rejected": -0.027587890625,
      "logps/chosen": -372.0,
      "logps/rejected": -342.0,
      "loss": 0.5003,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0732421875,
      "rewards/margins": 6.0,
      "rewards/rejected": -5.9375,
      "step": 649
    },
    {
      "epoch": 0.7975460122699386,
      "grad_norm": 36.01649609906715,
      "learning_rate": 1.1126033057851241e-06,
      "logits/chosen": 0.0264892578125,
      "logits/rejected": -0.1298828125,
      "logps/chosen": -432.0,
      "logps/rejected": -364.0,
      "loss": 0.4749,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.71875,
      "rewards/margins": 6.5,
      "rewards/rejected": -5.78125,
      "step": 650
    },
    {
      "epoch": 0.7987730061349694,
      "grad_norm": 37.059100709512336,
      "learning_rate": 1.1119834710743803e-06,
      "logits/chosen": -0.12109375,
      "logits/rejected": -0.1845703125,
      "logps/chosen": -376.0,
      "logps/rejected": -362.0,
      "loss": 0.4208,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.125,
      "step": 651
    },
    {
      "epoch": 0.8,
      "grad_norm": 40.43433181753938,
      "learning_rate": 1.1113636363636364e-06,
      "logits/chosen": -0.031494140625,
      "logits/rejected": -0.1455078125,
      "logps/chosen": -376.0,
      "logps/rejected": -360.0,
      "loss": 0.4169,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.5078125,
      "rewards/margins": 6.625,
      "rewards/rejected": -7.125,
      "step": 652
    },
    {
      "epoch": 0.8012269938650307,
      "grad_norm": 32.30744337627645,
      "learning_rate": 1.1107438016528925e-06,
      "logits/chosen": 0.032958984375,
      "logits/rejected": -0.142578125,
      "logps/chosen": -358.0,
      "logps/rejected": -346.0,
      "loss": 0.2863,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.0498046875,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.59375,
      "step": 653
    },
    {
      "epoch": 0.8024539877300614,
      "grad_norm": 23.943395122229603,
      "learning_rate": 1.1101239669421487e-06,
      "logits/chosen": 0.050537109375,
      "logits/rejected": -0.15234375,
      "logps/chosen": -400.0,
      "logps/rejected": -336.0,
      "loss": 0.226,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.333984375,
      "rewards/margins": 7.96875,
      "rewards/rejected": -7.625,
      "step": 654
    },
    {
      "epoch": 0.803680981595092,
      "grad_norm": 31.423267487900187,
      "learning_rate": 1.109504132231405e-06,
      "logits/chosen": -0.007659912109375,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -396.0,
      "logps/rejected": -366.0,
      "loss": 0.3332,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.1943359375,
      "rewards/margins": 7.125,
      "rewards/rejected": -7.3125,
      "step": 655
    },
    {
      "epoch": 0.8049079754601227,
      "grad_norm": 35.483670047647635,
      "learning_rate": 1.1088842975206612e-06,
      "logits/chosen": 0.055419921875,
      "logits/rejected": -0.08740234375,
      "logps/chosen": -380.0,
      "logps/rejected": -344.0,
      "loss": 0.4163,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.5,
      "step": 656
    },
    {
      "epoch": 0.8061349693251534,
      "grad_norm": 27.21341640433181,
      "learning_rate": 1.1082644628099173e-06,
      "logits/chosen": -0.10107421875,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -372.0,
      "logps/rejected": -354.0,
      "loss": 0.3384,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.388671875,
      "rewards/margins": 7.1875,
      "rewards/rejected": -7.5625,
      "step": 657
    },
    {
      "epoch": 0.807361963190184,
      "grad_norm": 31.34176371809714,
      "learning_rate": 1.1076446280991737e-06,
      "logits/chosen": -0.08349609375,
      "logits/rejected": -0.212890625,
      "logps/chosen": -402.0,
      "logps/rejected": -366.0,
      "loss": 0.3063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.412109375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.625,
      "step": 658
    },
    {
      "epoch": 0.8085889570552147,
      "grad_norm": 32.62830203002507,
      "learning_rate": 1.1070247933884298e-06,
      "logits/chosen": -0.0291748046875,
      "logits/rejected": -0.16796875,
      "logps/chosen": -374.0,
      "logps/rejected": -348.0,
      "loss": 0.3798,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1171875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -7.71875,
      "step": 659
    },
    {
      "epoch": 0.8098159509202454,
      "grad_norm": 29.335771689820604,
      "learning_rate": 1.106404958677686e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.220703125,
      "logps/chosen": -412.0,
      "logps/rejected": -374.0,
      "loss": 0.3215,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.053466796875,
      "rewards/margins": 7.8125,
      "rewards/rejected": -7.75,
      "step": 660
    },
    {
      "epoch": 0.811042944785276,
      "grad_norm": 39.19555957381655,
      "learning_rate": 1.1057851239669423e-06,
      "logits/chosen": 0.054443359375,
      "logits/rejected": -0.052734375,
      "logps/chosen": -374.0,
      "logps/rejected": -318.0,
      "loss": 0.5135,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -1.09375,
      "rewards/margins": 6.46875,
      "rewards/rejected": -7.5625,
      "step": 661
    },
    {
      "epoch": 0.8122699386503067,
      "grad_norm": 25.59951308246572,
      "learning_rate": 1.1051652892561982e-06,
      "logits/chosen": 0.05224609375,
      "logits/rejected": -0.06884765625,
      "logps/chosen": -372.0,
      "logps/rejected": -342.0,
      "loss": 0.218,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.3671875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.8125,
      "step": 662
    },
    {
      "epoch": 0.8134969325153374,
      "grad_norm": 29.103568092770104,
      "learning_rate": 1.1045454545454546e-06,
      "logits/chosen": 0.1142578125,
      "logits/rejected": -0.080078125,
      "logps/chosen": -362.0,
      "logps/rejected": -340.0,
      "loss": 0.3666,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.8125,
      "step": 663
    },
    {
      "epoch": 0.8147239263803681,
      "grad_norm": 24.88151445188299,
      "learning_rate": 1.1039256198347107e-06,
      "logits/chosen": -0.2255859375,
      "logits/rejected": -0.35546875,
      "logps/chosen": -388.0,
      "logps/rejected": -376.0,
      "loss": 0.213,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.7578125,
      "rewards/margins": 9.0625,
      "rewards/rejected": -8.3125,
      "step": 664
    },
    {
      "epoch": 0.8159509202453987,
      "grad_norm": 34.48345990804285,
      "learning_rate": 1.1033057851239669e-06,
      "logits/chosen": 0.1259765625,
      "logits/rejected": -0.0419921875,
      "logps/chosen": -342.0,
      "logps/rejected": -308.0,
      "loss": 0.4057,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2470703125,
      "rewards/margins": 7.125,
      "rewards/rejected": -7.375,
      "step": 665
    },
    {
      "epoch": 0.8171779141104294,
      "grad_norm": 35.79692152845337,
      "learning_rate": 1.1026859504132232e-06,
      "logits/chosen": 0.0791015625,
      "logits/rejected": -0.0927734375,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4703,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.68359375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.59375,
      "step": 666
    },
    {
      "epoch": 0.8184049079754602,
      "grad_norm": 29.967347606106397,
      "learning_rate": 1.1020661157024794e-06,
      "logits/chosen": -0.010986328125,
      "logits/rejected": -0.2109375,
      "logps/chosen": -376.0,
      "logps/rejected": -344.0,
      "loss": 0.2918,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.7421875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.78125,
      "step": 667
    },
    {
      "epoch": 0.8196319018404908,
      "grad_norm": 30.862796893024488,
      "learning_rate": 1.1014462809917355e-06,
      "logits/chosen": 0.0123291015625,
      "logits/rejected": -0.119140625,
      "logps/chosen": -392.0,
      "logps/rejected": -360.0,
      "loss": 0.3721,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.65625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.4375,
      "step": 668
    },
    {
      "epoch": 0.8208588957055215,
      "grad_norm": 35.17919160040694,
      "learning_rate": 1.1008264462809919e-06,
      "logits/chosen": -0.052490234375,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -340.0,
      "logps/rejected": -338.0,
      "loss": 0.3167,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6953125,
      "rewards/margins": 7.09375,
      "rewards/rejected": -6.40625,
      "step": 669
    },
    {
      "epoch": 0.8220858895705522,
      "grad_norm": 25.88170844832394,
      "learning_rate": 1.100206611570248e-06,
      "logits/chosen": 0.17578125,
      "logits/rejected": 0.0250244140625,
      "logps/chosen": -324.0,
      "logps/rejected": -302.0,
      "loss": 0.2514,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 1.0390625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -5.78125,
      "step": 670
    },
    {
      "epoch": 0.8233128834355828,
      "grad_norm": 30.762328731871737,
      "learning_rate": 1.0995867768595042e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.12109375,
      "logps/chosen": -372.0,
      "logps/rejected": -332.0,
      "loss": 0.2995,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.7421875,
      "rewards/margins": 7.5,
      "rewards/rejected": -5.78125,
      "step": 671
    },
    {
      "epoch": 0.8245398773006135,
      "grad_norm": 30.48600462690235,
      "learning_rate": 1.0989669421487603e-06,
      "logits/chosen": 0.054931640625,
      "logits/rejected": -0.2197265625,
      "logps/chosen": -446.0,
      "logps/rejected": -332.0,
      "loss": 0.3199,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 2.015625,
      "rewards/margins": 7.71875,
      "rewards/rejected": -5.6875,
      "step": 672
    },
    {
      "epoch": 0.8257668711656442,
      "grad_norm": 32.03977574623459,
      "learning_rate": 1.0983471074380164e-06,
      "logits/chosen": 0.1416015625,
      "logits/rejected": -0.0986328125,
      "logps/chosen": -390.0,
      "logps/rejected": -328.0,
      "loss": 0.3096,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.109375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -5.125,
      "step": 673
    },
    {
      "epoch": 0.8269938650306748,
      "grad_norm": 32.93059543816188,
      "learning_rate": 1.0977272727272728e-06,
      "logits/chosen": 0.162109375,
      "logits/rejected": 0.00286865234375,
      "logps/chosen": -358.0,
      "logps/rejected": -330.0,
      "loss": 0.3985,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 1.0546875,
      "rewards/margins": 6.59375,
      "rewards/rejected": -5.53125,
      "step": 674
    },
    {
      "epoch": 0.8282208588957055,
      "grad_norm": 34.971531039669564,
      "learning_rate": 1.097107438016529e-06,
      "logits/chosen": 0.08203125,
      "logits/rejected": -0.130859375,
      "logps/chosen": -426.0,
      "logps/rejected": -368.0,
      "loss": 0.4407,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.3828125,
      "rewards/margins": 7.40625,
      "rewards/rejected": -6.0,
      "step": 675
    },
    {
      "epoch": 0.8294478527607362,
      "grad_norm": 34.2224078373622,
      "learning_rate": 1.0964876033057853e-06,
      "logits/chosen": -0.037353515625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -394.0,
      "logps/rejected": -352.0,
      "loss": 0.3813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.375,
      "step": 676
    },
    {
      "epoch": 0.8306748466257668,
      "grad_norm": 22.049121537748125,
      "learning_rate": 1.0958677685950414e-06,
      "logits/chosen": 0.09423828125,
      "logits/rejected": -0.08447265625,
      "logps/chosen": -338.0,
      "logps/rejected": -316.0,
      "loss": 0.2974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2099609375,
      "rewards/margins": 7.1875,
      "rewards/rejected": -6.96875,
      "step": 677
    },
    {
      "epoch": 0.8319018404907975,
      "grad_norm": 31.042587162348006,
      "learning_rate": 1.0952479338842976e-06,
      "logits/chosen": 0.07421875,
      "logits/rejected": -0.1376953125,
      "logps/chosen": -422.0,
      "logps/rejected": -362.0,
      "loss": 0.4105,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.439453125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.53125,
      "step": 678
    },
    {
      "epoch": 0.8331288343558282,
      "grad_norm": 32.125565947605324,
      "learning_rate": 1.0946280991735537e-06,
      "logits/chosen": 0.1689453125,
      "logits/rejected": 0.06201171875,
      "logps/chosen": -344.0,
      "logps/rejected": -328.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.0625,
      "step": 679
    },
    {
      "epoch": 0.8343558282208589,
      "grad_norm": 30.54691503767555,
      "learning_rate": 1.0940082644628098e-06,
      "logits/chosen": 0.025390625,
      "logits/rejected": -0.150390625,
      "logps/chosen": -454.0,
      "logps/rejected": -374.0,
      "loss": 0.3146,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.25,
      "rewards/margins": 8.125,
      "rewards/rejected": -6.875,
      "step": 680
    },
    {
      "epoch": 0.8355828220858895,
      "grad_norm": 25.35697096492706,
      "learning_rate": 1.0933884297520662e-06,
      "logits/chosen": -0.0262451171875,
      "logits/rejected": -0.1796875,
      "logps/chosen": -348.0,
      "logps/rejected": -338.0,
      "loss": 0.2471,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.72265625,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.71875,
      "step": 681
    },
    {
      "epoch": 0.8368098159509203,
      "grad_norm": 29.372252652247152,
      "learning_rate": 1.0927685950413223e-06,
      "logits/chosen": -0.000934600830078125,
      "logits/rejected": -0.115234375,
      "logps/chosen": -352.0,
      "logps/rejected": -334.0,
      "loss": 0.3998,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5546875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.90625,
      "step": 682
    },
    {
      "epoch": 0.838036809815951,
      "grad_norm": 28.48014927993986,
      "learning_rate": 1.0921487603305785e-06,
      "logits/chosen": -0.0264892578125,
      "logits/rejected": -0.1328125,
      "logps/chosen": -374.0,
      "logps/rejected": -330.0,
      "loss": 0.3192,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.67578125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.375,
      "step": 683
    },
    {
      "epoch": 0.8392638036809816,
      "grad_norm": 30.9956365019602,
      "learning_rate": 1.0915289256198348e-06,
      "logits/chosen": 0.2001953125,
      "logits/rejected": 0.08544921875,
      "logps/chosen": -322.0,
      "logps/rejected": -352.0,
      "loss": 0.3553,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.6953125,
      "rewards/margins": 6.3125,
      "rewards/rejected": -7.03125,
      "step": 684
    },
    {
      "epoch": 0.8404907975460123,
      "grad_norm": 31.024167516184594,
      "learning_rate": 1.090909090909091e-06,
      "logits/chosen": 0.040283203125,
      "logits/rejected": -0.12255859375,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.3423,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.400390625,
      "rewards/margins": 8.625,
      "rewards/rejected": -8.25,
      "step": 685
    },
    {
      "epoch": 0.841717791411043,
      "grad_norm": 25.880225574187453,
      "learning_rate": 1.0902892561983471e-06,
      "logits/chosen": 0.031982421875,
      "logits/rejected": -0.095703125,
      "logps/chosen": -400.0,
      "logps/rejected": -348.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.27734375,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.65625,
      "step": 686
    },
    {
      "epoch": 0.8429447852760736,
      "grad_norm": 28.80339614616542,
      "learning_rate": 1.0896694214876035e-06,
      "logits/chosen": 0.03759765625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -380.0,
      "logps/rejected": -364.0,
      "loss": 0.2973,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.484375,
      "rewards/margins": 7.90625,
      "rewards/rejected": -7.40625,
      "step": 687
    },
    {
      "epoch": 0.8441717791411043,
      "grad_norm": 30.11912577456189,
      "learning_rate": 1.0890495867768594e-06,
      "logits/chosen": -0.000240325927734375,
      "logits/rejected": -0.158203125,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.327,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.5625,
      "rewards/margins": 7.5,
      "rewards/rejected": -8.0625,
      "step": 688
    },
    {
      "epoch": 0.845398773006135,
      "grad_norm": 30.49529821286284,
      "learning_rate": 1.0884297520661158e-06,
      "logits/chosen": 0.027099609375,
      "logits/rejected": -0.058837890625,
      "logps/chosen": -382.0,
      "logps/rejected": -348.0,
      "loss": 0.344,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.271484375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.40625,
      "step": 689
    },
    {
      "epoch": 0.8466257668711656,
      "grad_norm": 41.86773593768432,
      "learning_rate": 1.0878099173553719e-06,
      "logits/chosen": 0.171875,
      "logits/rejected": -0.0174560546875,
      "logps/chosen": -414.0,
      "logps/rejected": -370.0,
      "loss": 0.5772,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.21875,
      "step": 690
    },
    {
      "epoch": 0.8478527607361963,
      "grad_norm": 27.177968327720407,
      "learning_rate": 1.087190082644628e-06,
      "logits/chosen": -0.162109375,
      "logits/rejected": -0.2734375,
      "logps/chosen": -366.0,
      "logps/rejected": -368.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.29296875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.90625,
      "step": 691
    },
    {
      "epoch": 0.849079754601227,
      "grad_norm": 38.42433716881971,
      "learning_rate": 1.0865702479338844e-06,
      "logits/chosen": 0.0179443359375,
      "logits/rejected": -0.142578125,
      "logps/chosen": -416.0,
      "logps/rejected": -358.0,
      "loss": 0.3292,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.8203125,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.53125,
      "step": 692
    },
    {
      "epoch": 0.8503067484662576,
      "grad_norm": 31.708672912333903,
      "learning_rate": 1.0859504132231405e-06,
      "logits/chosen": -0.00286865234375,
      "logits/rejected": -0.1904296875,
      "logps/chosen": -452.0,
      "logps/rejected": -386.0,
      "loss": 0.3787,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.0625,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.8125,
      "step": 693
    },
    {
      "epoch": 0.8515337423312883,
      "grad_norm": 39.65580173728978,
      "learning_rate": 1.0853305785123967e-06,
      "logits/chosen": 0.024658203125,
      "logits/rejected": -0.162109375,
      "logps/chosen": -384.0,
      "logps/rejected": -352.0,
      "loss": 0.5333,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": -0.671875,
      "rewards/margins": 6.09375,
      "rewards/rejected": -6.75,
      "step": 694
    },
    {
      "epoch": 0.852760736196319,
      "grad_norm": 38.14890644660088,
      "learning_rate": 1.084710743801653e-06,
      "logits/chosen": 0.06103515625,
      "logits/rejected": -0.0179443359375,
      "logps/chosen": -338.0,
      "logps/rejected": -322.0,
      "loss": 0.5352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4609375,
      "rewards/margins": 5.375,
      "rewards/rejected": -5.8125,
      "step": 695
    },
    {
      "epoch": 0.8539877300613496,
      "grad_norm": 32.915799758521196,
      "learning_rate": 1.0840909090909092e-06,
      "logits/chosen": 0.0947265625,
      "logits/rejected": -0.08154296875,
      "logps/chosen": -390.0,
      "logps/rejected": -336.0,
      "loss": 0.3241,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.365234375,
      "rewards/margins": 7.125,
      "rewards/rejected": -6.78125,
      "step": 696
    },
    {
      "epoch": 0.8552147239263803,
      "grad_norm": 35.9913417584187,
      "learning_rate": 1.0834710743801653e-06,
      "logits/chosen": -0.0771484375,
      "logits/rejected": -0.10009765625,
      "logps/chosen": -398.0,
      "logps/rejected": -384.0,
      "loss": 0.4041,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.9375,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.0625,
      "step": 697
    },
    {
      "epoch": 0.8564417177914111,
      "grad_norm": 31.064185654999054,
      "learning_rate": 1.0828512396694214e-06,
      "logits/chosen": 0.08984375,
      "logits/rejected": -0.027587890625,
      "logps/chosen": -404.0,
      "logps/rejected": -366.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.392578125,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.09375,
      "step": 698
    },
    {
      "epoch": 0.8576687116564418,
      "grad_norm": 26.06263909227069,
      "learning_rate": 1.0822314049586776e-06,
      "logits/chosen": 0.0260009765625,
      "logits/rejected": -0.053955078125,
      "logps/chosen": -346.0,
      "logps/rejected": -340.0,
      "loss": 0.2587,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.228515625,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.6875,
      "step": 699
    },
    {
      "epoch": 0.8588957055214724,
      "grad_norm": 33.7346004258113,
      "learning_rate": 1.081611570247934e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.06005859375,
      "logps/chosen": -380.0,
      "logps/rejected": -342.0,
      "loss": 0.398,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.0072021484375,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.28125,
      "step": 700
    },
    {
      "epoch": 0.8601226993865031,
      "grad_norm": 36.508608167851875,
      "learning_rate": 1.08099173553719e-06,
      "logits/chosen": 0.1630859375,
      "logits/rejected": 0.05810546875,
      "logps/chosen": -390.0,
      "logps/rejected": -342.0,
      "loss": 0.4454,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.267578125,
      "rewards/margins": 6.5625,
      "rewards/rejected": -6.8125,
      "step": 701
    },
    {
      "epoch": 0.8613496932515338,
      "grad_norm": 29.752428717784724,
      "learning_rate": 1.0803719008264462e-06,
      "logits/chosen": -0.005767822265625,
      "logits/rejected": -0.30078125,
      "logps/chosen": -406.0,
      "logps/rejected": -348.0,
      "loss": 0.2347,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.91015625,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.09375,
      "step": 702
    },
    {
      "epoch": 0.8625766871165644,
      "grad_norm": 28.46276315635888,
      "learning_rate": 1.0797520661157026e-06,
      "logits/chosen": 0.1513671875,
      "logits/rejected": 0.0250244140625,
      "logps/chosen": -424.0,
      "logps/rejected": -358.0,
      "loss": 0.3098,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1396484375,
      "rewards/margins": 6.53125,
      "rewards/rejected": -6.375,
      "step": 703
    },
    {
      "epoch": 0.8638036809815951,
      "grad_norm": 28.331719307949253,
      "learning_rate": 1.0791322314049587e-06,
      "logits/chosen": 0.06689453125,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -332.0,
      "logps/rejected": -316.0,
      "loss": 0.3639,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.07861328125,
      "rewards/margins": 6.96875,
      "rewards/rejected": -6.875,
      "step": 704
    },
    {
      "epoch": 0.8650306748466258,
      "grad_norm": 33.73975373967079,
      "learning_rate": 1.078512396694215e-06,
      "logits/chosen": -0.0615234375,
      "logits/rejected": -0.177734375,
      "logps/chosen": -424.0,
      "logps/rejected": -376.0,
      "loss": 0.3238,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.75,
      "rewards/margins": 8.0,
      "rewards/rejected": -7.25,
      "step": 705
    },
    {
      "epoch": 0.8662576687116564,
      "grad_norm": 32.06391249106975,
      "learning_rate": 1.077892561983471e-06,
      "logits/chosen": 0.01544189453125,
      "logits/rejected": -0.1796875,
      "logps/chosen": -410.0,
      "logps/rejected": -316.0,
      "loss": 0.3915,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.1748046875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.6875,
      "step": 706
    },
    {
      "epoch": 0.8674846625766871,
      "grad_norm": 32.50030520028076,
      "learning_rate": 1.0772727272727271e-06,
      "logits/chosen": -0.00185394287109375,
      "logits/rejected": -0.1650390625,
      "logps/chosen": -394.0,
      "logps/rejected": -326.0,
      "loss": 0.3089,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.0947265625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -7.375,
      "step": 707
    },
    {
      "epoch": 0.8687116564417178,
      "grad_norm": 27.35860386441147,
      "learning_rate": 1.0766528925619835e-06,
      "logits/chosen": 0.0245361328125,
      "logits/rejected": -0.193359375,
      "logps/chosen": -380.0,
      "logps/rejected": -328.0,
      "loss": 0.2856,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.65234375,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.625,
      "step": 708
    },
    {
      "epoch": 0.8699386503067484,
      "grad_norm": 22.967542887627133,
      "learning_rate": 1.0760330578512396e-06,
      "logits/chosen": 0.11376953125,
      "logits/rejected": -0.031982421875,
      "logps/chosen": -444.0,
      "logps/rejected": -360.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.984375,
      "rewards/margins": 8.4375,
      "rewards/rejected": -7.46875,
      "step": 709
    },
    {
      "epoch": 0.8711656441717791,
      "grad_norm": 30.012370581713046,
      "learning_rate": 1.075413223140496e-06,
      "logits/chosen": 0.1845703125,
      "logits/rejected": 0.06103515625,
      "logps/chosen": -372.0,
      "logps/rejected": -330.0,
      "loss": 0.3445,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1064453125,
      "rewards/margins": 6.46875,
      "rewards/rejected": -6.375,
      "step": 710
    },
    {
      "epoch": 0.8723926380368098,
      "grad_norm": 26.59602404250697,
      "learning_rate": 1.0747933884297521e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": -0.039306640625,
      "logps/chosen": -390.0,
      "logps/rejected": -332.0,
      "loss": 0.3562,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5703125,
      "rewards/margins": 7.0,
      "rewards/rejected": -6.4375,
      "step": 711
    },
    {
      "epoch": 0.8736196319018404,
      "grad_norm": 34.082753455253695,
      "learning_rate": 1.0741735537190083e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.00482177734375,
      "logps/chosen": -410.0,
      "logps/rejected": -356.0,
      "loss": 0.4269,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.328125,
      "rewards/margins": 7.03125,
      "rewards/rejected": -5.71875,
      "step": 712
    },
    {
      "epoch": 0.8748466257668711,
      "grad_norm": 24.8779582799026,
      "learning_rate": 1.0735537190082646e-06,
      "logits/chosen": 0.05810546875,
      "logits/rejected": -0.1484375,
      "logps/chosen": -386.0,
      "logps/rejected": -336.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.625,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.65625,
      "step": 713
    },
    {
      "epoch": 0.8760736196319019,
      "grad_norm": 27.823114674287325,
      "learning_rate": 1.0729338842975208e-06,
      "logits/chosen": 0.181640625,
      "logits/rejected": -0.038818359375,
      "logps/chosen": -370.0,
      "logps/rejected": -328.0,
      "loss": 0.3057,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.87109375,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.0,
      "step": 714
    },
    {
      "epoch": 0.8773006134969326,
      "grad_norm": 29.027125190845624,
      "learning_rate": 1.072314049586777e-06,
      "logits/chosen": -0.06494140625,
      "logits/rejected": -0.1318359375,
      "logps/chosen": -384.0,
      "logps/rejected": -328.0,
      "loss": 0.3391,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.015625,
      "rewards/margins": 7.75,
      "rewards/rejected": -6.71875,
      "step": 715
    },
    {
      "epoch": 0.8785276073619632,
      "grad_norm": 26.494530473617345,
      "learning_rate": 1.071694214876033e-06,
      "logits/chosen": -0.018798828125,
      "logits/rejected": -0.1865234375,
      "logps/chosen": -384.0,
      "logps/rejected": -324.0,
      "loss": 0.3073,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.1640625,
      "rewards/margins": 6.84375,
      "rewards/rejected": -5.65625,
      "step": 716
    },
    {
      "epoch": 0.8797546012269939,
      "grad_norm": 29.02718718589355,
      "learning_rate": 1.0710743801652892e-06,
      "logits/chosen": 0.07470703125,
      "logits/rejected": -0.08056640625,
      "logps/chosen": -344.0,
      "logps/rejected": -284.0,
      "loss": 0.3344,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.6875,
      "rewards/margins": 6.90625,
      "rewards/rejected": -6.21875,
      "step": 717
    },
    {
      "epoch": 0.8809815950920246,
      "grad_norm": 23.193149611746794,
      "learning_rate": 1.0704545454545455e-06,
      "logits/chosen": -0.034912109375,
      "logits/rejected": -0.1640625,
      "logps/chosen": -402.0,
      "logps/rejected": -356.0,
      "loss": 0.2308,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.5859375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -6.25,
      "step": 718
    },
    {
      "epoch": 0.8822085889570552,
      "grad_norm": 36.13316553444024,
      "learning_rate": 1.0698347107438017e-06,
      "logits/chosen": -0.07763671875,
      "logits/rejected": -0.19921875,
      "logps/chosen": -410.0,
      "logps/rejected": -368.0,
      "loss": 0.3568,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 1.8046875,
      "rewards/margins": 8.0,
      "rewards/rejected": -6.1875,
      "step": 719
    },
    {
      "epoch": 0.8834355828220859,
      "grad_norm": 25.470910809931194,
      "learning_rate": 1.0692148760330578e-06,
      "logits/chosen": -0.07421875,
      "logits/rejected": -0.279296875,
      "logps/chosen": -412.0,
      "logps/rejected": -350.0,
      "loss": 0.2706,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 1.9140625,
      "rewards/margins": 7.9375,
      "rewards/rejected": -6.03125,
      "step": 720
    },
    {
      "epoch": 0.8846625766871166,
      "grad_norm": 24.45362594207483,
      "learning_rate": 1.0685950413223142e-06,
      "logits/chosen": -0.0810546875,
      "logits/rejected": -0.337890625,
      "logps/chosen": -414.0,
      "logps/rejected": -364.0,
      "loss": 0.2805,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 1.5625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.25,
      "step": 721
    },
    {
      "epoch": 0.8858895705521472,
      "grad_norm": 26.653798150160988,
      "learning_rate": 1.0679752066115703e-06,
      "logits/chosen": -0.049072265625,
      "logits/rejected": -0.1376953125,
      "logps/chosen": -380.0,
      "logps/rejected": -358.0,
      "loss": 0.2495,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 1.609375,
      "rewards/margins": 7.875,
      "rewards/rejected": -6.28125,
      "step": 722
    },
    {
      "epoch": 0.8871165644171779,
      "grad_norm": 29.259615066962372,
      "learning_rate": 1.0673553719008265e-06,
      "logits/chosen": -0.043212890625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -376.0,
      "logps/rejected": -334.0,
      "loss": 0.3287,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.94921875,
      "rewards/margins": 7.4375,
      "rewards/rejected": -6.5,
      "step": 723
    },
    {
      "epoch": 0.8883435582822086,
      "grad_norm": 30.45569089414315,
      "learning_rate": 1.0667355371900826e-06,
      "logits/chosen": -0.0308837890625,
      "logits/rejected": -0.251953125,
      "logps/chosen": -382.0,
      "logps/rejected": -320.0,
      "loss": 0.3219,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 1.3359375,
      "rewards/margins": 7.8125,
      "rewards/rejected": -6.5,
      "step": 724
    },
    {
      "epoch": 0.8895705521472392,
      "grad_norm": 30.8228410953136,
      "learning_rate": 1.0661157024793387e-06,
      "logits/chosen": 0.12255859375,
      "logits/rejected": -0.05419921875,
      "logps/chosen": -404.0,
      "logps/rejected": -340.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.984375,
      "rewards/margins": 7.375,
      "rewards/rejected": -6.375,
      "step": 725
    },
    {
      "epoch": 0.8907975460122699,
      "grad_norm": 32.44931458712784,
      "learning_rate": 1.065495867768595e-06,
      "logits/chosen": -0.030029296875,
      "logits/rejected": -0.154296875,
      "logps/chosen": -340.0,
      "logps/rejected": -302.0,
      "loss": 0.2852,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.5703125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -6.96875,
      "step": 726
    },
    {
      "epoch": 0.8920245398773006,
      "grad_norm": 30.00382649809932,
      "learning_rate": 1.0648760330578512e-06,
      "logits/chosen": -0.0013427734375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -374.0,
      "logps/rejected": -338.0,
      "loss": 0.2567,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.84765625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -6.71875,
      "step": 727
    },
    {
      "epoch": 0.8932515337423312,
      "grad_norm": 32.084882438300085,
      "learning_rate": 1.0642561983471074e-06,
      "logits/chosen": -0.0244140625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -406.0,
      "logps/rejected": -384.0,
      "loss": 0.3182,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 1.234375,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.625,
      "step": 728
    },
    {
      "epoch": 0.894478527607362,
      "grad_norm": 24.427532348622456,
      "learning_rate": 1.0636363636363637e-06,
      "logits/chosen": -0.040771484375,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -396.0,
      "logps/rejected": -372.0,
      "loss": 0.2099,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.9765625,
      "rewards/margins": 8.875,
      "rewards/rejected": -7.90625,
      "step": 729
    },
    {
      "epoch": 0.8957055214723927,
      "grad_norm": 24.535854490724105,
      "learning_rate": 1.0630165289256199e-06,
      "logits/chosen": -0.103515625,
      "logits/rejected": -0.2060546875,
      "logps/chosen": -404.0,
      "logps/rejected": -380.0,
      "loss": 0.2127,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 1.515625,
      "rewards/margins": 8.8125,
      "rewards/rejected": -7.3125,
      "step": 730
    },
    {
      "epoch": 0.8969325153374234,
      "grad_norm": 34.14852950381922,
      "learning_rate": 1.062396694214876e-06,
      "logits/chosen": 0.1171875,
      "logits/rejected": 0.002349853515625,
      "logps/chosen": -438.0,
      "logps/rejected": -366.0,
      "loss": 0.3465,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.62890625,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.5625,
      "step": 731
    },
    {
      "epoch": 0.898159509202454,
      "grad_norm": 34.434393814707754,
      "learning_rate": 1.0617768595041324e-06,
      "logits/chosen": -0.058837890625,
      "logits/rejected": -0.2099609375,
      "logps/chosen": -380.0,
      "logps/rejected": -372.0,
      "loss": 0.4237,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.6875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.03125,
      "step": 732
    },
    {
      "epoch": 0.8993865030674847,
      "grad_norm": 28.68332287996042,
      "learning_rate": 1.0611570247933883e-06,
      "logits/chosen": 0.0673828125,
      "logits/rejected": -0.083984375,
      "logps/chosen": -366.0,
      "logps/rejected": -362.0,
      "loss": 0.2815,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1015625,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.5625,
      "step": 733
    },
    {
      "epoch": 0.9006134969325154,
      "grad_norm": 32.670441266137765,
      "learning_rate": 1.0605371900826447e-06,
      "logits/chosen": 0.06884765625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -384.0,
      "logps/rejected": -354.0,
      "loss": 0.3319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.90625,
      "step": 734
    },
    {
      "epoch": 0.901840490797546,
      "grad_norm": 28.766622871975752,
      "learning_rate": 1.0599173553719008e-06,
      "logits/chosen": -0.037841796875,
      "logits/rejected": -0.259765625,
      "logps/chosen": -434.0,
      "logps/rejected": -390.0,
      "loss": 0.2447,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2490234375,
      "rewards/margins": 8.875,
      "rewards/rejected": -8.625,
      "step": 735
    },
    {
      "epoch": 0.9030674846625767,
      "grad_norm": 31.10013526298491,
      "learning_rate": 1.059297520661157e-06,
      "logits/chosen": 0.1533203125,
      "logits/rejected": 0.029541015625,
      "logps/chosen": -362.0,
      "logps/rejected": -340.0,
      "loss": 0.4211,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": -0.53515625,
      "rewards/margins": 7.03125,
      "rewards/rejected": -7.5625,
      "step": 736
    },
    {
      "epoch": 0.9042944785276074,
      "grad_norm": 38.03816635846729,
      "learning_rate": 1.0586776859504133e-06,
      "logits/chosen": 0.04541015625,
      "logits/rejected": -0.051513671875,
      "logps/chosen": -360.0,
      "logps/rejected": -348.0,
      "loss": 0.4641,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.478515625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.5625,
      "step": 737
    },
    {
      "epoch": 0.905521472392638,
      "grad_norm": 32.716309617319716,
      "learning_rate": 1.0580578512396694e-06,
      "logits/chosen": -0.0380859375,
      "logits/rejected": -0.1474609375,
      "logps/chosen": -450.0,
      "logps/rejected": -364.0,
      "loss": 0.3437,
      "rewards/accuracies": 0.8359375,
      "rewards/chosen": 0.60546875,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.59375,
      "step": 738
    },
    {
      "epoch": 0.9067484662576687,
      "grad_norm": 29.627927045089628,
      "learning_rate": 1.0574380165289258e-06,
      "logits/chosen": 0.060302734375,
      "logits/rejected": -0.0654296875,
      "logps/chosen": -346.0,
      "logps/rejected": -340.0,
      "loss": 0.2827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06640625,
      "rewards/margins": 7.1875,
      "rewards/rejected": -7.25,
      "step": 739
    },
    {
      "epoch": 0.9079754601226994,
      "grad_norm": 29.046321525927297,
      "learning_rate": 1.056818181818182e-06,
      "logits/chosen": -0.0308837890625,
      "logits/rejected": -0.18359375,
      "logps/chosen": -344.0,
      "logps/rejected": -326.0,
      "loss": 0.3577,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.02392578125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.53125,
      "step": 740
    },
    {
      "epoch": 0.90920245398773,
      "grad_norm": 29.826427625052336,
      "learning_rate": 1.056198347107438e-06,
      "logits/chosen": -0.00982666015625,
      "logits/rejected": -0.162109375,
      "logps/chosen": -356.0,
      "logps/rejected": -328.0,
      "loss": 0.3514,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.8359375,
      "rewards/margins": 8.75,
      "rewards/rejected": -7.875,
      "step": 741
    },
    {
      "epoch": 0.9104294478527607,
      "grad_norm": 28.054610141633944,
      "learning_rate": 1.0555785123966942e-06,
      "logits/chosen": -0.033935546875,
      "logits/rejected": -0.1982421875,
      "logps/chosen": -368.0,
      "logps/rejected": -372.0,
      "loss": 0.2813,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.6640625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -7.90625,
      "step": 742
    },
    {
      "epoch": 0.9116564417177914,
      "grad_norm": 35.487753263657986,
      "learning_rate": 1.0549586776859503e-06,
      "logits/chosen": 0.01055908203125,
      "logits/rejected": -0.040771484375,
      "logps/chosen": -424.0,
      "logps/rejected": -392.0,
      "loss": 0.3821,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.2265625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.0,
      "step": 743
    },
    {
      "epoch": 0.912883435582822,
      "grad_norm": 28.496366117513464,
      "learning_rate": 1.0543388429752067e-06,
      "logits/chosen": 0.035400390625,
      "logits/rejected": -0.14453125,
      "logps/chosen": -380.0,
      "logps/rejected": -340.0,
      "loss": 0.2775,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.9609375,
      "rewards/margins": 8.6875,
      "rewards/rejected": -7.71875,
      "step": 744
    },
    {
      "epoch": 0.9141104294478528,
      "grad_norm": 30.005979043165016,
      "learning_rate": 1.0537190082644628e-06,
      "logits/chosen": 0.08642578125,
      "logits/rejected": -0.0003509521484375,
      "logps/chosen": -346.0,
      "logps/rejected": -318.0,
      "loss": 0.3381,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.1044921875,
      "rewards/margins": 6.84375,
      "rewards/rejected": -6.75,
      "step": 745
    },
    {
      "epoch": 0.9153374233128835,
      "grad_norm": 36.365949373279655,
      "learning_rate": 1.053099173553719e-06,
      "logits/chosen": 0.09033203125,
      "logits/rejected": -0.07958984375,
      "logps/chosen": -408.0,
      "logps/rejected": -356.0,
      "loss": 0.4283,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 7.4375,
      "rewards/rejected": -7.0,
      "step": 746
    },
    {
      "epoch": 0.9165644171779141,
      "grad_norm": 30.46892734256714,
      "learning_rate": 1.0524793388429753e-06,
      "logits/chosen": -0.1123046875,
      "logits/rejected": -0.21875,
      "logps/chosen": -342.0,
      "logps/rejected": -344.0,
      "loss": 0.2905,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": -0.0234375,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.75,
      "step": 747
    },
    {
      "epoch": 0.9177914110429448,
      "grad_norm": 37.226894144134455,
      "learning_rate": 1.0518595041322315e-06,
      "logits/chosen": 0.125,
      "logits/rejected": 0.01458740234375,
      "logps/chosen": -380.0,
      "logps/rejected": -354.0,
      "loss": 0.4992,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 6.28125,
      "rewards/rejected": -6.78125,
      "step": 748
    },
    {
      "epoch": 0.9190184049079755,
      "grad_norm": 34.24194812875213,
      "learning_rate": 1.0512396694214876e-06,
      "logits/chosen": -0.04541015625,
      "logits/rejected": -0.2431640625,
      "logps/chosen": -424.0,
      "logps/rejected": -358.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2470703125,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.34375,
      "step": 749
    },
    {
      "epoch": 0.9202453987730062,
      "grad_norm": 35.848596312284045,
      "learning_rate": 1.050619834710744e-06,
      "logits/chosen": 0.01190185546875,
      "logits/rejected": -0.119140625,
      "logps/chosen": -412.0,
      "logps/rejected": -348.0,
      "loss": 0.4472,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.09326171875,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.28125,
      "step": 750
    },
    {
      "epoch": 0.9214723926380368,
      "grad_norm": 36.49684940562663,
      "learning_rate": 1.05e-06,
      "logits/chosen": -0.0064697265625,
      "logits/rejected": -0.169921875,
      "logps/chosen": -428.0,
      "logps/rejected": -366.0,
      "loss": 0.3626,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5234375,
      "rewards/margins": 8.3125,
      "rewards/rejected": -7.8125,
      "step": 751
    },
    {
      "epoch": 0.9226993865030675,
      "grad_norm": 31.9902592360704,
      "learning_rate": 1.0493801652892563e-06,
      "logits/chosen": -0.028564453125,
      "logits/rejected": -0.275390625,
      "logps/chosen": -424.0,
      "logps/rejected": -368.0,
      "loss": 0.3427,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.640625,
      "rewards/margins": 8.375,
      "rewards/rejected": -7.71875,
      "step": 752
    },
    {
      "epoch": 0.9239263803680982,
      "grad_norm": 30.210419690529548,
      "learning_rate": 1.0487603305785124e-06,
      "logits/chosen": 0.142578125,
      "logits/rejected": 0.01361083984375,
      "logps/chosen": -408.0,
      "logps/rejected": -362.0,
      "loss": 0.3032,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.126953125,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.25,
      "step": 753
    },
    {
      "epoch": 0.9251533742331288,
      "grad_norm": 33.70924536763799,
      "learning_rate": 1.0481404958677685e-06,
      "logits/chosen": 0.06640625,
      "logits/rejected": -0.1044921875,
      "logps/chosen": -394.0,
      "logps/rejected": -348.0,
      "loss": 0.4405,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.1416015625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.21875,
      "step": 754
    },
    {
      "epoch": 0.9263803680981595,
      "grad_norm": 28.923057351049156,
      "learning_rate": 1.0475206611570249e-06,
      "logits/chosen": 0.01611328125,
      "logits/rejected": -0.1064453125,
      "logps/chosen": -388.0,
      "logps/rejected": -366.0,
      "loss": 0.313,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.291015625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.5,
      "step": 755
    },
    {
      "epoch": 0.9276073619631902,
      "grad_norm": 26.956563960777117,
      "learning_rate": 1.046900826446281e-06,
      "logits/chosen": 0.115234375,
      "logits/rejected": 0.02978515625,
      "logps/chosen": -342.0,
      "logps/rejected": -326.0,
      "loss": 0.2891,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.640625,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.46875,
      "step": 756
    },
    {
      "epoch": 0.9288343558282208,
      "grad_norm": 25.54044780724446,
      "learning_rate": 1.0462809917355372e-06,
      "logits/chosen": 0.15625,
      "logits/rejected": -0.0595703125,
      "logps/chosen": -404.0,
      "logps/rejected": -344.0,
      "loss": 0.27,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7578125,
      "rewards/margins": 8.0625,
      "rewards/rejected": -7.3125,
      "step": 757
    },
    {
      "epoch": 0.9300613496932515,
      "grad_norm": 35.658646153584634,
      "learning_rate": 1.0456611570247935e-06,
      "logits/chosen": 0.08740234375,
      "logits/rejected": -0.080078125,
      "logps/chosen": -376.0,
      "logps/rejected": -320.0,
      "loss": 0.3867,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.55859375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.28125,
      "step": 758
    },
    {
      "epoch": 0.9312883435582822,
      "grad_norm": 29.003566311314795,
      "learning_rate": 1.0450413223140497e-06,
      "logits/chosen": 0.1123046875,
      "logits/rejected": -0.11572265625,
      "logps/chosen": -396.0,
      "logps/rejected": -340.0,
      "loss": 0.2888,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16015625,
      "rewards/margins": 6.9375,
      "rewards/rejected": -6.78125,
      "step": 759
    },
    {
      "epoch": 0.9325153374233128,
      "grad_norm": 25.36330419002366,
      "learning_rate": 1.0444214876033058e-06,
      "logits/chosen": -0.0986328125,
      "logits/rejected": -0.35546875,
      "logps/chosen": -442.0,
      "logps/rejected": -366.0,
      "loss": 0.2887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1796875,
      "rewards/margins": 8.625,
      "rewards/rejected": -7.4375,
      "step": 760
    },
    {
      "epoch": 0.9337423312883436,
      "grad_norm": 31.72216949313869,
      "learning_rate": 1.043801652892562e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": 0.0277099609375,
      "logps/chosen": -382.0,
      "logps/rejected": -354.0,
      "loss": 0.3707,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.3984375,
      "rewards/margins": 7.21875,
      "rewards/rejected": -6.8125,
      "step": 761
    },
    {
      "epoch": 0.9349693251533743,
      "grad_norm": 31.01314079949462,
      "learning_rate": 1.043181818181818e-06,
      "logits/chosen": 0.1591796875,
      "logits/rejected": -0.020751953125,
      "logps/chosen": -428.0,
      "logps/rejected": -364.0,
      "loss": 0.3149,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.2373046875,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.28125,
      "step": 762
    },
    {
      "epoch": 0.9361963190184049,
      "grad_norm": 28.119069224585317,
      "learning_rate": 1.0425619834710744e-06,
      "logits/chosen": 0.0986328125,
      "logits/rejected": 0.0164794921875,
      "logps/chosen": -400.0,
      "logps/rejected": -364.0,
      "loss": 0.3398,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.4375,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.125,
      "step": 763
    },
    {
      "epoch": 0.9374233128834356,
      "grad_norm": 26.42805050753053,
      "learning_rate": 1.0419421487603306e-06,
      "logits/chosen": -0.1328125,
      "logits/rejected": -0.287109375,
      "logps/chosen": -330.0,
      "logps/rejected": -316.0,
      "loss": 0.2955,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.494140625,
      "rewards/margins": 7.0,
      "rewards/rejected": -7.46875,
      "step": 764
    },
    {
      "epoch": 0.9386503067484663,
      "grad_norm": 35.659703021872964,
      "learning_rate": 1.041322314049587e-06,
      "logits/chosen": 0.1298828125,
      "logits/rejected": 0.005828857421875,
      "logps/chosen": -404.0,
      "logps/rejected": -320.0,
      "loss": 0.3833,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.2041015625,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.34375,
      "step": 765
    },
    {
      "epoch": 0.939877300613497,
      "grad_norm": 27.867245312963597,
      "learning_rate": 1.040702479338843e-06,
      "logits/chosen": 0.04345703125,
      "logits/rejected": -0.1494140625,
      "logps/chosen": -446.0,
      "logps/rejected": -340.0,
      "loss": 0.3188,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.515625,
      "rewards/margins": 7.5,
      "rewards/rejected": -7.0,
      "step": 766
    },
    {
      "epoch": 0.9411042944785276,
      "grad_norm": 33.0423901915957,
      "learning_rate": 1.0400826446280992e-06,
      "logits/chosen": 0.0390625,
      "logits/rejected": -0.10498046875,
      "logps/chosen": -386.0,
      "logps/rejected": -338.0,
      "loss": 0.355,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.1689453125,
      "rewards/margins": 7.5625,
      "rewards/rejected": -7.375,
      "step": 767
    },
    {
      "epoch": 0.9423312883435583,
      "grad_norm": 31.83881144027911,
      "learning_rate": 1.0394628099173554e-06,
      "logits/chosen": 0.003570556640625,
      "logits/rejected": -0.1611328125,
      "logps/chosen": -410.0,
      "logps/rejected": -384.0,
      "loss": 0.3826,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6484375,
      "rewards/margins": 7.59375,
      "rewards/rejected": -6.96875,
      "step": 768
    },
    {
      "epoch": 0.943558282208589,
      "grad_norm": 24.722102647279602,
      "learning_rate": 1.0388429752066115e-06,
      "logits/chosen": 0.00079345703125,
      "logits/rejected": -0.138671875,
      "logps/chosen": -378.0,
      "logps/rejected": -344.0,
      "loss": 0.2777,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.068359375,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.59375,
      "step": 769
    },
    {
      "epoch": 0.9447852760736196,
      "grad_norm": 33.3608331180557,
      "learning_rate": 1.0382231404958679e-06,
      "logits/chosen": -0.021728515625,
      "logits/rejected": -0.265625,
      "logps/chosen": -416.0,
      "logps/rejected": -372.0,
      "loss": 0.321,
      "rewards/accuracies": 0.8515625,
      "rewards/chosen": 0.92578125,
      "rewards/margins": 7.5,
      "rewards/rejected": -6.59375,
      "step": 770
    },
    {
      "epoch": 0.9460122699386503,
      "grad_norm": 36.53871156772584,
      "learning_rate": 1.037603305785124e-06,
      "logits/chosen": -0.00537109375,
      "logits/rejected": -0.01507568359375,
      "logps/chosen": -430.0,
      "logps/rejected": -360.0,
      "loss": 0.5217,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6484375,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.875,
      "step": 771
    },
    {
      "epoch": 0.947239263803681,
      "grad_norm": 30.200058873190507,
      "learning_rate": 1.0369834710743801e-06,
      "logits/chosen": 0.1328125,
      "logits/rejected": -0.0869140625,
      "logps/chosen": -424.0,
      "logps/rejected": -362.0,
      "loss": 0.3536,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.38671875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -6.90625,
      "step": 772
    },
    {
      "epoch": 0.9484662576687116,
      "grad_norm": 31.95788929467944,
      "learning_rate": 1.0363636363636365e-06,
      "logits/chosen": 0.1484375,
      "logits/rejected": -0.037841796875,
      "logps/chosen": -374.0,
      "logps/rejected": -324.0,
      "loss": 0.3406,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.65234375,
      "rewards/margins": 6.5,
      "rewards/rejected": -7.15625,
      "step": 773
    },
    {
      "epoch": 0.9496932515337423,
      "grad_norm": 34.11224513726983,
      "learning_rate": 1.0357438016528926e-06,
      "logits/chosen": 0.1396484375,
      "logits/rejected": -0.0037078857421875,
      "logps/chosen": -362.0,
      "logps/rejected": -326.0,
      "loss": 0.4295,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -1.359375,
      "rewards/margins": 5.1875,
      "rewards/rejected": -6.5625,
      "step": 774
    },
    {
      "epoch": 0.950920245398773,
      "grad_norm": 31.83399815065193,
      "learning_rate": 1.0351239669421488e-06,
      "logits/chosen": 0.2197265625,
      "logits/rejected": 0.04296875,
      "logps/chosen": -418.0,
      "logps/rejected": -362.0,
      "loss": 0.3519,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.32421875,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.53125,
      "step": 775
    },
    {
      "epoch": 0.9521472392638037,
      "grad_norm": 29.278356141500073,
      "learning_rate": 1.0345041322314051e-06,
      "logits/chosen": -0.01409912109375,
      "logits/rejected": -0.1708984375,
      "logps/chosen": -414.0,
      "logps/rejected": -366.0,
      "loss": 0.3978,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0888671875,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.75,
      "step": 776
    },
    {
      "epoch": 0.9533742331288344,
      "grad_norm": 33.3868265617977,
      "learning_rate": 1.033884297520661e-06,
      "logits/chosen": 0.0267333984375,
      "logits/rejected": -0.1015625,
      "logps/chosen": -408.0,
      "logps/rejected": -346.0,
      "loss": 0.3944,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.3125,
      "step": 777
    },
    {
      "epoch": 0.9546012269938651,
      "grad_norm": 30.60810394473529,
      "learning_rate": 1.0332644628099174e-06,
      "logits/chosen": 0.0703125,
      "logits/rejected": -0.130859375,
      "logps/chosen": -400.0,
      "logps/rejected": -328.0,
      "loss": 0.2893,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.74609375,
      "rewards/margins": 6.0,
      "rewards/rejected": -6.75,
      "step": 778
    },
    {
      "epoch": 0.9558282208588957,
      "grad_norm": 31.819603506171298,
      "learning_rate": 1.0326446280991735e-06,
      "logits/chosen": 0.00994873046875,
      "logits/rejected": -0.1533203125,
      "logps/chosen": -442.0,
      "logps/rejected": -362.0,
      "loss": 0.331,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": 0.16796875,
      "rewards/margins": 7.59375,
      "rewards/rejected": -7.40625,
      "step": 779
    },
    {
      "epoch": 0.9570552147239264,
      "grad_norm": 33.50608635895235,
      "learning_rate": 1.0320247933884297e-06,
      "logits/chosen": 0.13671875,
      "logits/rejected": -0.037353515625,
      "logps/chosen": -386.0,
      "logps/rejected": -360.0,
      "loss": 0.4421,
      "rewards/accuracies": 0.7890625,
      "rewards/chosen": -0.91015625,
      "rewards/margins": 5.96875,
      "rewards/rejected": -6.875,
      "step": 780
    },
    {
      "epoch": 0.9582822085889571,
      "grad_norm": 27.175413708110298,
      "learning_rate": 1.031404958677686e-06,
      "logits/chosen": -0.0087890625,
      "logits/rejected": -0.2392578125,
      "logps/chosen": -426.0,
      "logps/rejected": -356.0,
      "loss": 0.2289,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.9140625,
      "rewards/margins": 8.75,
      "rewards/rejected": -7.8125,
      "step": 781
    },
    {
      "epoch": 0.9595092024539877,
      "grad_norm": 27.961517528382746,
      "learning_rate": 1.0307851239669422e-06,
      "logits/chosen": 0.0341796875,
      "logits/rejected": -0.055419921875,
      "logps/chosen": -370.0,
      "logps/rejected": -354.0,
      "loss": 0.2727,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.0152587890625,
      "rewards/margins": 7.21875,
      "rewards/rejected": -7.1875,
      "step": 782
    },
    {
      "epoch": 0.9607361963190184,
      "grad_norm": 36.53707624363368,
      "learning_rate": 1.0301652892561983e-06,
      "logits/chosen": 0.0115966796875,
      "logits/rejected": -0.12353515625,
      "logps/chosen": -420.0,
      "logps/rejected": -370.0,
      "loss": 0.3333,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.408203125,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.5,
      "step": 783
    },
    {
      "epoch": 0.9619631901840491,
      "grad_norm": 33.01812506650061,
      "learning_rate": 1.0295454545454547e-06,
      "logits/chosen": 0.09716796875,
      "logits/rejected": 0.056640625,
      "logps/chosen": -386.0,
      "logps/rejected": -354.0,
      "loss": 0.3887,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.69140625,
      "rewards/margins": 6.4375,
      "rewards/rejected": -7.125,
      "step": 784
    },
    {
      "epoch": 0.9631901840490797,
      "grad_norm": 25.64548904648115,
      "learning_rate": 1.0289256198347108e-06,
      "logits/chosen": 0.126953125,
      "logits/rejected": -0.0400390625,
      "logps/chosen": -384.0,
      "logps/rejected": -324.0,
      "loss": 0.2131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0047607421875,
      "rewards/margins": 7.40625,
      "rewards/rejected": -7.375,
      "step": 785
    },
    {
      "epoch": 0.9644171779141104,
      "grad_norm": 24.085836304747,
      "learning_rate": 1.028305785123967e-06,
      "logits/chosen": 0.11376953125,
      "logits/rejected": -0.09033203125,
      "logps/chosen": -372.0,
      "logps/rejected": -334.0,
      "loss": 0.2197,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.427734375,
      "rewards/margins": 7.375,
      "rewards/rejected": -7.8125,
      "step": 786
    },
    {
      "epoch": 0.9656441717791411,
      "grad_norm": 32.97798807698007,
      "learning_rate": 1.027685950413223e-06,
      "logits/chosen": 0.0269775390625,
      "logits/rejected": -0.08740234375,
      "logps/chosen": -412.0,
      "logps/rejected": -348.0,
      "loss": 0.3407,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.271484375,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.9375,
      "step": 787
    },
    {
      "epoch": 0.9668711656441717,
      "grad_norm": 24.19044914351049,
      "learning_rate": 1.0270661157024792e-06,
      "logits/chosen": -0.00860595703125,
      "logits/rejected": -0.1787109375,
      "logps/chosen": -390.0,
      "logps/rejected": -366.0,
      "loss": 0.2123,
      "rewards/accuracies": 0.9140625,
      "rewards/chosen": 0.486328125,
      "rewards/margins": 8.1875,
      "rewards/rejected": -7.6875,
      "step": 788
    },
    {
      "epoch": 0.9680981595092024,
      "grad_norm": 30.061433994827276,
      "learning_rate": 1.0264462809917356e-06,
      "logits/chosen": 0.12060546875,
      "logits/rejected": -0.06396484375,
      "logps/chosen": -384.0,
      "logps/rejected": -332.0,
      "loss": 0.383,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.52734375,
      "rewards/margins": 6.71875,
      "rewards/rejected": -7.25,
      "step": 789
    },
    {
      "epoch": 0.9693251533742331,
      "grad_norm": 27.089223909733843,
      "learning_rate": 1.0258264462809917e-06,
      "logits/chosen": -0.028564453125,
      "logits/rejected": -0.2373046875,
      "logps/chosen": -388.0,
      "logps/rejected": -330.0,
      "loss": 0.1804,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": -0.1533203125,
      "rewards/margins": 7.3125,
      "rewards/rejected": -7.46875,
      "step": 790
    },
    {
      "epoch": 0.9705521472392638,
      "grad_norm": 34.35477577844613,
      "learning_rate": 1.0252066115702479e-06,
      "logits/chosen": -0.0303955078125,
      "logits/rejected": -0.23046875,
      "logps/chosen": -398.0,
      "logps/rejected": -346.0,
      "loss": 0.3642,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.349609375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.90625,
      "step": 791
    },
    {
      "epoch": 0.9717791411042945,
      "grad_norm": 33.05641026053793,
      "learning_rate": 1.0245867768595042e-06,
      "logits/chosen": 0.0006256103515625,
      "logits/rejected": -0.1279296875,
      "logps/chosen": -438.0,
      "logps/rejected": -374.0,
      "loss": 0.406,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.380859375,
      "rewards/margins": 6.8125,
      "rewards/rejected": -6.4375,
      "step": 792
    },
    {
      "epoch": 0.9730061349693252,
      "grad_norm": 32.69405943160363,
      "learning_rate": 1.0239669421487604e-06,
      "logits/chosen": 0.0284423828125,
      "logits/rejected": -0.09765625,
      "logps/chosen": -392.0,
      "logps/rejected": -362.0,
      "loss": 0.4018,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.10546875,
      "rewards/margins": 6.75,
      "rewards/rejected": -6.84375,
      "step": 793
    },
    {
      "epoch": 0.9742331288343559,
      "grad_norm": 33.74613547302303,
      "learning_rate": 1.0233471074380167e-06,
      "logits/chosen": 0.0830078125,
      "logits/rejected": -0.043701171875,
      "logps/chosen": -358.0,
      "logps/rejected": -336.0,
      "loss": 0.4877,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": -0.6015625,
      "rewards/margins": 6.59375,
      "rewards/rejected": -7.1875,
      "step": 794
    },
    {
      "epoch": 0.9754601226993865,
      "grad_norm": 22.619679414305867,
      "learning_rate": 1.0227272727272727e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.296875,
      "logps/chosen": -392.0,
      "logps/rejected": -372.0,
      "loss": 0.281,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.80859375,
      "rewards/margins": 9.25,
      "rewards/rejected": -8.4375,
      "step": 795
    },
    {
      "epoch": 0.9766871165644172,
      "grad_norm": 35.85933901249144,
      "learning_rate": 1.0221074380165288e-06,
      "logits/chosen": -0.0260009765625,
      "logits/rejected": -0.2353515625,
      "logps/chosen": -410.0,
      "logps/rejected": -338.0,
      "loss": 0.3447,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.78515625,
      "rewards/margins": 7.25,
      "rewards/rejected": -8.0625,
      "step": 796
    },
    {
      "epoch": 0.9779141104294479,
      "grad_norm": 24.484140263550707,
      "learning_rate": 1.0214876033057852e-06,
      "logits/chosen": 0.059814453125,
      "logits/rejected": -0.11962890625,
      "logps/chosen": -404.0,
      "logps/rejected": -324.0,
      "loss": 0.243,
      "rewards/accuracies": 0.8671875,
      "rewards/chosen": -0.0966796875,
      "rewards/margins": 7.15625,
      "rewards/rejected": -7.25,
      "step": 797
    },
    {
      "epoch": 0.9791411042944785,
      "grad_norm": 35.333603529709684,
      "learning_rate": 1.0208677685950413e-06,
      "logits/chosen": 0.00506591796875,
      "logits/rejected": -0.173828125,
      "logps/chosen": -430.0,
      "logps/rejected": -378.0,
      "loss": 0.3546,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.1455078125,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.625,
      "step": 798
    },
    {
      "epoch": 0.9803680981595092,
      "grad_norm": 32.17706869798208,
      "learning_rate": 1.0202479338842976e-06,
      "logits/chosen": -0.05029296875,
      "logits/rejected": -0.201171875,
      "logps/chosen": -400.0,
      "logps/rejected": -360.0,
      "loss": 0.3323,
      "rewards/accuracies": 0.8203125,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 7.71875,
      "rewards/rejected": -7.28125,
      "step": 799
    },
    {
      "epoch": 0.9815950920245399,
      "grad_norm": 32.44337864899576,
      "learning_rate": 1.0196280991735538e-06,
      "logits/chosen": -0.0224609375,
      "logits/rejected": -0.208984375,
      "logps/chosen": -386.0,
      "logps/rejected": -344.0,
      "loss": 0.3141,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 7.28125,
      "rewards/rejected": -7.375,
      "step": 800
    },
    {
      "epoch": 0.9828220858895705,
      "grad_norm": 37.380385495976945,
      "learning_rate": 1.01900826446281e-06,
      "logits/chosen": -0.024658203125,
      "logits/rejected": -0.17578125,
      "logps/chosen": -436.0,
      "logps/rejected": -382.0,
      "loss": 0.4546,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.0001010894775390625,
      "rewards/margins": 6.6875,
      "rewards/rejected": -6.6875,
      "step": 801
    },
    {
      "epoch": 0.9840490797546012,
      "grad_norm": 35.755684948235334,
      "learning_rate": 1.0183884297520663e-06,
      "logits/chosen": -0.07421875,
      "logits/rejected": -0.21484375,
      "logps/chosen": -386.0,
      "logps/rejected": -348.0,
      "loss": 0.472,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1552734375,
      "rewards/margins": 7.0,
      "rewards/rejected": -7.15625,
      "step": 802
    },
    {
      "epoch": 0.9852760736196319,
      "grad_norm": 29.08898198089266,
      "learning_rate": 1.0177685950413224e-06,
      "logits/chosen": -0.1513671875,
      "logits/rejected": -0.302734375,
      "logps/chosen": -398.0,
      "logps/rejected": -348.0,
      "loss": 0.3346,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.54296875,
      "rewards/margins": 7.75,
      "rewards/rejected": -7.21875,
      "step": 803
    },
    {
      "epoch": 0.9865030674846625,
      "grad_norm": 30.37947277261302,
      "learning_rate": 1.0171487603305786e-06,
      "logits/chosen": 0.04150390625,
      "logits/rejected": -0.06689453125,
      "logps/chosen": -358.0,
      "logps/rejected": -346.0,
      "loss": 0.3638,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.392578125,
      "rewards/margins": 6.9375,
      "rewards/rejected": -7.34375,
      "step": 804
    },
    {
      "epoch": 0.9877300613496932,
      "grad_norm": 34.04488703971709,
      "learning_rate": 1.0165289256198347e-06,
      "logits/chosen": -0.078125,
      "logits/rejected": -0.2119140625,
      "logps/chosen": -434.0,
      "logps/rejected": -394.0,
      "loss": 0.4569,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.4296875,
      "rewards/margins": 7.78125,
      "rewards/rejected": -7.375,
      "step": 805
    },
    {
      "epoch": 0.9889570552147239,
      "grad_norm": 35.48502129676359,
      "learning_rate": 1.0159090909090908e-06,
      "logits/chosen": 0.16015625,
      "logits/rejected": 0.03564453125,
      "logps/chosen": -412.0,
      "logps/rejected": -384.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8046875,
      "rewards/chosen": 0.31640625,
      "rewards/margins": 7.15625,
      "rewards/rejected": -6.84375,
      "step": 806
    },
    {
      "epoch": 0.9901840490797545,
      "grad_norm": 33.61353577378301,
      "learning_rate": 1.0152892561983472e-06,
      "logits/chosen": 0.061767578125,
      "logits/rejected": -0.0908203125,
      "logps/chosen": -400.0,
      "logps/rejected": -350.0,
      "loss": 0.3144,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.484375,
      "rewards/margins": 8.25,
      "rewards/rejected": -7.75,
      "step": 807
    },
    {
      "epoch": 0.9914110429447853,
      "grad_norm": 31.091169744856,
      "learning_rate": 1.0146694214876033e-06,
      "logits/chosen": -0.083984375,
      "logits/rejected": -0.0966796875,
      "logps/chosen": -364.0,
      "logps/rejected": -352.0,
      "loss": 0.3745,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.640625,
      "rewards/margins": 7.5625,
      "rewards/rejected": -8.1875,
      "step": 808
    },
    {
      "epoch": 0.992638036809816,
      "grad_norm": 34.39338760048118,
      "learning_rate": 1.0140495867768595e-06,
      "logits/chosen": -0.00872802734375,
      "logits/rejected": -0.2314453125,
      "logps/chosen": -438.0,
      "logps/rejected": -360.0,
      "loss": 0.319,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.8515625,
      "rewards/margins": 8.5625,
      "rewards/rejected": -7.71875,
      "step": 809
    },
    {
      "epoch": 0.9938650306748467,
      "grad_norm": 26.950235054794184,
      "learning_rate": 1.0134297520661158e-06,
      "logits/chosen": -0.11572265625,
      "logits/rejected": -0.28515625,
      "logps/chosen": -446.0,
      "logps/rejected": -384.0,
      "loss": 0.2346,
      "rewards/accuracies": 0.8828125,
      "rewards/chosen": 0.44140625,
      "rewards/margins": 7.9375,
      "rewards/rejected": -7.5,
      "step": 810
    },
    {
      "epoch": 0.9950920245398773,
      "grad_norm": 26.578554179921486,
      "learning_rate": 1.012809917355372e-06,
      "logits/chosen": -0.08203125,
      "logits/rejected": -0.21484375,
      "logps/chosen": -394.0,
      "logps/rejected": -380.0,
      "loss": 0.3194,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.44921875,
      "rewards/margins": 7.46875,
      "rewards/rejected": -7.03125,
      "step": 811
    },
    {
      "epoch": 0.996319018404908,
      "grad_norm": 34.32887556980461,
      "learning_rate": 1.0121900826446281e-06,
      "logits/chosen": -0.05908203125,
      "logits/rejected": -0.125,
      "logps/chosen": -364.0,
      "logps/rejected": -340.0,
      "loss": 0.3746,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09619140625,
      "rewards/margins": 7.625,
      "rewards/rejected": -7.71875,
      "step": 812
    },
    {
      "epoch": 0.9975460122699387,
      "grad_norm": 25.82868308838759,
      "learning_rate": 1.0115702479338843e-06,
      "logits/chosen": 0.01611328125,
      "logits/rejected": -0.1123046875,
      "logps/chosen": -342.0,
      "logps/rejected": -336.0,
      "loss": 0.2067,
      "rewards/accuracies": 0.8984375,
      "rewards/chosen": 0.46484375,
      "rewards/margins": 7.84375,
      "rewards/rejected": -7.375,
      "step": 813
    },
    {
      "epoch": 0.9987730061349693,
      "grad_norm": 35.210336167923685,
      "learning_rate": 1.0109504132231404e-06,
      "logits/chosen": 0.054443359375,
      "logits/rejected": -0.0206298828125,
      "logps/chosen": -388.0,
      "logps/rejected": -370.0,
      "loss": 0.3789,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.80859375,
      "rewards/margins": 6.125,
      "rewards/rejected": -6.90625,
      "step": 814
    },
    {
      "epoch": 1.0,
      "grad_norm": 31.93988486773496,
      "learning_rate": 1.0103305785123968e-06,
      "logits/chosen": 0.1279296875,
      "logits/rejected": -0.061279296875,
      "logps/chosen": -388.0,
      "logps/rejected": -364.0,
      "loss": 0.2533,
      "rewards/accuracies": 0.8991597294807434,
      "rewards/chosen": -0.095703125,
      "rewards/margins": 7.53125,
      "rewards/rejected": -7.625,
      "step": 815
    }
  ],
  "logging_steps": 1,
  "max_steps": 2445,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}