{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 2548,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003924646781789639,
      "grad_norm": 1860.1517973668297,
      "learning_rate": 7.843137254901961e-09,
      "logits/chosen": -0.9052734375,
      "logits/rejected": -0.833984375,
      "logps/chosen": -513.5999755859375,
      "logps/rejected": -1219.0,
      "loss": 0.69,
      "rewards/accuracies": 0.20000000298023224,
      "rewards/chosen": -0.0027511597145348787,
      "rewards/margins": 0.00695877056568861,
      "rewards/rejected": -0.00970458984375,
      "step": 5
    },
    {
      "epoch": 0.007849293563579277,
      "grad_norm": 335.9986506406163,
      "learning_rate": 1.7647058823529412e-08,
      "logits/chosen": -0.870312511920929,
      "logits/rejected": -0.765625,
      "logps/chosen": -494.20001220703125,
      "logps/rejected": -1058.800048828125,
      "loss": 0.6895,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.000614166259765625,
      "rewards/margins": 0.01049957238137722,
      "rewards/rejected": -0.011149597354233265,
      "step": 10
    },
    {
      "epoch": 0.011773940345368918,
      "grad_norm": 858.6889922093546,
      "learning_rate": 2.745098039215686e-08,
      "logits/chosen": -0.804492175579071,
      "logits/rejected": -0.875683605670929,
      "logps/chosen": -550.5999755859375,
      "logps/rejected": -980.4000244140625,
      "loss": 0.6858,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.0034469603560864925,
      "rewards/margins": 0.017237091436982155,
      "rewards/rejected": -0.02072753943502903,
      "step": 15
    },
    {
      "epoch": 0.015698587127158554,
      "grad_norm": 387.9712168159993,
      "learning_rate": 3.7254901960784314e-08,
      "logits/chosen": -0.7513672113418579,
      "logits/rejected": -0.7955077886581421,
      "logps/chosen": -577.0,
      "logps/rejected": -1268.0,
      "loss": 0.6728,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.00810928363353014,
      "rewards/margins": 0.06101684644818306,
      "rewards/rejected": -0.069091796875,
      "step": 20
    },
    {
      "epoch": 0.019623233908948195,
      "grad_norm": 542.8616111958607,
      "learning_rate": 4.705882352941176e-08,
      "logits/chosen": -0.6324707269668579,
      "logits/rejected": -0.667919933795929,
      "logps/chosen": -579.2000122070312,
      "logps/rejected": -1094.4000244140625,
      "loss": 0.6745,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.03064880333840847,
      "rewards/margins": 0.17856445908546448,
      "rewards/rejected": -0.20915527641773224,
      "step": 25
    },
    {
      "epoch": 0.023547880690737835,
      "grad_norm": 297.4355896155964,
      "learning_rate": 5.686274509803921e-08,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.8431640863418579,
      "logps/chosen": -539.0,
      "logps/rejected": -789.2000122070312,
      "loss": 0.6743,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.037353515625,
      "rewards/margins": 0.128773495554924,
      "rewards/rejected": -0.16629639267921448,
      "step": 30
    },
    {
      "epoch": 0.027472527472527472,
      "grad_norm": 319.8297869456425,
      "learning_rate": 6.666666666666667e-08,
      "logits/chosen": -0.9609375,
      "logits/rejected": -0.911328136920929,
      "logps/chosen": -595.0,
      "logps/rejected": -772.0,
      "loss": 0.6981,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1180419921875,
      "rewards/margins": 0.12451171875,
      "rewards/rejected": -0.24226073920726776,
      "step": 35
    },
    {
      "epoch": 0.03139717425431711,
      "grad_norm": 367.5138091282664,
      "learning_rate": 7.647058823529412e-08,
      "logits/chosen": -0.814257800579071,
      "logits/rejected": -0.7435547113418579,
      "logps/chosen": -546.0,
      "logps/rejected": -747.7999877929688,
      "loss": 0.6711,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12700195610523224,
      "rewards/margins": 0.18607787787914276,
      "rewards/rejected": -0.31318360567092896,
      "step": 40
    },
    {
      "epoch": 0.03532182103610675,
      "grad_norm": 345.01437651872163,
      "learning_rate": 8.627450980392157e-08,
      "logits/chosen": -0.935839831829071,
      "logits/rejected": -0.787890613079071,
      "logps/chosen": -541.5999755859375,
      "logps/rejected": -1172.5999755859375,
      "loss": 0.6417,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.19189453125,
      "rewards/margins": 0.519775390625,
      "rewards/rejected": -0.7127929925918579,
      "step": 45
    },
    {
      "epoch": 0.03924646781789639,
      "grad_norm": 363.1844024747151,
      "learning_rate": 9.607843137254902e-08,
      "logits/chosen": -0.831347644329071,
      "logits/rejected": -0.4720703065395355,
      "logps/chosen": -541.5999755859375,
      "logps/rejected": -823.5999755859375,
      "loss": 0.6912,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3492187559604645,
      "rewards/margins": 0.32426756620407104,
      "rewards/rejected": -0.672656238079071,
      "step": 50
    },
    {
      "epoch": 0.04317111459968603,
      "grad_norm": 311.9226152138488,
      "learning_rate": 1.0588235294117647e-07,
      "logits/chosen": -0.8345702886581421,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -596.2000122070312,
      "logps/rejected": -936.4000244140625,
      "loss": 0.6431,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4423828125,
      "rewards/margins": 0.3084960877895355,
      "rewards/rejected": -0.749218761920929,
      "step": 55
    },
    {
      "epoch": 0.04709576138147567,
      "grad_norm": 722.8769766049414,
      "learning_rate": 1.1568627450980393e-07,
      "logits/chosen": -0.68359375,
      "logits/rejected": -0.615234375,
      "logps/chosen": -645.7999877929688,
      "logps/rejected": -1062.4000244140625,
      "loss": 0.6699,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.652148425579071,
      "rewards/margins": 0.5669921636581421,
      "rewards/rejected": -1.21875,
      "step": 60
    },
    {
      "epoch": 0.05102040816326531,
      "grad_norm": 285.90678471040326,
      "learning_rate": 1.2549019607843138e-07,
      "logits/chosen": -0.61328125,
      "logits/rejected": -0.68017578125,
      "logps/chosen": -589.0,
      "logps/rejected": -1323.5999755859375,
      "loss": 0.6803,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.6572265625,
      "rewards/margins": 0.7967529296875,
      "rewards/rejected": -1.453515648841858,
      "step": 65
    },
    {
      "epoch": 0.054945054945054944,
      "grad_norm": 363.0400840313176,
      "learning_rate": 1.352941176470588e-07,
      "logits/chosen": -0.805859386920929,
      "logits/rejected": -0.6656249761581421,
      "logps/chosen": -641.0,
      "logps/rejected": -824.4000244140625,
      "loss": 0.6517,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5804687738418579,
      "rewards/margins": 0.194091796875,
      "rewards/rejected": -0.7738281488418579,
      "step": 70
    },
    {
      "epoch": 0.05886970172684458,
      "grad_norm": 318.73145080709014,
      "learning_rate": 1.4509803921568628e-07,
      "logits/chosen": -0.725390613079071,
      "logits/rejected": -0.6519531011581421,
      "logps/chosen": -578.5999755859375,
      "logps/rejected": -833.5999755859375,
      "loss": 0.6273,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.4833984375,
      "rewards/margins": 0.4777374267578125,
      "rewards/rejected": -0.960156261920929,
      "step": 75
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 326.30421607714834,
      "learning_rate": 1.549019607843137e-07,
      "logits/chosen": -0.877148449420929,
      "logits/rejected": -1.01171875,
      "logps/chosen": -587.5999755859375,
      "logps/rejected": -923.5999755859375,
      "loss": 0.6407,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4662109315395355,
      "rewards/margins": 0.7145019769668579,
      "rewards/rejected": -1.182031273841858,
      "step": 80
    },
    {
      "epoch": 0.06671899529042387,
      "grad_norm": 273.7676420030317,
      "learning_rate": 1.6470588235294117e-07,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.7451171875,
      "logps/chosen": -554.5999755859375,
      "logps/rejected": -969.5999755859375,
      "loss": 0.6422,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4033203125,
      "rewards/margins": 0.59912109375,
      "rewards/rejected": -1.00390625,
      "step": 85
    },
    {
      "epoch": 0.0706436420722135,
      "grad_norm": 276.2713609784365,
      "learning_rate": 1.745098039215686e-07,
      "logits/chosen": -0.7359069585800171,
      "logits/rejected": -0.718945324420929,
      "logps/chosen": -574.0,
      "logps/rejected": -1228.4000244140625,
      "loss": 0.6464,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.4722656309604645,
      "rewards/margins": 1.069970726966858,
      "rewards/rejected": -1.5421874523162842,
      "step": 90
    },
    {
      "epoch": 0.07456828885400314,
      "grad_norm": 444.3108141965041,
      "learning_rate": 1.8431372549019607e-07,
      "logits/chosen": -0.658007800579071,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -607.7999877929688,
      "logps/rejected": -1555.199951171875,
      "loss": 0.7092,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.708984375,
      "rewards/margins": 2.055957078933716,
      "rewards/rejected": -2.76171875,
      "step": 95
    },
    {
      "epoch": 0.07849293563579278,
      "grad_norm": 397.4101593896955,
      "learning_rate": 1.9411764705882353e-07,
      "logits/chosen": -0.7914062738418579,
      "logits/rejected": -0.7640625238418579,
      "logps/chosen": -584.2000122070312,
      "logps/rejected": -1188.0,
      "loss": 0.6429,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.619921863079071,
      "rewards/margins": 0.9295898675918579,
      "rewards/rejected": -1.5499999523162842,
      "step": 100
    },
    {
      "epoch": 0.08241758241758242,
      "grad_norm": 283.0201549502429,
      "learning_rate": 2.0392156862745097e-07,
      "logits/chosen": -0.825634777545929,
      "logits/rejected": -0.6175781488418579,
      "logps/chosen": -598.4000244140625,
      "logps/rejected": -1074.0,
      "loss": 0.6478,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.548632800579071,
      "rewards/margins": 1.2087891101837158,
      "rewards/rejected": -1.7566406726837158,
      "step": 105
    },
    {
      "epoch": 0.08634222919937205,
      "grad_norm": 287.0379189727827,
      "learning_rate": 2.137254901960784e-07,
      "logits/chosen": -0.8001953363418579,
      "logits/rejected": -0.7435547113418579,
      "logps/chosen": -571.7999877929688,
      "logps/rejected": -1074.4000244140625,
      "loss": 0.5771,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.42578125,
      "rewards/margins": 0.983203113079071,
      "rewards/rejected": -1.41015625,
      "step": 110
    },
    {
      "epoch": 0.09026687598116169,
      "grad_norm": 321.93171872472175,
      "learning_rate": 2.235294117647059e-07,
      "logits/chosen": -0.8115234375,
      "logits/rejected": -0.82421875,
      "logps/chosen": -579.5999755859375,
      "logps/rejected": -1218.4000244140625,
      "loss": 0.6258,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4654296934604645,
      "rewards/margins": 1.711523413658142,
      "rewards/rejected": -2.1773438453674316,
      "step": 115
    },
    {
      "epoch": 0.09419152276295134,
      "grad_norm": 300.0281791747476,
      "learning_rate": 2.3333333333333333e-07,
      "logits/chosen": -0.9496093988418579,
      "logits/rejected": -0.552539050579071,
      "logps/chosen": -607.0,
      "logps/rejected": -1024.0,
      "loss": 0.7552,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.540722668170929,
      "rewards/margins": 0.888537585735321,
      "rewards/rejected": -1.4298827648162842,
      "step": 120
    },
    {
      "epoch": 0.09811616954474098,
      "grad_norm": 329.33333376809475,
      "learning_rate": 2.431372549019608e-07,
      "logits/chosen": -0.711718738079071,
      "logits/rejected": -0.750537097454071,
      "logps/chosen": -575.0,
      "logps/rejected": -817.2000122070312,
      "loss": 0.6368,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.524218738079071,
      "rewards/margins": 0.536083996295929,
      "rewards/rejected": -1.0593750476837158,
      "step": 125
    },
    {
      "epoch": 0.10204081632653061,
      "grad_norm": 373.6220849541357,
      "learning_rate": 2.5294117647058823e-07,
      "logits/chosen": -0.849609375,
      "logits/rejected": -0.757031261920929,
      "logps/chosen": -645.5999755859375,
      "logps/rejected": -1260.0,
      "loss": 0.612,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6324218511581421,
      "rewards/margins": 2.3255858421325684,
      "rewards/rejected": -2.9652342796325684,
      "step": 130
    },
    {
      "epoch": 0.10596546310832025,
      "grad_norm": 358.41850674247,
      "learning_rate": 2.6274509803921567e-07,
      "logits/chosen": -0.8316406011581421,
      "logits/rejected": -0.8363281488418579,
      "logps/chosen": -623.0,
      "logps/rejected": -1427.5999755859375,
      "loss": 0.6482,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7017577886581421,
      "rewards/margins": 3.0085082054138184,
      "rewards/rejected": -3.7054686546325684,
      "step": 135
    },
    {
      "epoch": 0.10989010989010989,
      "grad_norm": 316.80003751151435,
      "learning_rate": 2.725490196078431e-07,
      "logits/chosen": -0.7067626714706421,
      "logits/rejected": -0.6484375,
      "logps/chosen": -627.0,
      "logps/rejected": -1434.4000244140625,
      "loss": 0.6312,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.647656261920929,
      "rewards/margins": 4.139452934265137,
      "rewards/rejected": -4.784765720367432,
      "step": 140
    },
    {
      "epoch": 0.11381475667189953,
      "grad_norm": 290.0660051454535,
      "learning_rate": 2.823529411764706e-07,
      "logits/chosen": -0.82421875,
      "logits/rejected": -0.7044922113418579,
      "logps/chosen": -652.2000122070312,
      "logps/rejected": -900.0,
      "loss": 1.0935,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.9491211175918579,
      "rewards/margins": 0.512194812297821,
      "rewards/rejected": -1.462499976158142,
      "step": 145
    },
    {
      "epoch": 0.11773940345368916,
      "grad_norm": 314.4314130075464,
      "learning_rate": 2.9215686274509803e-07,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.6943359375,
      "logps/chosen": -524.4000244140625,
      "logps/rejected": -874.2000122070312,
      "loss": 0.6167,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.17934569716453552,
      "rewards/margins": 1.3493163585662842,
      "rewards/rejected": -1.531152367591858,
      "step": 150
    },
    {
      "epoch": 0.12166405023547881,
      "grad_norm": 294.50171311831446,
      "learning_rate": 3.0196078431372546e-07,
      "logits/chosen": -0.8636718988418579,
      "logits/rejected": -0.652539074420929,
      "logps/chosen": -579.4000244140625,
      "logps/rejected": -773.2000122070312,
      "loss": 0.6286,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.332275390625,
      "rewards/margins": 0.6551513671875,
      "rewards/rejected": -0.9873046875,
      "step": 155
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 252.88916483278535,
      "learning_rate": 3.1176470588235295e-07,
      "logits/chosen": -0.7947753667831421,
      "logits/rejected": -0.6732422113418579,
      "logps/chosen": -615.2000122070312,
      "logps/rejected": -1618.800048828125,
      "loss": 0.6639,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.663281261920929,
      "rewards/margins": 3.423046827316284,
      "rewards/rejected": -4.079687595367432,
      "step": 160
    },
    {
      "epoch": 0.12951334379905807,
      "grad_norm": 275.98194349762656,
      "learning_rate": 3.215686274509804e-07,
      "logits/chosen": -0.771289050579071,
      "logits/rejected": -0.734375,
      "logps/chosen": -552.0,
      "logps/rejected": -1398.0,
      "loss": 0.5767,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.545117199420929,
      "rewards/margins": 2.964062452316284,
      "rewards/rejected": -3.5093750953674316,
      "step": 165
    },
    {
      "epoch": 0.13343799058084774,
      "grad_norm": 271.17620415083053,
      "learning_rate": 3.313725490196078e-07,
      "logits/chosen": -0.7646484375,
      "logits/rejected": -0.7806640863418579,
      "logps/chosen": -609.4000244140625,
      "logps/rejected": -1800.0,
      "loss": 0.6012,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.693164050579071,
      "rewards/margins": 5.034765720367432,
      "rewards/rejected": -5.727343559265137,
      "step": 170
    },
    {
      "epoch": 0.13736263736263737,
      "grad_norm": 256.75081616624254,
      "learning_rate": 3.411764705882353e-07,
      "logits/chosen": -0.7586914300918579,
      "logits/rejected": -0.8060547113418579,
      "logps/chosen": -593.5999755859375,
      "logps/rejected": -1155.199951171875,
      "loss": 0.6383,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.527148425579071,
      "rewards/margins": 2.0811219215393066,
      "rewards/rejected": -2.602343797683716,
      "step": 175
    },
    {
      "epoch": 0.141287284144427,
      "grad_norm": 269.7054715248193,
      "learning_rate": 3.509803921568627e-07,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.7914062738418579,
      "logps/chosen": -536.0,
      "logps/rejected": -1013.5999755859375,
      "loss": 0.6478,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.17219848930835724,
      "rewards/margins": 2.3841185569763184,
      "rewards/rejected": -2.552197217941284,
      "step": 180
    },
    {
      "epoch": 0.14521193092621665,
      "grad_norm": 246.74652246086606,
      "learning_rate": 3.607843137254902e-07,
      "logits/chosen": -0.808789074420929,
      "logits/rejected": -0.6083984375,
      "logps/chosen": -576.0,
      "logps/rejected": -1140.0,
      "loss": 0.76,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.345703125,
      "rewards/margins": 2.107421875,
      "rewards/rejected": -2.4569334983825684,
      "step": 185
    },
    {
      "epoch": 0.14913657770800628,
      "grad_norm": 363.3764528545936,
      "learning_rate": 3.705882352941176e-07,
      "logits/chosen": -0.8529297113418579,
      "logits/rejected": -0.8177734613418579,
      "logps/chosen": -613.2000122070312,
      "logps/rejected": -1315.199951171875,
      "loss": 1.0447,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.768505871295929,
      "rewards/margins": 2.6612305641174316,
      "rewards/rejected": -3.4263672828674316,
      "step": 190
    },
    {
      "epoch": 0.15306122448979592,
      "grad_norm": 286.27553378423505,
      "learning_rate": 3.8039215686274506e-07,
      "logits/chosen": -0.732714831829071,
      "logits/rejected": -0.7886718511581421,
      "logps/chosen": -586.7999877929688,
      "logps/rejected": -858.0,
      "loss": 0.606,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5263671875,
      "rewards/margins": 0.698535144329071,
      "rewards/rejected": -1.2253906726837158,
      "step": 195
    },
    {
      "epoch": 0.15698587127158556,
      "grad_norm": 294.9473324971511,
      "learning_rate": 3.9019607843137255e-07,
      "logits/chosen": -0.7537841796875,
      "logits/rejected": -0.58203125,
      "logps/chosen": -566.5999755859375,
      "logps/rejected": -1228.0,
      "loss": 0.6602,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.447265625,
      "rewards/margins": 2.4923095703125,
      "rewards/rejected": -2.9378905296325684,
      "step": 200
    },
    {
      "epoch": 0.1609105180533752,
      "grad_norm": 1660.6445913019616,
      "learning_rate": 4e-07,
      "logits/chosen": -0.7925781011581421,
      "logits/rejected": -0.583691418170929,
      "logps/chosen": -603.5999755859375,
      "logps/rejected": -1099.5999755859375,
      "loss": 0.9399,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4469238221645355,
      "rewards/margins": 2.514404296875,
      "rewards/rejected": -2.959179639816284,
      "step": 205
    },
    {
      "epoch": 0.16483516483516483,
      "grad_norm": 273.7387795931627,
      "learning_rate": 4.098039215686274e-07,
      "logits/chosen": -0.6646972894668579,
      "logits/rejected": -0.5250488519668579,
      "logps/chosen": -522.7999877929688,
      "logps/rejected": -956.0,
      "loss": 0.5348,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3116455078125,
      "rewards/margins": 1.844824194908142,
      "rewards/rejected": -2.159960985183716,
      "step": 210
    },
    {
      "epoch": 0.16875981161695447,
      "grad_norm": 587.2379199542737,
      "learning_rate": 4.196078431372549e-07,
      "logits/chosen": -0.534863293170929,
      "logits/rejected": -0.302490234375,
      "logps/chosen": -733.2000122070312,
      "logps/rejected": -1077.199951171875,
      "loss": 0.7713,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.4500000476837158,
      "rewards/margins": 1.783789038658142,
      "rewards/rejected": -3.2328124046325684,
      "step": 215
    },
    {
      "epoch": 0.1726844583987441,
      "grad_norm": 449.79556065161387,
      "learning_rate": 4.294117647058823e-07,
      "logits/chosen": -0.858203113079071,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -599.7999877929688,
      "logps/rejected": -1282.4000244140625,
      "loss": 0.606,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5912109613418579,
      "rewards/margins": 2.364941358566284,
      "rewards/rejected": -2.9576172828674316,
      "step": 220
    },
    {
      "epoch": 0.17660910518053374,
      "grad_norm": 275.4695285622311,
      "learning_rate": 4.392156862745098e-07,
      "logits/chosen": -0.928515613079071,
      "logits/rejected": -0.846875011920929,
      "logps/chosen": -559.4000244140625,
      "logps/rejected": -1283.5999755859375,
      "loss": 0.7063,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.01401367224752903,
      "rewards/margins": 2.22900390625,
      "rewards/rejected": -2.211343288421631,
      "step": 225
    },
    {
      "epoch": 0.18053375196232338,
      "grad_norm": 324.5416359813328,
      "learning_rate": 4.4901960784313727e-07,
      "logits/chosen": -0.901562511920929,
      "logits/rejected": -0.76953125,
      "logps/chosen": -511.20001220703125,
      "logps/rejected": -1008.0,
      "loss": 0.6912,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.02666015550494194,
      "rewards/margins": 1.14892578125,
      "rewards/rejected": -1.1226074695587158,
      "step": 230
    },
    {
      "epoch": 0.18445839874411302,
      "grad_norm": 599.2933607412381,
      "learning_rate": 4.5882352941176465e-07,
      "logits/chosen": -0.5595703125,
      "logits/rejected": -0.742968738079071,
      "logps/chosen": -620.0,
      "logps/rejected": -1565.199951171875,
      "loss": 0.7198,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.798828125,
      "rewards/margins": 3.500781297683716,
      "rewards/rejected": -4.299609184265137,
      "step": 235
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 453.42789134731476,
      "learning_rate": 4.6862745098039214e-07,
      "logits/chosen": -0.7710937261581421,
      "logits/rejected": -0.6927734613418579,
      "logps/chosen": -611.0,
      "logps/rejected": -1336.4000244140625,
      "loss": 0.7769,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.717968761920929,
      "rewards/margins": 3.280322313308716,
      "rewards/rejected": -3.9984374046325684,
      "step": 240
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 267.98410103235364,
      "learning_rate": 4.784313725490196e-07,
      "logits/chosen": -0.8500000238418579,
      "logits/rejected": -0.7671874761581421,
      "logps/chosen": -540.4000244140625,
      "logps/rejected": -1352.4000244140625,
      "loss": 0.5741,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09402160346508026,
      "rewards/margins": 3.9405274391174316,
      "rewards/rejected": -4.039355278015137,
      "step": 245
    },
    {
      "epoch": 0.19623233908948196,
      "grad_norm": 707.4705816753057,
      "learning_rate": 4.88235294117647e-07,
      "logits/chosen": -0.4747558534145355,
      "logits/rejected": -0.5716797113418579,
      "logps/chosen": -622.4000244140625,
      "logps/rejected": -1199.199951171875,
      "loss": 0.756,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.887499988079071,
      "rewards/margins": 2.986572265625,
      "rewards/rejected": -3.8804688453674316,
      "step": 250
    },
    {
      "epoch": 0.2001569858712716,
      "grad_norm": 311.25367892275995,
      "learning_rate": 4.980392156862744e-07,
      "logits/chosen": -0.790820300579071,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -616.0,
      "logps/rejected": -1541.199951171875,
      "loss": 0.795,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.924609363079071,
      "rewards/margins": 5.010546684265137,
      "rewards/rejected": -5.932421684265137,
      "step": 255
    },
    {
      "epoch": 0.20408163265306123,
      "grad_norm": 273.2504002042366,
      "learning_rate": 4.999962457728629e-07,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.883984386920929,
      "logps/chosen": -545.5999755859375,
      "logps/rejected": -1814.199951171875,
      "loss": 0.6524,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.01232299767434597,
      "rewards/margins": 7.173828125,
      "rewards/rejected": -7.188500881195068,
      "step": 260
    },
    {
      "epoch": 0.20800627943485087,
      "grad_norm": 386.0703772351185,
      "learning_rate": 4.999809944183633e-07,
      "logits/chosen": -0.965624988079071,
      "logits/rejected": -0.806445300579071,
      "logps/chosen": -555.5999755859375,
      "logps/rejected": -1182.0,
      "loss": 0.5838,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.17722168564796448,
      "rewards/margins": 3.8692383766174316,
      "rewards/rejected": -4.051953315734863,
      "step": 265
    },
    {
      "epoch": 0.2119309262166405,
      "grad_norm": 363.38945168397976,
      "learning_rate": 4.999540120124667e-07,
      "logits/chosen": -0.9228515625,
      "logits/rejected": -0.827929675579071,
      "logps/chosen": -680.5999755859375,
      "logps/rejected": -1344.0,
      "loss": 1.4162,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.4054687023162842,
      "rewards/margins": 4.409570217132568,
      "rewards/rejected": -5.814062595367432,
      "step": 270
    },
    {
      "epoch": 0.21585557299843014,
      "grad_norm": 349.2863159938502,
      "learning_rate": 4.999152998213973e-07,
      "logits/chosen": -0.8363281488418579,
      "logits/rejected": -0.836132824420929,
      "logps/chosen": -617.2000122070312,
      "logps/rejected": -1427.5999755859375,
      "loss": 0.8933,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.9867187738418579,
      "rewards/margins": 4.2177734375,
      "rewards/rejected": -5.197656154632568,
      "step": 275
    },
    {
      "epoch": 0.21978021978021978,
      "grad_norm": 341.3050310143687,
      "learning_rate": 4.998648596618319e-07,
      "logits/chosen": -0.6988281011581421,
      "logits/rejected": -0.731640636920929,
      "logps/chosen": -542.4000244140625,
      "logps/rejected": -1192.800048828125,
      "loss": 0.6889,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.3380371034145355,
      "rewards/margins": 2.8482422828674316,
      "rewards/rejected": -3.184765577316284,
      "step": 280
    },
    {
      "epoch": 0.2237048665620094,
      "grad_norm": 264.13447191753863,
      "learning_rate": 4.998026939008151e-07,
      "logits/chosen": -0.794726550579071,
      "logits/rejected": -0.8296874761581421,
      "logps/chosen": -558.2000122070312,
      "logps/rejected": -1818.800048828125,
      "loss": 0.7419,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.11296387016773224,
      "rewards/margins": 4.834228515625,
      "rewards/rejected": -4.951464653015137,
      "step": 285
    },
    {
      "epoch": 0.22762951334379905,
      "grad_norm": 235.90163210955296,
      "learning_rate": 4.997288054556473e-07,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.938281238079071,
      "logps/chosen": -568.2000122070312,
      "logps/rejected": -1393.199951171875,
      "loss": 0.6146,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.009814453311264515,
      "rewards/margins": 3.017285108566284,
      "rewards/rejected": -3.0252442359924316,
      "step": 290
    },
    {
      "epoch": 0.2315541601255887,
      "grad_norm": 487.7633396211925,
      "learning_rate": 4.996431977937491e-07,
      "logits/chosen": -0.751171886920929,
      "logits/rejected": -0.7835937738418579,
      "logps/chosen": -575.5999755859375,
      "logps/rejected": -1507.199951171875,
      "loss": 0.7426,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.57940673828125,
      "rewards/margins": 3.4756836891174316,
      "rewards/rejected": -4.058984279632568,
      "step": 295
    },
    {
      "epoch": 0.23547880690737832,
      "grad_norm": 323.47123384894564,
      "learning_rate": 4.995458749324971e-07,
      "logits/chosen": -0.8770507574081421,
      "logits/rejected": -0.8338867425918579,
      "logps/chosen": -565.0,
      "logps/rejected": -1039.199951171875,
      "loss": 0.8031,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.43232423067092896,
      "rewards/margins": 1.32568359375,
      "rewards/rejected": -1.758203148841858,
      "step": 300
    },
    {
      "epoch": 0.239403453689168,
      "grad_norm": 270.8103356776693,
      "learning_rate": 4.994368414390367e-07,
      "logits/chosen": -1.0314452648162842,
      "logits/rejected": -0.991406261920929,
      "logps/chosen": -572.0,
      "logps/rejected": -1198.800048828125,
      "loss": 0.6641,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.02260742150247097,
      "rewards/margins": 2.2123045921325684,
      "rewards/rejected": -2.2328734397888184,
      "step": 305
    },
    {
      "epoch": 0.24332810047095763,
      "grad_norm": 288.0445096652215,
      "learning_rate": 4.993161024300668e-07,
      "logits/chosen": -1.046875,
      "logits/rejected": -0.885937511920929,
      "logps/chosen": -533.0,
      "logps/rejected": -1134.800048828125,
      "loss": 0.5854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.292236328125,
      "rewards/margins": 1.783288598060608,
      "rewards/rejected": -2.0785155296325684,
      "step": 310
    },
    {
      "epoch": 0.24725274725274726,
      "grad_norm": 382.4663872065362,
      "learning_rate": 4.991836635716002e-07,
      "logits/chosen": -0.8199218511581421,
      "logits/rejected": -0.790234386920929,
      "logps/chosen": -664.7999877929688,
      "logps/rejected": -1738.0,
      "loss": 0.6363,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.0265624523162842,
      "rewards/margins": 4.701171875,
      "rewards/rejected": -5.73828125,
      "step": 315
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 294.9353777509841,
      "learning_rate": 4.990395310786977e-07,
      "logits/chosen": -0.7574218511581421,
      "logits/rejected": -0.49140626192092896,
      "logps/chosen": -599.2000122070312,
      "logps/rejected": -1340.0,
      "loss": 0.7501,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7625732421875,
      "rewards/margins": 2.9893555641174316,
      "rewards/rejected": -3.755078077316284,
      "step": 320
    },
    {
      "epoch": 0.25510204081632654,
      "grad_norm": 229.35868214070382,
      "learning_rate": 4.988837117151762e-07,
      "logits/chosen": -1.16015625,
      "logits/rejected": -1.0183594226837158,
      "logps/chosen": -543.0,
      "logps/rejected": -1171.199951171875,
      "loss": 0.6229,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02496337890625,
      "rewards/margins": 2.74737548828125,
      "rewards/rejected": -2.722268581390381,
      "step": 325
    },
    {
      "epoch": 0.25902668759811615,
      "grad_norm": 420.41123905203597,
      "learning_rate": 4.987162127932915e-07,
      "logits/chosen": -0.996874988079071,
      "logits/rejected": -0.829296886920929,
      "logps/chosen": -638.2000122070312,
      "logps/rejected": -1208.4000244140625,
      "loss": 0.8938,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6075195074081421,
      "rewards/margins": 3.18829345703125,
      "rewards/rejected": -3.7974610328674316,
      "step": 330
    },
    {
      "epoch": 0.2629513343799058,
      "grad_norm": 445.68414071276385,
      "learning_rate": 4.985370421733947e-07,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.9085937738418579,
      "logps/chosen": -643.2000122070312,
      "logps/rejected": -1869.5999755859375,
      "loss": 0.8261,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.908007800579071,
      "rewards/margins": 4.804883003234863,
      "rewards/rejected": -5.716796875,
      "step": 335
    },
    {
      "epoch": 0.2668759811616955,
      "grad_norm": 3230.561548002655,
      "learning_rate": 4.983462082635645e-07,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.7763671875,
      "logps/chosen": -881.0,
      "logps/rejected": -1303.199951171875,
      "loss": 1.7682,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.4846680164337158,
      "rewards/margins": 1.897070288658142,
      "rewards/rejected": -3.3905029296875,
      "step": 340
    },
    {
      "epoch": 0.2708006279434851,
      "grad_norm": 282.7319769352763,
      "learning_rate": 4.981437200192111e-07,
      "logits/chosen": -0.8480468988418579,
      "logits/rejected": -0.7740234136581421,
      "logps/chosen": -597.0,
      "logps/rejected": -1562.0,
      "loss": 0.743,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3536376953125,
      "rewards/margins": 4.258624076843262,
      "rewards/rejected": -4.605535984039307,
      "step": 345
    },
    {
      "epoch": 0.27472527472527475,
      "grad_norm": 2875.1565859787693,
      "learning_rate": 4.979295869426567e-07,
      "logits/chosen": -1.017187476158142,
      "logits/rejected": -0.9554687738418579,
      "logps/chosen": -703.5999755859375,
      "logps/rejected": -1348.0,
      "loss": 1.3944,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.19765624403953552,
      "rewards/margins": 3.201367139816284,
      "rewards/rejected": -2.99755859375,
      "step": 350
    },
    {
      "epoch": 0.27864992150706436,
      "grad_norm": 366.3840006437179,
      "learning_rate": 4.9770381908269e-07,
      "logits/chosen": -0.8101562261581421,
      "logits/rejected": -0.734375,
      "logps/chosen": -550.7999877929688,
      "logps/rejected": -1122.4000244140625,
      "loss": 0.7218,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.353515625,
      "rewards/margins": 3.2555174827575684,
      "rewards/rejected": -3.6089844703674316,
      "step": 355
    },
    {
      "epoch": 0.282574568288854,
      "grad_norm": 622.3198434223275,
      "learning_rate": 4.974664270340938e-07,
      "logits/chosen": -0.8558593988418579,
      "logits/rejected": -0.690234363079071,
      "logps/chosen": -631.2000122070312,
      "logps/rejected": -1258.4000244140625,
      "loss": 0.8599,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0007812976837158,
      "rewards/margins": 3.776562452316284,
      "rewards/rejected": -4.772656440734863,
      "step": 360
    },
    {
      "epoch": 0.28649921507064363,
      "grad_norm": 272.2433601050908,
      "learning_rate": 4.972174219371483e-07,
      "logits/chosen": -0.846875011920929,
      "logits/rejected": -0.7345947027206421,
      "logps/chosen": -531.4000244140625,
      "logps/rejected": -933.5999755859375,
      "loss": 0.764,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.3663574159145355,
      "rewards/margins": 0.6440185308456421,
      "rewards/rejected": -1.0105469226837158,
      "step": 365
    },
    {
      "epoch": 0.2904238618524333,
      "grad_norm": 337.65866747470125,
      "learning_rate": 4.969568154771084e-07,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.939257800579071,
      "logps/chosen": -532.2000122070312,
      "logps/rejected": -1138.4000244140625,
      "loss": 0.73,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07926330715417862,
      "rewards/margins": 3.8841552734375,
      "rewards/rejected": -3.9633851051330566,
      "step": 370
    },
    {
      "epoch": 0.2943485086342229,
      "grad_norm": 306.8222528225917,
      "learning_rate": 4.966846198836546e-07,
      "logits/chosen": -0.9039062261581421,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -592.5999755859375,
      "logps/rejected": -1002.4000244140625,
      "loss": 0.697,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.46357423067092896,
      "rewards/margins": 2.0025634765625,
      "rewards/rejected": -2.469921827316284,
      "step": 375
    },
    {
      "epoch": 0.29827315541601257,
      "grad_norm": 308.63013530474944,
      "learning_rate": 4.964008479303201e-07,
      "logits/chosen": -0.8902343511581421,
      "logits/rejected": -0.7977539300918579,
      "logps/chosen": -653.5999755859375,
      "logps/rejected": -1828.0,
      "loss": 0.6599,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.77001953125,
      "rewards/margins": 7.347070217132568,
      "rewards/rejected": -8.096875190734863,
      "step": 380
    },
    {
      "epoch": 0.3021978021978022,
      "grad_norm": 303.84487501348246,
      "learning_rate": 4.961055129338909e-07,
      "logits/chosen": -0.9593750238418579,
      "logits/rejected": -0.9517577886581421,
      "logps/chosen": -709.7999877929688,
      "logps/rejected": -1726.0,
      "loss": 1.8301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.588476538658142,
      "rewards/margins": 4.822656154632568,
      "rewards/rejected": -6.411913871765137,
      "step": 385
    },
    {
      "epoch": 0.30612244897959184,
      "grad_norm": 329.03810353241545,
      "learning_rate": 4.957986287537806e-07,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -1.0478515625,
      "logps/chosen": -529.4000244140625,
      "logps/rejected": -1189.199951171875,
      "loss": 0.8585,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.15141601860523224,
      "rewards/margins": 0.929736316204071,
      "rewards/rejected": -1.0779297351837158,
      "step": 390
    },
    {
      "epoch": 0.31004709576138145,
      "grad_norm": 161.87882838816964,
      "learning_rate": 4.954802097913804e-07,
      "logits/chosen": -1.107812523841858,
      "logits/rejected": -1.0320312976837158,
      "logps/chosen": -567.4000244140625,
      "logps/rejected": -1369.5999755859375,
      "loss": 1.1442,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.05150146409869194,
      "rewards/margins": 1.7119629383087158,
      "rewards/rejected": -1.6647460460662842,
      "step": 395
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 188.38062568259156,
      "learning_rate": 4.95150270989383e-07,
      "logits/chosen": -1.06640625,
      "logits/rejected": -1.105859398841858,
      "logps/chosen": -561.7999877929688,
      "logps/rejected": -1457.5999755859375,
      "loss": 0.7362,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006420898251235485,
      "rewards/margins": 3.827685594558716,
      "rewards/rejected": -3.816699266433716,
      "step": 400
    },
    {
      "epoch": 0.3178963893249607,
      "grad_norm": 370.3309477664289,
      "learning_rate": 4.948088278310819e-07,
      "logits/chosen": -0.994140625,
      "logits/rejected": -0.751757800579071,
      "logps/chosen": -603.4000244140625,
      "logps/rejected": -1836.0,
      "loss": 0.6836,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6573730707168579,
      "rewards/margins": 5.8603515625,
      "rewards/rejected": -6.511328220367432,
      "step": 405
    },
    {
      "epoch": 0.3218210361067504,
      "grad_norm": 272.86479574717,
      "learning_rate": 4.94455896339644e-07,
      "logits/chosen": -0.925585925579071,
      "logits/rejected": -0.9310547113418579,
      "logps/chosen": -628.0,
      "logps/rejected": -1905.199951171875,
      "loss": 0.7483,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.64727783203125,
      "rewards/margins": 6.256543159484863,
      "rewards/rejected": -6.889843940734863,
      "step": 410
    },
    {
      "epoch": 0.32574568288854006,
      "grad_norm": 288.77868132686604,
      "learning_rate": 4.940914930773583e-07,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.9156249761581421,
      "logps/chosen": -577.7999877929688,
      "logps/rejected": -1159.800048828125,
      "loss": 0.6352,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0064376830123364925,
      "rewards/margins": 3.628222703933716,
      "rewards/rejected": -3.6314454078674316,
      "step": 415
    },
    {
      "epoch": 0.32967032967032966,
      "grad_norm": 260.7540525148575,
      "learning_rate": 4.937156351448585e-07,
      "logits/chosen": -0.8832031488418579,
      "logits/rejected": -0.8255859613418579,
      "logps/chosen": -576.0,
      "logps/rejected": -1759.5999755859375,
      "loss": 0.6843,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14577636122703552,
      "rewards/margins": 6.726953029632568,
      "rewards/rejected": -6.876757621765137,
      "step": 420
    },
    {
      "epoch": 0.33359497645211933,
      "grad_norm": 356.72617019617803,
      "learning_rate": 4.933283401803202e-07,
      "logits/chosen": -0.8539062738418579,
      "logits/rejected": -0.7740234136581421,
      "logps/chosen": -673.4000244140625,
      "logps/rejected": -2036.4000244140625,
      "loss": 0.7117,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.135107398033142,
      "rewards/margins": 7.883593559265137,
      "rewards/rejected": -9.01171875,
      "step": 425
    },
    {
      "epoch": 0.33751962323390894,
      "grad_norm": 369.17477933410345,
      "learning_rate": 4.929296263586339e-07,
      "logits/chosen": -0.8042968511581421,
      "logits/rejected": -0.728515625,
      "logps/chosen": -671.2000122070312,
      "logps/rejected": -1152.0,
      "loss": 1.0829,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.488671898841858,
      "rewards/margins": 2.67529296875,
      "rewards/rejected": -4.167187690734863,
      "step": 430
    },
    {
      "epoch": 0.3414442700156986,
      "grad_norm": 342.53370956015306,
      "learning_rate": 4.925195123905512e-07,
      "logits/chosen": -0.8187500238418579,
      "logits/rejected": -0.760546863079071,
      "logps/chosen": -592.2000122070312,
      "logps/rejected": -712.7999877929688,
      "loss": 0.8659,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.317626953125,
      "rewards/margins": 0.15390625596046448,
      "rewards/rejected": -0.47175294160842896,
      "step": 435
    },
    {
      "epoch": 0.3453689167974882,
      "grad_norm": 316.2669761466369,
      "learning_rate": 4.920980175218074e-07,
      "logits/chosen": -1.0656249523162842,
      "logits/rejected": -0.8324218988418579,
      "logps/chosen": -529.5999755859375,
      "logps/rejected": -944.5999755859375,
      "loss": 0.7786,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.10061035305261612,
      "rewards/margins": 2.2132811546325684,
      "rewards/rejected": -2.3133788108825684,
      "step": 440
    },
    {
      "epoch": 0.3492935635792779,
      "grad_norm": 259.4097116651841,
      "learning_rate": 4.916651615322181e-07,
      "logits/chosen": -0.8226562738418579,
      "logits/rejected": -0.595654308795929,
      "logps/chosen": -645.4000244140625,
      "logps/rejected": -1304.0,
      "loss": 0.7435,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0554687976837158,
      "rewards/margins": 3.744311571121216,
      "rewards/rejected": -4.798828125,
      "step": 445
    },
    {
      "epoch": 0.3532182103610675,
      "grad_norm": 963.8252720076981,
      "learning_rate": 4.91220964734751e-07,
      "logits/chosen": -0.876953125,
      "logits/rejected": -0.8750976324081421,
      "logps/chosen": -583.5999755859375,
      "logps/rejected": -1125.5999755859375,
      "loss": 0.9044,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.879638671875,
      "rewards/margins": 2.414013624191284,
      "rewards/rejected": -3.2876954078674316,
      "step": 450
    },
    {
      "epoch": 0.35714285714285715,
      "grad_norm": 213.58189055354018,
      "learning_rate": 4.907654479745725e-07,
      "logits/chosen": -1.0828125476837158,
      "logits/rejected": -0.925000011920929,
      "logps/chosen": -558.5999755859375,
      "logps/rejected": -805.2000122070312,
      "loss": 0.8811,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.20045165717601776,
      "rewards/margins": 0.7815917730331421,
      "rewards/rejected": -0.982617199420929,
      "step": 455
    },
    {
      "epoch": 0.36106750392464676,
      "grad_norm": 236.42406239959158,
      "learning_rate": 4.9029863262807e-07,
      "logits/chosen": -1.1531250476837158,
      "logits/rejected": -1.0167968273162842,
      "logps/chosen": -543.0,
      "logps/rejected": -1326.800048828125,
      "loss": 0.6618,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.28864747285842896,
      "rewards/margins": 3.3189454078674316,
      "rewards/rejected": -3.6043944358825684,
      "step": 460
    },
    {
      "epoch": 0.3649921507064364,
      "grad_norm": 595.3267958804568,
      "learning_rate": 4.89820540601848e-07,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.798828125,
      "logps/chosen": -694.7999877929688,
      "logps/rejected": -1132.4000244140625,
      "loss": 1.0159,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 2.3833250999450684,
      "rewards/rejected": -3.811718702316284,
      "step": 465
    },
    {
      "epoch": 0.36891679748822603,
      "grad_norm": 264.004259527748,
      "learning_rate": 4.893311943317007e-07,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.947265625,
      "logps/chosen": -649.5999755859375,
      "logps/rejected": -2252.39990234375,
      "loss": 0.8053,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0626952648162842,
      "rewards/margins": 9.125683784484863,
      "rewards/rejected": -10.189453125,
      "step": 470
    },
    {
      "epoch": 0.3728414442700157,
      "grad_norm": 216.82928160907792,
      "learning_rate": 4.88830616781559e-07,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.969531238079071,
      "logps/chosen": -572.0,
      "logps/rejected": -1562.4000244140625,
      "loss": 0.7727,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.523193359375,
      "rewards/margins": 7.296582221984863,
      "rewards/rejected": -7.820508003234863,
      "step": 475
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 1156.364814935398,
      "learning_rate": 4.883188314424125e-07,
      "logits/chosen": -1.0027344226837158,
      "logits/rejected": -1.029687523841858,
      "logps/chosen": -570.0,
      "logps/rejected": -1293.5999755859375,
      "loss": 0.7663,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7353515625,
      "rewards/margins": 4.598925590515137,
      "rewards/rejected": -5.339062690734863,
      "step": 480
    },
    {
      "epoch": 0.38069073783359497,
      "grad_norm": 339.73808141945983,
      "learning_rate": 4.877958623312075e-07,
      "logits/chosen": -0.8853515386581421,
      "logits/rejected": -0.815234363079071,
      "logps/chosen": -618.0,
      "logps/rejected": -1814.0,
      "loss": 0.6026,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.018164038658142,
      "rewards/margins": 9.2734375,
      "rewards/rejected": -10.276562690734863,
      "step": 485
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 259.8659125966099,
      "learning_rate": 4.872617339897197e-07,
      "logits/chosen": -0.9164062738418579,
      "logits/rejected": -0.6982421875,
      "logps/chosen": -614.5999755859375,
      "logps/rejected": -1255.5999755859375,
      "loss": 0.7775,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8882812261581421,
      "rewards/margins": 2.587109327316284,
      "rewards/rejected": -3.4652342796325684,
      "step": 490
    },
    {
      "epoch": 0.38854003139717425,
      "grad_norm": 268.61927398585664,
      "learning_rate": 4.867164714834029e-07,
      "logits/chosen": -0.9410156011581421,
      "logits/rejected": -0.837695300579071,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -1368.0,
      "loss": 0.7588,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3047729432582855,
      "rewards/margins": 4.621172904968262,
      "rewards/rejected": -4.917578220367432,
      "step": 495
    },
    {
      "epoch": 0.3924646781789639,
      "grad_norm": 290.79655756208746,
      "learning_rate": 4.861601004002124e-07,
      "logits/chosen": -1.0378906726837158,
      "logits/rejected": -0.9710937738418579,
      "logps/chosen": -663.7999877929688,
      "logps/rejected": -1743.5999755859375,
      "loss": 0.6985,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.793749988079071,
      "rewards/margins": 6.474023342132568,
      "rewards/rejected": -7.251172065734863,
      "step": 500
    },
    {
      "epoch": 0.3963893249607535,
      "grad_norm": 369.83021882754997,
      "learning_rate": 4.855926468494039e-07,
      "logits/chosen": -0.7408202886581421,
      "logits/rejected": -0.6973632574081421,
      "logps/chosen": -675.2000122070312,
      "logps/rejected": -2061.60009765625,
      "loss": 0.6686,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.407812476158142,
      "rewards/margins": 10.521020889282227,
      "rewards/rejected": -11.927343368530273,
      "step": 505
    },
    {
      "epoch": 0.4003139717425432,
      "grad_norm": 395.84694151552645,
      "learning_rate": 4.85014137460309e-07,
      "logits/chosen": -0.9693359136581421,
      "logits/rejected": -0.916796863079071,
      "logps/chosen": -783.0,
      "logps/rejected": -2390.800048828125,
      "loss": 1.7314,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.859765648841858,
      "rewards/margins": 13.375781059265137,
      "rewards/rejected": -15.284375190734863,
      "step": 510
    },
    {
      "epoch": 0.4042386185243328,
      "grad_norm": 316.478235561291,
      "learning_rate": 4.844245993810852e-07,
      "logits/chosen": -1.075781226158142,
      "logits/rejected": -1.0390625,
      "logps/chosen": -549.4000244140625,
      "logps/rejected": -2257.199951171875,
      "loss": 0.7685,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.13803711533546448,
      "rewards/margins": 9.566259384155273,
      "rewards/rejected": -9.427624702453613,
      "step": 515
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 296.5861548044356,
      "learning_rate": 4.838240602774415e-07,
      "logits/chosen": -1.1609375476837158,
      "logits/rejected": -0.9276367425918579,
      "logps/chosen": -497.20001220703125,
      "logps/rejected": -1052.800048828125,
      "loss": 0.7525,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.14340820908546448,
      "rewards/margins": 3.2542967796325684,
      "rewards/rejected": -3.108203172683716,
      "step": 520
    },
    {
      "epoch": 0.41208791208791207,
      "grad_norm": 336.7246694338917,
      "learning_rate": 4.832125483313411e-07,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -1.0244140625,
      "logps/chosen": -573.5999755859375,
      "logps/rejected": -1751.5999755859375,
      "loss": 0.7926,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7437499761581421,
      "rewards/margins": 4.339453220367432,
      "rewards/rejected": -5.078711032867432,
      "step": 525
    },
    {
      "epoch": 0.41601255886970173,
      "grad_norm": 294.1556856223044,
      "learning_rate": 4.825900922396774e-07,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -1.0222656726837158,
      "logps/chosen": -648.4000244140625,
      "logps/rejected": -1959.5999755859375,
      "loss": 0.793,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.107812523841858,
      "rewards/margins": 6.189550876617432,
      "rewards/rejected": -7.29296875,
      "step": 530
    },
    {
      "epoch": 0.41993720565149134,
      "grad_norm": 337.3277698661428,
      "learning_rate": 4.819567212129291e-07,
      "logits/chosen": -0.9136718511581421,
      "logits/rejected": -0.8441406488418579,
      "logps/chosen": -639.5999755859375,
      "logps/rejected": -1329.5999755859375,
      "loss": 1.2094,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.244531273841858,
      "rewards/margins": 4.271191596984863,
      "rewards/rejected": -5.514843940734863,
      "step": 535
    },
    {
      "epoch": 0.423861852433281,
      "grad_norm": 1258.3299505336577,
      "learning_rate": 4.813124649737878e-07,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.8310546875,
      "logps/chosen": -550.7999877929688,
      "logps/rejected": -1403.5999755859375,
      "loss": 0.7282,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.32342529296875,
      "rewards/margins": 4.252343654632568,
      "rewards/rejected": -4.58203125,
      "step": 540
    },
    {
      "epoch": 0.42778649921507067,
      "grad_norm": 234.81129410833725,
      "learning_rate": 4.806573537557642e-07,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -0.9750000238418579,
      "logps/chosen": -596.5999755859375,
      "logps/rejected": -1854.800048828125,
      "loss": 0.7931,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4482421875,
      "rewards/margins": 7.752783298492432,
      "rewards/rejected": -8.206640243530273,
      "step": 545
    },
    {
      "epoch": 0.4317111459968603,
      "grad_norm": 437.0334874933319,
      "learning_rate": 4.79991418301769e-07,
      "logits/chosen": -1.021875023841858,
      "logits/rejected": -1.0011718273162842,
      "logps/chosen": -567.4000244140625,
      "logps/rejected": -819.2000122070312,
      "loss": 0.7744,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.4495605528354645,
      "rewards/margins": 0.4500976502895355,
      "rewards/rejected": -0.8994140625,
      "step": 550
    },
    {
      "epoch": 0.43563579277864994,
      "grad_norm": 693.5746776002596,
      "learning_rate": 4.793146898626701e-07,
      "logits/chosen": -0.8248046636581421,
      "logits/rejected": -0.6751953363418579,
      "logps/chosen": -675.5999755859375,
      "logps/rejected": -2172.800048828125,
      "loss": 0.715,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.153906226158142,
      "rewards/margins": 7.668066501617432,
      "rewards/rejected": -8.826952934265137,
      "step": 555
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 223.20450272113177,
      "learning_rate": 4.78627200195826e-07,
      "logits/chosen": -0.937304675579071,
      "logits/rejected": -0.76123046875,
      "logps/chosen": -651.5999755859375,
      "logps/rejected": -1396.4000244140625,
      "loss": 0.7269,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.09814453125,
      "rewards/margins": 3.3218750953674316,
      "rewards/rejected": -4.420312404632568,
      "step": 560
    },
    {
      "epoch": 0.4434850863422292,
      "grad_norm": 419.29134052563506,
      "learning_rate": 4.77928981563596e-07,
      "logits/chosen": -1.0964844226837158,
      "logits/rejected": -0.9609375,
      "logps/chosen": -544.2000122070312,
      "logps/rejected": -1392.800048828125,
      "loss": 0.7812,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.4098144471645355,
      "rewards/margins": 4.203515529632568,
      "rewards/rejected": -4.615332126617432,
      "step": 565
    },
    {
      "epoch": 0.4474097331240188,
      "grad_norm": 198.24272966431352,
      "learning_rate": 4.772200667318258e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -1.0242187976837158,
      "logps/chosen": -546.7999877929688,
      "logps/rejected": -1213.5999755859375,
      "loss": 0.6118,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.22709961235523224,
      "rewards/margins": 4.431347846984863,
      "rewards/rejected": -4.658203125,
      "step": 570
    },
    {
      "epoch": 0.4513343799058085,
      "grad_norm": 520.9529529384506,
      "learning_rate": 4.765004889683095e-07,
      "logits/chosen": -0.852343738079071,
      "logits/rejected": -0.9375,
      "logps/chosen": -611.7999877929688,
      "logps/rejected": -1518.4000244140625,
      "loss": 0.8811,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.060937523841858,
      "rewards/margins": 6.569726467132568,
      "rewards/rejected": -7.621484279632568,
      "step": 575
    },
    {
      "epoch": 0.4552590266875981,
      "grad_norm": 316.48072376054654,
      "learning_rate": 4.757702820412293e-07,
      "logits/chosen": -1.041015625,
      "logits/rejected": -1.0750000476837158,
      "logps/chosen": -647.2000122070312,
      "logps/rejected": -2172.0,
      "loss": 0.7209,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7367187738418579,
      "rewards/margins": 11.070556640625,
      "rewards/rejected": -11.810937881469727,
      "step": 580
    },
    {
      "epoch": 0.45918367346938777,
      "grad_norm": 276.2108236751764,
      "learning_rate": 4.750294802175702e-07,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.990234375,
      "logps/chosen": -577.4000244140625,
      "logps/rejected": -1262.0,
      "loss": 0.7624,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.33784180879592896,
      "rewards/margins": 4.562841892242432,
      "rewards/rejected": -4.888964653015137,
      "step": 585
    },
    {
      "epoch": 0.4631083202511774,
      "grad_norm": 244.93423934331574,
      "learning_rate": 4.742781182615122e-07,
      "logits/chosen": -0.940234363079071,
      "logits/rejected": -0.7298828363418579,
      "logps/chosen": -812.4000244140625,
      "logps/rejected": -923.5999755859375,
      "loss": 2.7372,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.5571045875549316,
      "rewards/margins": -0.04082031175494194,
      "rewards/rejected": -2.5169920921325684,
      "step": 590
    },
    {
      "epoch": 0.46703296703296704,
      "grad_norm": 329.47387457702837,
      "learning_rate": 4.7351623143279864e-07,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.862500011920929,
      "logps/chosen": -580.0,
      "logps/rejected": -901.5999755859375,
      "loss": 0.7034,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.35670167207717896,
      "rewards/margins": 1.2123534679412842,
      "rewards/rejected": -1.5681641101837158,
      "step": 595
    },
    {
      "epoch": 0.47095761381475665,
      "grad_norm": 283.9771732684289,
      "learning_rate": 4.727438554850818e-07,
      "logits/chosen": -1.0632812976837158,
      "logits/rejected": -0.787109375,
      "logps/chosen": -578.4000244140625,
      "logps/rejected": -950.0,
      "loss": 0.6667,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10161133110523224,
      "rewards/margins": 1.5872070789337158,
      "rewards/rejected": -1.687158226966858,
      "step": 600
    },
    {
      "epoch": 0.4748822605965463,
      "grad_norm": 1157.9495227310063,
      "learning_rate": 4.71961026664245e-07,
      "logits/chosen": -0.9013671875,
      "logits/rejected": -0.611523449420929,
      "logps/chosen": -619.7999877929688,
      "logps/rejected": -1113.199951171875,
      "loss": 1.1712,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.674243152141571,
      "rewards/margins": 0.8258056640625,
      "rewards/rejected": -1.501123070716858,
      "step": 605
    },
    {
      "epoch": 0.478806907378336,
      "grad_norm": 266.7003574003261,
      "learning_rate": 4.711677817067014e-07,
      "logits/chosen": -0.9322265386581421,
      "logits/rejected": -0.8646484613418579,
      "logps/chosen": -552.5999755859375,
      "logps/rejected": -840.7999877929688,
      "loss": 0.6828,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.758593738079071,
      "rewards/margins": 0.7606445550918579,
      "rewards/rejected": -1.5187499523162842,
      "step": 610
    },
    {
      "epoch": 0.4827315541601256,
      "grad_norm": 265.2214122889903,
      "learning_rate": 4.7036415783767056e-07,
      "logits/chosen": -0.8921874761581421,
      "logits/rejected": -0.878710925579071,
      "logps/chosen": -627.0,
      "logps/rejected": -1195.199951171875,
      "loss": 0.7415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5669921636581421,
      "rewards/margins": 2.649853467941284,
      "rewards/rejected": -3.2164063453674316,
      "step": 615
    },
    {
      "epoch": 0.48665620094191525,
      "grad_norm": 347.17364503204266,
      "learning_rate": 4.695501927694311e-07,
      "logits/chosen": -0.7435547113418579,
      "logits/rejected": -0.5848388671875,
      "logps/chosen": -553.4000244140625,
      "logps/rejected": -779.5999755859375,
      "loss": 0.8473,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4566406309604645,
      "rewards/margins": 0.4586425721645355,
      "rewards/rejected": -0.9150390625,
      "step": 620
    },
    {
      "epoch": 0.49058084772370486,
      "grad_norm": 464.4968712161022,
      "learning_rate": 4.687259246995511e-07,
      "logits/chosen": -0.548535168170929,
      "logits/rejected": -0.5033203363418579,
      "logps/chosen": -670.2000122070312,
      "logps/rejected": -1071.5999755859375,
      "loss": 0.7753,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.215234398841858,
      "rewards/margins": 2.16357421875,
      "rewards/rejected": -3.37890625,
      "step": 625
    },
    {
      "epoch": 0.4945054945054945,
      "grad_norm": 432.7965934590861,
      "learning_rate": 4.6789139230909536e-07,
      "logits/chosen": -0.706835925579071,
      "logits/rejected": -0.6341797113418579,
      "logps/chosen": -633.7999877929688,
      "logps/rejected": -1528.4000244140625,
      "loss": 0.7926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.165624976158142,
      "rewards/margins": 4.195410251617432,
      "rewards/rejected": -5.367968559265137,
      "step": 630
    },
    {
      "epoch": 0.49843014128728413,
      "grad_norm": 402.8423076896506,
      "learning_rate": 4.670466347608109e-07,
      "logits/chosen": -0.900585949420929,
      "logits/rejected": -0.7969726324081421,
      "logps/chosen": -532.7999877929688,
      "logps/rejected": -995.2000122070312,
      "loss": 0.7522,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.24494628608226776,
      "rewards/margins": 1.950769066810608,
      "rewards/rejected": -2.199023485183716,
      "step": 635
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 439.8609816531325,
      "learning_rate": 4.66191691697288e-07,
      "logits/chosen": -0.765820324420929,
      "logits/rejected": -0.714550793170929,
      "logps/chosen": -639.0,
      "logps/rejected": -1130.800048828125,
      "loss": 0.7523,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.523022472858429,
      "rewards/margins": 2.6073241233825684,
      "rewards/rejected": -3.1294922828674316,
      "step": 640
    },
    {
      "epoch": 0.5062794348508635,
      "grad_norm": 532.4641276049992,
      "learning_rate": 4.65326603239101e-07,
      "logits/chosen": -0.924609363079071,
      "logits/rejected": -0.656542956829071,
      "logps/chosen": -704.4000244140625,
      "logps/rejected": -1241.199951171875,
      "loss": 0.9228,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.474609375,
      "rewards/margins": 2.469860792160034,
      "rewards/rejected": -3.94140625,
      "step": 645
    },
    {
      "epoch": 0.5102040816326531,
      "grad_norm": 267.99331019227907,
      "learning_rate": 4.6445140998292475e-07,
      "logits/chosen": -0.7022949457168579,
      "logits/rejected": -0.823437511920929,
      "logps/chosen": -702.4000244140625,
      "logps/rejected": -1944.4000244140625,
      "loss": 0.7588,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.4855468273162842,
      "rewards/margins": 3.823046922683716,
      "rewards/rejected": -5.305468559265137,
      "step": 650
    },
    {
      "epoch": 0.5141287284144427,
      "grad_norm": 3354.299776318259,
      "learning_rate": 4.635661529996299e-07,
      "logits/chosen": -0.936718761920929,
      "logits/rejected": -0.7671874761581421,
      "logps/chosen": -620.7999877929688,
      "logps/rejected": -853.2000122070312,
      "loss": 1.218,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.025482177734375,
      "rewards/margins": 0.14824219048023224,
      "rewards/rejected": -1.17578125,
      "step": 655
    },
    {
      "epoch": 0.5180533751962323,
      "grad_norm": 311.7074437107188,
      "learning_rate": 4.626708738323552e-07,
      "logits/chosen": -0.961132824420929,
      "logits/rejected": -0.7959960699081421,
      "logps/chosen": -629.5999755859375,
      "logps/rejected": -1054.4000244140625,
      "loss": 0.7012,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6058593988418579,
      "rewards/margins": 2.354687452316284,
      "rewards/rejected": -2.952343702316284,
      "step": 660
    },
    {
      "epoch": 0.521978021978022,
      "grad_norm": 257.18642020687207,
      "learning_rate": 4.617656144945584e-07,
      "logits/chosen": -0.83740234375,
      "logits/rejected": -0.7818359136581421,
      "logps/chosen": -647.4000244140625,
      "logps/rejected": -1304.800048828125,
      "loss": 0.6154,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.571118175983429,
      "rewards/margins": 2.636523485183716,
      "rewards/rejected": -3.2074217796325684,
      "step": 665
    },
    {
      "epoch": 0.5259026687598116,
      "grad_norm": 379.4689355670139,
      "learning_rate": 4.6085041746804444e-07,
      "logits/chosen": -0.862500011920929,
      "logits/rejected": -0.659960925579071,
      "logps/chosen": -623.5999755859375,
      "logps/rejected": -997.2000122070312,
      "loss": 0.8785,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.44011229276657104,
      "rewards/margins": 1.1105468273162842,
      "rewards/rejected": -1.5496094226837158,
      "step": 670
    },
    {
      "epoch": 0.5298273155416012,
      "grad_norm": 318.02367835468067,
      "learning_rate": 4.5992532570097154e-07,
      "logits/chosen": -0.95263671875,
      "logits/rejected": -0.790234386920929,
      "logps/chosen": -584.2000122070312,
      "logps/rejected": -990.4000244140625,
      "loss": 0.7747,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.4706054627895355,
      "rewards/margins": 1.324853539466858,
      "rewards/rejected": -1.792578101158142,
      "step": 675
    },
    {
      "epoch": 0.533751962323391,
      "grad_norm": 283.1491034294907,
      "learning_rate": 4.5899038260583647e-07,
      "logits/chosen": -0.777148425579071,
      "logits/rejected": -0.74609375,
      "logps/chosen": -605.5999755859375,
      "logps/rejected": -1722.4000244140625,
      "loss": 0.6798,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.9537109136581421,
      "rewards/margins": 4.061327934265137,
      "rewards/rejected": -5.012499809265137,
      "step": 680
    },
    {
      "epoch": 0.5376766091051806,
      "grad_norm": 443.0396062062658,
      "learning_rate": 4.580456320574366e-07,
      "logits/chosen": -0.7437499761581421,
      "logits/rejected": -0.6504882574081421,
      "logps/chosen": -689.5999755859375,
      "logps/rejected": -1464.800048828125,
      "loss": 0.6895,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.6398437023162842,
      "rewards/margins": 4.106249809265137,
      "rewards/rejected": -5.73828125,
      "step": 685
    },
    {
      "epoch": 0.5416012558869702,
      "grad_norm": 292.8832161201978,
      "learning_rate": 4.570911183908114e-07,
      "logits/chosen": -0.614550769329071,
      "logits/rejected": -0.631640613079071,
      "logps/chosen": -602.7999877929688,
      "logps/rejected": -1043.5999755859375,
      "loss": 0.734,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.040136694908142,
      "rewards/margins": 2.271484375,
      "rewards/rejected": -3.3109374046325684,
      "step": 690
    },
    {
      "epoch": 0.5455259026687598,
      "grad_norm": 240.21381547493925,
      "learning_rate": 4.5612688639916176e-07,
      "logits/chosen": -0.852734386920929,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -579.0,
      "logps/rejected": -1415.199951171875,
      "loss": 0.7245,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.610644519329071,
      "rewards/margins": 4.798876762390137,
      "rewards/rejected": -5.402734279632568,
      "step": 695
    },
    {
      "epoch": 0.5494505494505495,
      "grad_norm": 280.4585356068312,
      "learning_rate": 4.5515298133174766e-07,
      "logits/chosen": -0.632519543170929,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -613.2000122070312,
      "logps/rejected": -1282.4000244140625,
      "loss": 1.2626,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.0603516101837158,
      "rewards/margins": 2.582324266433716,
      "rewards/rejected": -3.6507811546325684,
      "step": 700
    },
    {
      "epoch": 0.5533751962323391,
      "grad_norm": 327.85716485040365,
      "learning_rate": 4.5416944889176535e-07,
      "logits/chosen": -0.571484386920929,
      "logits/rejected": -0.5960937738418579,
      "logps/chosen": -612.4000244140625,
      "logps/rejected": -1235.5999755859375,
      "loss": 0.7488,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.2449219226837158,
      "rewards/margins": 3.2882323265075684,
      "rewards/rejected": -4.537109375,
      "step": 705
    },
    {
      "epoch": 0.5572998430141287,
      "grad_norm": 391.2880042298395,
      "learning_rate": 4.531763352342019e-07,
      "logits/chosen": -0.611035168170929,
      "logits/rejected": -0.42036134004592896,
      "logps/chosen": -682.7999877929688,
      "logps/rejected": -1603.199951171875,
      "loss": 0.7565,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.5710937976837158,
      "rewards/margins": 5.111620903015137,
      "rewards/rejected": -6.689062595367432,
      "step": 710
    },
    {
      "epoch": 0.5612244897959183,
      "grad_norm": 283.3916548512159,
      "learning_rate": 4.521736869636698e-07,
      "logits/chosen": -0.822460949420929,
      "logits/rejected": -0.7582031488418579,
      "logps/chosen": -676.0,
      "logps/rejected": -1640.4000244140625,
      "loss": 0.8273,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.768750011920929,
      "rewards/margins": 4.426513671875,
      "rewards/rejected": -5.185156345367432,
      "step": 715
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 1068.3224605989385,
      "learning_rate": 4.511615511322197e-07,
      "logits/chosen": -0.9585937261581421,
      "logits/rejected": -0.720507800579071,
      "logps/chosen": -695.2000122070312,
      "logps/rejected": -1079.5999755859375,
      "loss": 1.4214,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.1215941905975342,
      "rewards/margins": 0.832812488079071,
      "rewards/rejected": -1.952490210533142,
      "step": 720
    },
    {
      "epoch": 0.5690737833594977,
      "grad_norm": 298.31017842942333,
      "learning_rate": 4.501399752371322e-07,
      "logits/chosen": -0.798046886920929,
      "logits/rejected": -0.85546875,
      "logps/chosen": -767.5999755859375,
      "logps/rejected": -1399.5999755859375,
      "loss": 1.4761,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3289062976837158,
      "rewards/margins": 1.8918945789337158,
      "rewards/rejected": -3.221484422683716,
      "step": 725
    },
    {
      "epoch": 0.5729984301412873,
      "grad_norm": 250.82457370244083,
      "learning_rate": 4.491090072186892e-07,
      "logits/chosen": -0.835742175579071,
      "logits/rejected": -0.751757800579071,
      "logps/chosen": -574.5999755859375,
      "logps/rejected": -972.4000244140625,
      "loss": 0.6722,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.613964855670929,
      "rewards/margins": 1.294335961341858,
      "rewards/rejected": -1.910546898841858,
      "step": 730
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 227.80378228080215,
      "learning_rate": 4.48068695457924e-07,
      "logits/chosen": -0.8921874761581421,
      "logits/rejected": -0.673291027545929,
      "logps/chosen": -549.5999755859375,
      "logps/rejected": -882.7999877929688,
      "loss": 0.7539,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7723633050918579,
      "rewards/margins": 0.755664050579071,
      "rewards/rejected": -1.528906226158142,
      "step": 735
    },
    {
      "epoch": 0.5808477237048666,
      "grad_norm": 257.6869223422177,
      "learning_rate": 4.4701908877435097e-07,
      "logits/chosen": -0.867968738079071,
      "logits/rejected": -0.6353515386581421,
      "logps/chosen": -740.0,
      "logps/rejected": -1174.800048828125,
      "loss": 1.4271,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.5593750476837158,
      "rewards/margins": 0.518359363079071,
      "rewards/rejected": -2.08203125,
      "step": 740
    },
    {
      "epoch": 0.5847723704866562,
      "grad_norm": 255.0370569818503,
      "learning_rate": 4.459602364236743e-07,
      "logits/chosen": -0.8744140863418579,
      "logits/rejected": -0.7916015386581421,
      "logps/chosen": -833.2000122070312,
      "logps/rejected": -1466.4000244140625,
      "loss": 1.6125,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.0752441883087158,
      "rewards/margins": 0.6664062738418579,
      "rewards/rejected": -1.7414062023162842,
      "step": 745
    },
    {
      "epoch": 0.5886970172684458,
      "grad_norm": 242.59000764182318,
      "learning_rate": 4.4489218809547703e-07,
      "logits/chosen": -1.130859375,
      "logits/rejected": -0.9449218511581421,
      "logps/chosen": -545.7999877929688,
      "logps/rejected": -898.0,
      "loss": 0.6866,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.03001098707318306,
      "rewards/margins": 1.678125023841858,
      "rewards/rejected": -1.7080078125,
      "step": 750
    },
    {
      "epoch": 0.5926216640502355,
      "grad_norm": 369.993383860242,
      "learning_rate": 4.4381499391088865e-07,
      "logits/chosen": -0.8501952886581421,
      "logits/rejected": -0.7896484136581421,
      "logps/chosen": -621.4000244140625,
      "logps/rejected": -916.7999877929688,
      "loss": 0.9415,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.6733642816543579,
      "rewards/margins": 0.833740234375,
      "rewards/rejected": -1.505468726158142,
      "step": 755
    },
    {
      "epoch": 0.5965463108320251,
      "grad_norm": 249.97546947980268,
      "learning_rate": 4.427287044202337e-07,
      "logits/chosen": -0.910937488079071,
      "logits/rejected": -0.7279297113418579,
      "logps/chosen": -675.2000122070312,
      "logps/rejected": -1241.199951171875,
      "loss": 0.7058,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.973828136920929,
      "rewards/margins": 2.3216795921325684,
      "rewards/rejected": -3.2953124046325684,
      "step": 760
    },
    {
      "epoch": 0.6004709576138147,
      "grad_norm": 257.56927516079315,
      "learning_rate": 4.4163337060065887e-07,
      "logits/chosen": -0.87890625,
      "logits/rejected": -0.896484375,
      "logps/chosen": -629.0,
      "logps/rejected": -1517.199951171875,
      "loss": 0.6272,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.9144531488418579,
      "rewards/margins": 4.019378662109375,
      "rewards/rejected": -4.935156345367432,
      "step": 765
    },
    {
      "epoch": 0.6043956043956044,
      "grad_norm": 280.47872687091984,
      "learning_rate": 4.4052904385374114e-07,
      "logits/chosen": -0.8369140625,
      "logits/rejected": -0.7449951171875,
      "logps/chosen": -579.2000122070312,
      "logps/rejected": -1576.0,
      "loss": 0.6111,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0947265625,
      "rewards/margins": 3.318164110183716,
      "rewards/rejected": -4.405468940734863,
      "step": 770
    },
    {
      "epoch": 0.6083202511773941,
      "grad_norm": 374.23011286357405,
      "learning_rate": 4.3941577600307555e-07,
      "logits/chosen": -0.753710925579071,
      "logits/rejected": -0.798828125,
      "logps/chosen": -699.5999755859375,
      "logps/rejected": -1140.800048828125,
      "loss": 0.886,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.239843726158142,
      "rewards/margins": 1.5227539539337158,
      "rewards/rejected": -2.7554688453674316,
      "step": 775
    },
    {
      "epoch": 0.6122448979591837,
      "grad_norm": 366.07688362071013,
      "learning_rate": 4.3829361929184334e-07,
      "logits/chosen": -0.899218738079071,
      "logits/rejected": -0.769335925579071,
      "logps/chosen": -561.7999877929688,
      "logps/rejected": -1325.5999755859375,
      "loss": 0.6718,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6612304449081421,
      "rewards/margins": 2.8272948265075684,
      "rewards/rejected": -3.4917969703674316,
      "step": 780
    },
    {
      "epoch": 0.6161695447409733,
      "grad_norm": 211.96266950843466,
      "learning_rate": 4.3716262638036007e-07,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.919140636920929,
      "logps/chosen": -640.0,
      "logps/rejected": -1335.199951171875,
      "loss": 0.7121,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6673828363418579,
      "rewards/margins": 3.0785155296325684,
      "rewards/rejected": -3.7408204078674316,
      "step": 785
    },
    {
      "epoch": 0.6200941915227629,
      "grad_norm": 342.72533090308946,
      "learning_rate": 4.3602285034360445e-07,
      "logits/chosen": -1.0832030773162842,
      "logits/rejected": -0.93359375,
      "logps/chosen": -595.4000244140625,
      "logps/rejected": -860.4000244140625,
      "loss": 0.7937,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.32373046875,
      "rewards/margins": 1.1676514148712158,
      "rewards/rejected": -1.490332007408142,
      "step": 790
    },
    {
      "epoch": 0.6240188383045526,
      "grad_norm": 315.72643355373924,
      "learning_rate": 4.348743446687278e-07,
      "logits/chosen": -1.0792968273162842,
      "logits/rejected": -0.900390625,
      "logps/chosen": -612.5999755859375,
      "logps/rejected": -925.2000122070312,
      "loss": 0.8053,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.07958984375,
      "rewards/margins": 0.6826171875,
      "rewards/rejected": -1.764062523841858,
      "step": 795
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 661.6505882469437,
      "learning_rate": 4.337171632525439e-07,
      "logits/chosen": -0.9742187261581421,
      "logits/rejected": -0.7484375238418579,
      "logps/chosen": -679.2000122070312,
      "logps/rejected": -1009.2000122070312,
      "loss": 0.8377,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.199609398841858,
      "rewards/margins": 1.073828101158142,
      "rewards/rejected": -2.272656202316284,
      "step": 800
    },
    {
      "epoch": 0.6318681318681318,
      "grad_norm": 403.9927890934692,
      "learning_rate": 4.3255136039899974e-07,
      "logits/chosen": -1.0636718273162842,
      "logits/rejected": -1.066796898841858,
      "logps/chosen": -653.0,
      "logps/rejected": -1766.0,
      "loss": 0.7884,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.203710913658142,
      "rewards/margins": 3.4921875,
      "rewards/rejected": -4.694531440734863,
      "step": 805
    },
    {
      "epoch": 0.6357927786499215,
      "grad_norm": 270.74902576807415,
      "learning_rate": 4.3137699081662725e-07,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.914257824420929,
      "logps/chosen": -639.2000122070312,
      "logps/rejected": -930.7999877929688,
      "loss": 0.8164,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.965039074420929,
      "rewards/margins": 1.1763184070587158,
      "rewards/rejected": -2.141406297683716,
      "step": 810
    },
    {
      "epoch": 0.6397174254317112,
      "grad_norm": 189.00772501651608,
      "learning_rate": 4.301941096159756e-07,
      "logits/chosen": -1.1746094226837158,
      "logits/rejected": -0.944140613079071,
      "logps/chosen": -530.0,
      "logps/rejected": -1248.0,
      "loss": 0.7042,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3652587831020355,
      "rewards/margins": 2.81005859375,
      "rewards/rejected": -3.1800780296325684,
      "step": 815
    },
    {
      "epoch": 0.6436420722135008,
      "grad_norm": 187.131983231018,
      "learning_rate": 4.290027723070256e-07,
      "logits/chosen": -1.1437499523162842,
      "logits/rejected": -1.0378906726837158,
      "logps/chosen": -619.7999877929688,
      "logps/rejected": -1162.0,
      "loss": 1.5805,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.164892554283142,
      "rewards/margins": 0.98876953125,
      "rewards/rejected": -2.151318311691284,
      "step": 820
    },
    {
      "epoch": 0.6475667189952904,
      "grad_norm": 349.5466846080361,
      "learning_rate": 4.278030347965841e-07,
      "logits/chosen": -0.7923828363418579,
      "logits/rejected": -0.7435547113418579,
      "logps/chosen": -669.5999755859375,
      "logps/rejected": -1281.199951171875,
      "loss": 0.7616,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.087890625,
      "rewards/margins": 2.810351610183716,
      "rewards/rejected": -3.901562452316284,
      "step": 825
    },
    {
      "epoch": 0.6514913657770801,
      "grad_norm": 352.39658355708855,
      "learning_rate": 4.2659495338566085e-07,
      "logits/chosen": -0.7734375,
      "logits/rejected": -0.498779296875,
      "logps/chosen": -589.7999877929688,
      "logps/rejected": -1256.800048828125,
      "loss": 0.9013,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.193359375,
      "rewards/margins": 1.9484374523162842,
      "rewards/rejected": -3.143749952316284,
      "step": 830
    },
    {
      "epoch": 0.6554160125588697,
      "grad_norm": 263.30406396579053,
      "learning_rate": 4.2537858476682635e-07,
      "logits/chosen": -0.977343738079071,
      "logits/rejected": -0.775390625,
      "logps/chosen": -599.7999877929688,
      "logps/rejected": -1358.0,
      "loss": 0.6198,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8765624761581421,
      "rewards/margins": 2.4515624046325684,
      "rewards/rejected": -3.33203125,
      "step": 835
    },
    {
      "epoch": 0.6593406593406593,
      "grad_norm": 334.1630688355394,
      "learning_rate": 4.2415398602155117e-07,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.874804675579071,
      "logps/chosen": -626.5999755859375,
      "logps/rejected": -1528.800048828125,
      "loss": 0.7559,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8642578125,
      "rewards/margins": 2.535937547683716,
      "rewards/rejected": -3.399121046066284,
      "step": 840
    },
    {
      "epoch": 0.6632653061224489,
      "grad_norm": 304.14098169053153,
      "learning_rate": 4.229212146175273e-07,
      "logits/chosen": -1.1796875,
      "logits/rejected": -1.0792968273162842,
      "logps/chosen": -648.4000244140625,
      "logps/rejected": -960.7999877929688,
      "loss": 0.6851,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.631640613079071,
      "rewards/margins": 0.9090820550918579,
      "rewards/rejected": -1.5382812023162842,
      "step": 845
    },
    {
      "epoch": 0.6671899529042387,
      "grad_norm": 277.8119896926451,
      "learning_rate": 4.216803284059717e-07,
      "logits/chosen": -1.0173828601837158,
      "logits/rejected": -0.8753906488418579,
      "logps/chosen": -549.0,
      "logps/rejected": -1230.0,
      "loss": 0.6818,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.40922850370407104,
      "rewards/margins": 1.7654297351837158,
      "rewards/rejected": -2.1761717796325684,
      "step": 850
    },
    {
      "epoch": 0.6711145996860283,
      "grad_norm": 281.94783036928794,
      "learning_rate": 4.204313856189108e-07,
      "logits/chosen": -0.960156261920929,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -641.4000244140625,
      "logps/rejected": -957.2000122070312,
      "loss": 0.8362,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.907421886920929,
      "rewards/margins": 0.73193359375,
      "rewards/rejected": -1.639062523841858,
      "step": 855
    },
    {
      "epoch": 0.6750392464678179,
      "grad_norm": 295.8724971819344,
      "learning_rate": 4.1917444486644857e-07,
      "logits/chosen": -0.842578113079071,
      "logits/rejected": -0.859375,
      "logps/chosen": -655.4000244140625,
      "logps/rejected": -1231.199951171875,
      "loss": 0.7587,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9525390863418579,
      "rewards/margins": 1.5539062023162842,
      "rewards/rejected": -2.50390625,
      "step": 860
    },
    {
      "epoch": 0.6789638932496075,
      "grad_norm": 870.2343483781506,
      "learning_rate": 4.179095651340154e-07,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.8076171875,
      "logps/chosen": -640.5999755859375,
      "logps/rejected": -1141.5999755859375,
      "loss": 0.7235,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6763671636581421,
      "rewards/margins": 1.538476586341858,
      "rewards/rejected": -2.215625047683716,
      "step": 865
    },
    {
      "epoch": 0.6828885400313972,
      "grad_norm": 1218.6065576246704,
      "learning_rate": 4.166368057796006e-07,
      "logits/chosen": -1.123046875,
      "logits/rejected": -0.9886718988418579,
      "logps/chosen": -571.0,
      "logps/rejected": -1169.5999755859375,
      "loss": 0.7354,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3378662168979645,
      "rewards/margins": 1.2373046875,
      "rewards/rejected": -1.57421875,
      "step": 870
    },
    {
      "epoch": 0.6868131868131868,
      "grad_norm": 284.04882518330146,
      "learning_rate": 4.1535622653096615e-07,
      "logits/chosen": -1.1398437023162842,
      "logits/rejected": -1.005859375,
      "logps/chosen": -559.2000122070312,
      "logps/rejected": -862.7999877929688,
      "loss": 0.8557,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.34942626953125,
      "rewards/margins": 0.6876159906387329,
      "rewards/rejected": -1.035791039466858,
      "step": 875
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 351.52575231490005,
      "learning_rate": 4.1406788748284474e-07,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.767382800579071,
      "logps/chosen": -656.5999755859375,
      "logps/rejected": -1103.5999755859375,
      "loss": 0.8229,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.9498046636581421,
      "rewards/margins": 2.4554686546325684,
      "rewards/rejected": -3.4066405296325684,
      "step": 880
    },
    {
      "epoch": 0.6946624803767661,
      "grad_norm": 288.21572204489064,
      "learning_rate": 4.1277184909411885e-07,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -1.0281250476837158,
      "logps/chosen": -645.5999755859375,
      "logps/rejected": -1780.4000244140625,
      "loss": 0.7937,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.8538573980331421,
      "rewards/margins": 5.649609565734863,
      "rewards/rejected": -6.51171875,
      "step": 885
    },
    {
      "epoch": 0.6985871271585558,
      "grad_norm": 246.96096415764885,
      "learning_rate": 4.11468172184984e-07,
      "logits/chosen": -1.1726562976837158,
      "logits/rejected": -0.9212890863418579,
      "logps/chosen": -596.5999755859375,
      "logps/rejected": -856.7999877929688,
      "loss": 0.6652,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3467773497104645,
      "rewards/margins": 1.478759765625,
      "rewards/rejected": -1.8254883289337158,
      "step": 890
    },
    {
      "epoch": 0.7025117739403454,
      "grad_norm": 256.5656668089437,
      "learning_rate": 4.101569179340945e-07,
      "logits/chosen": -1.049218773841858,
      "logits/rejected": -1.072656273841858,
      "logps/chosen": -836.0,
      "logps/rejected": -1392.800048828125,
      "loss": 1.6441,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.5397460460662842,
      "rewards/margins": 2.423632860183716,
      "rewards/rejected": -3.9605469703674316,
      "step": 895
    },
    {
      "epoch": 0.706436420722135,
      "grad_norm": 448.1909653707126,
      "learning_rate": 4.088381478756923e-07,
      "logits/chosen": -1.060546875,
      "logits/rejected": -0.9671875238418579,
      "logps/chosen": -672.5999755859375,
      "logps/rejected": -948.4000244140625,
      "loss": 0.824,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.108984351158142,
      "rewards/margins": 1.5286376476287842,
      "rewards/rejected": -2.633984327316284,
      "step": 900
    },
    {
      "epoch": 0.7103610675039247,
      "grad_norm": 214.59794145190085,
      "learning_rate": 4.0751192389671954e-07,
      "logits/chosen": -1.076562523841858,
      "logits/rejected": -1.015039086341858,
      "logps/chosen": -658.5999755859375,
      "logps/rejected": -1305.5999755859375,
      "loss": 0.7728,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.1671874523162842,
      "rewards/margins": 2.5663084983825684,
      "rewards/rejected": -3.7367186546325684,
      "step": 905
    },
    {
      "epoch": 0.7142857142857143,
      "grad_norm": 297.72053278043416,
      "learning_rate": 4.061783082339143e-07,
      "logits/chosen": -1.252343773841858,
      "logits/rejected": -1.088476538658142,
      "logps/chosen": -590.2000122070312,
      "logps/rejected": -994.0,
      "loss": 0.7774,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5185791254043579,
      "rewards/margins": 0.4697265625,
      "rewards/rejected": -0.9867187738418579,
      "step": 910
    },
    {
      "epoch": 0.7182103610675039,
      "grad_norm": 240.76548164280746,
      "learning_rate": 4.048373634708898e-07,
      "logits/chosen": -1.243749976158142,
      "logits/rejected": -1.120703101158142,
      "logps/chosen": -522.2000122070312,
      "logps/rejected": -1224.0,
      "loss": 0.7031,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.28192138671875,
      "rewards/margins": 2.61328125,
      "rewards/rejected": -2.90087890625,
      "step": 915
    },
    {
      "epoch": 0.7221350078492935,
      "grad_norm": 306.0935776387031,
      "learning_rate": 4.034891525351978e-07,
      "logits/chosen": -1.029296875,
      "logits/rejected": -1.0070312023162842,
      "logps/chosen": -673.0,
      "logps/rejected": -1285.5999755859375,
      "loss": 0.8853,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.3253905773162842,
      "rewards/margins": 1.559106469154358,
      "rewards/rejected": -2.87890625,
      "step": 920
    },
    {
      "epoch": 0.7260596546310832,
      "grad_norm": 196.65668029230338,
      "learning_rate": 4.0213373869537506e-07,
      "logits/chosen": -1.0,
      "logits/rejected": -0.8974609375,
      "logps/chosen": -621.0,
      "logps/rejected": -1234.0,
      "loss": 0.7398,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.87109375,
      "rewards/margins": 3.1632933616638184,
      "rewards/rejected": -4.038671970367432,
      "step": 925
    },
    {
      "epoch": 0.7299843014128728,
      "grad_norm": 484.99861926320517,
      "learning_rate": 4.007711855579748e-07,
      "logits/chosen": -1.0929687023162842,
      "logits/rejected": -0.964648425579071,
      "logps/chosen": -574.7999877929688,
      "logps/rejected": -1098.800048828125,
      "loss": 0.7542,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.41315919160842896,
      "rewards/margins": 1.4187500476837158,
      "rewards/rejected": -1.832421898841858,
      "step": 930
    },
    {
      "epoch": 0.7339089481946625,
      "grad_norm": 200.48480850270482,
      "learning_rate": 3.994015570645818e-07,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -1.004296898841858,
      "logps/chosen": -693.5999755859375,
      "logps/rejected": -1138.0,
      "loss": 1.6044,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2338043451309204,
      "rewards/margins": 1.5314452648162842,
      "rewards/rejected": -2.764453172683716,
      "step": 935
    },
    {
      "epoch": 0.7378335949764521,
      "grad_norm": 244.8338799779736,
      "learning_rate": 3.980249174888111e-07,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.883007824420929,
      "logps/chosen": -565.2000122070312,
      "logps/rejected": -1248.0,
      "loss": 0.7342,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.43701171875,
      "rewards/margins": 2.3031249046325684,
      "rewards/rejected": -2.7451171875,
      "step": 940
    },
    {
      "epoch": 0.7417582417582418,
      "grad_norm": 336.49344844457914,
      "learning_rate": 3.966413314332924e-07,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.983203113079071,
      "logps/chosen": -613.5999755859375,
      "logps/rejected": -958.0,
      "loss": 0.7613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7037109136581421,
      "rewards/margins": 1.518457055091858,
      "rewards/rejected": -2.215625047683716,
      "step": 945
    },
    {
      "epoch": 0.7456828885400314,
      "grad_norm": 292.43718897249255,
      "learning_rate": 3.952508638266382e-07,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -1.013281226158142,
      "logps/chosen": -621.0,
      "logps/rejected": -1465.5999755859375,
      "loss": 0.6333,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8167968988418579,
      "rewards/margins": 2.70361328125,
      "rewards/rejected": -3.5218749046325684,
      "step": 950
    },
    {
      "epoch": 0.749607535321821,
      "grad_norm": 275.8587162345732,
      "learning_rate": 3.93853579920397e-07,
      "logits/chosen": -0.886523425579071,
      "logits/rejected": -0.72509765625,
      "logps/chosen": -659.4000244140625,
      "logps/rejected": -1100.800048828125,
      "loss": 0.7558,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0048828125,
      "rewards/margins": 1.314453125,
      "rewards/rejected": -2.31640625,
      "step": 955
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 325.822713164713,
      "learning_rate": 3.9244954528599103e-07,
      "logits/chosen": -0.8714843988418579,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -687.5999755859375,
      "logps/rejected": -1541.199951171875,
      "loss": 0.9647,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.0031249523162842,
      "rewards/margins": 3.8460936546325684,
      "rewards/rejected": -4.856249809265137,
      "step": 960
    },
    {
      "epoch": 0.7574568288854003,
      "grad_norm": 171.48238044306925,
      "learning_rate": 3.91038825811639e-07,
      "logits/chosen": -1.116796851158142,
      "logits/rejected": -1.2199218273162842,
      "logps/chosen": -613.2000122070312,
      "logps/rejected": -1200.0,
      "loss": 0.7788,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.17431640625,
      "rewards/margins": 2.653759717941284,
      "rewards/rejected": -2.833984375,
      "step": 965
    },
    {
      "epoch": 0.7613814756671899,
      "grad_norm": 383.86084588047146,
      "learning_rate": 3.896214876992644e-07,
      "logits/chosen": -1.25390625,
      "logits/rejected": -1.1375000476837158,
      "logps/chosen": -498.79998779296875,
      "logps/rejected": -720.0,
      "loss": 0.9011,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.04228515550494194,
      "rewards/margins": 0.111083984375,
      "rewards/rejected": -0.06887207180261612,
      "step": 970
    },
    {
      "epoch": 0.7653061224489796,
      "grad_norm": 289.14771635440246,
      "learning_rate": 3.881975974613887e-07,
      "logits/chosen": -0.8335937261581421,
      "logits/rejected": -0.856640636920929,
      "logps/chosen": -533.0,
      "logps/rejected": -821.2000122070312,
      "loss": 0.7366,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3547607362270355,
      "rewards/margins": 0.618359386920929,
      "rewards/rejected": -0.973828136920929,
      "step": 975
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 284.4055949745424,
      "learning_rate": 3.867672219180099e-07,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.9378906488418579,
      "logps/chosen": -692.2000122070312,
      "logps/rejected": -1210.0,
      "loss": 0.7379,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.3054687976837158,
      "rewards/margins": 1.713769555091858,
      "rewards/rejected": -3.0179686546325684,
      "step": 980
    },
    {
      "epoch": 0.7731554160125589,
      "grad_norm": 302.79252427488603,
      "learning_rate": 3.85330428193467e-07,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.6962890625,
      "logps/chosen": -704.4000244140625,
      "logps/rejected": -1060.4000244140625,
      "loss": 0.7635,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3894531726837158,
      "rewards/margins": 1.4890625476837158,
      "rewards/rejected": -2.878124952316284,
      "step": 985
    },
    {
      "epoch": 0.7770800627943485,
      "grad_norm": 236.09991763335572,
      "learning_rate": 3.8388728371329007e-07,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.92578125,
      "logps/chosen": -549.4000244140625,
      "logps/rejected": -1347.199951171875,
      "loss": 0.7818,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.41572266817092896,
      "rewards/margins": 2.26611328125,
      "rewards/rejected": -2.6861815452575684,
      "step": 990
    },
    {
      "epoch": 0.7810047095761381,
      "grad_norm": 189.1518522953691,
      "learning_rate": 3.8243785620103573e-07,
      "logits/chosen": -1.066796898841858,
      "logits/rejected": -0.986132800579071,
      "logps/chosen": -539.7999877929688,
      "logps/rejected": -828.4000244140625,
      "loss": 0.7408,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21256104111671448,
      "rewards/margins": 0.778881847858429,
      "rewards/rejected": -0.992663562297821,
      "step": 995
    },
    {
      "epoch": 0.7849293563579278,
      "grad_norm": 246.43023420113335,
      "learning_rate": 3.809822136751096e-07,
      "logits/chosen": -1.023828148841858,
      "logits/rejected": -0.842578113079071,
      "logps/chosen": -620.4000244140625,
      "logps/rejected": -1082.0,
      "loss": 0.6677,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.628857433795929,
      "rewards/margins": 2.334179639816284,
      "rewards/rejected": -2.9625000953674316,
      "step": 1000
    },
    {
      "epoch": 0.7888540031397174,
      "grad_norm": 230.18729610937373,
      "learning_rate": 3.795204244455739e-07,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.739501953125,
      "logps/chosen": -630.5999755859375,
      "logps/rejected": -1036.4000244140625,
      "loss": 0.6985,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.1511719226837158,
      "rewards/margins": 1.7020752429962158,
      "rewards/rejected": -2.850781202316284,
      "step": 1005
    },
    {
      "epoch": 0.792778649921507,
      "grad_norm": 192.72300145569358,
      "learning_rate": 3.78052557110942e-07,
      "logits/chosen": -1.154296875,
      "logits/rejected": -0.9859374761581421,
      "logps/chosen": -658.7999877929688,
      "logps/rejected": -1194.4000244140625,
      "loss": 0.6845,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8006836175918579,
      "rewards/margins": 2.0352540016174316,
      "rewards/rejected": -2.8363280296325684,
      "step": 1010
    },
    {
      "epoch": 0.7967032967032966,
      "grad_norm": 327.35809923406936,
      "learning_rate": 3.7657868055495913e-07,
      "logits/chosen": -1.211328148841858,
      "logits/rejected": -0.9175781011581421,
      "logps/chosen": -590.5999755859375,
      "logps/rejected": -819.0,
      "loss": 0.6715,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.4542236328125,
      "rewards/margins": 0.797900378704071,
      "rewards/rejected": -1.252343773841858,
      "step": 1015
    },
    {
      "epoch": 0.8006279434850864,
      "grad_norm": 291.0648541503122,
      "learning_rate": 3.7509886394336994e-07,
      "logits/chosen": -1.172265648841858,
      "logits/rejected": -1.0128905773162842,
      "logps/chosen": -593.0,
      "logps/rejected": -1042.4000244140625,
      "loss": 0.6761,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6396484375,
      "rewards/margins": 1.413476586341858,
      "rewards/rejected": -2.05078125,
      "step": 1020
    },
    {
      "epoch": 0.804552590266876,
      "grad_norm": 397.38512430127247,
      "learning_rate": 3.736131767206727e-07,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.915234386920929,
      "logps/chosen": -664.0,
      "logps/rejected": -1511.5999755859375,
      "loss": 0.8675,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.2082030773162842,
      "rewards/margins": 4.03924560546875,
      "rewards/rejected": -5.252343654632568,
      "step": 1025
    },
    {
      "epoch": 0.8084772370486656,
      "grad_norm": 482.96849368177965,
      "learning_rate": 3.7212168860686036e-07,
      "logits/chosen": -1.0398437976837158,
      "logits/rejected": -0.9248046875,
      "logps/chosen": -681.2000122070312,
      "logps/rejected": -1326.4000244140625,
      "loss": 0.9576,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0427734851837158,
      "rewards/margins": 3.4398436546325684,
      "rewards/rejected": -4.483202934265137,
      "step": 1030
    },
    {
      "epoch": 0.8124018838304553,
      "grad_norm": 320.5751018986243,
      "learning_rate": 3.706244695941489e-07,
      "logits/chosen": -1.248437523841858,
      "logits/rejected": -1.1355469226837158,
      "logps/chosen": -531.0,
      "logps/rejected": -1367.5999755859375,
      "loss": 0.7062,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.28271484375,
      "rewards/margins": 3.5589842796325684,
      "rewards/rejected": -3.8404297828674316,
      "step": 1035
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 373.3698030592153,
      "learning_rate": 3.691215899436922e-07,
      "logits/chosen": -1.211328148841858,
      "logits/rejected": -0.8648437261581421,
      "logps/chosen": -683.5999755859375,
      "logps/rejected": -1132.0,
      "loss": 0.9729,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.164453148841858,
      "rewards/margins": 2.4525389671325684,
      "rewards/rejected": -3.620312452316284,
      "step": 1040
    },
    {
      "epoch": 0.8202511773940345,
      "grad_norm": 246.25122076994776,
      "learning_rate": 3.676131201822859e-07,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.92041015625,
      "logps/chosen": -644.4000244140625,
      "logps/rejected": -1180.0,
      "loss": 0.7411,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.080078125,
      "rewards/margins": 1.6384766101837158,
      "rewards/rejected": -2.7164063453674316,
      "step": 1045
    },
    {
      "epoch": 0.8241758241758241,
      "grad_norm": 220.1800157094643,
      "learning_rate": 3.6609913109905664e-07,
      "logits/chosen": -1.157812476158142,
      "logits/rejected": -1.013281226158142,
      "logps/chosen": -544.5999755859375,
      "logps/rejected": -1139.5999755859375,
      "loss": 0.7129,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6009277105331421,
      "rewards/margins": 2.585864305496216,
      "rewards/rejected": -3.1890625953674316,
      "step": 1050
    },
    {
      "epoch": 0.8281004709576139,
      "grad_norm": 197.21896395037345,
      "learning_rate": 3.6457969374214057e-07,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.955078125,
      "logps/chosen": -611.4000244140625,
      "logps/rejected": -1136.0,
      "loss": 0.7326,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5743163824081421,
      "rewards/margins": 1.8181641101837158,
      "rewards/rejected": -2.388671875,
      "step": 1055
    },
    {
      "epoch": 0.8320251177394035,
      "grad_norm": 211.81609915400716,
      "learning_rate": 3.630548794153494e-07,
      "logits/chosen": -1.0486328601837158,
      "logits/rejected": -0.85546875,
      "logps/chosen": -578.0,
      "logps/rejected": -859.2000122070312,
      "loss": 0.938,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.521527111530304,
      "rewards/margins": 0.7535644769668579,
      "rewards/rejected": -1.275781273841858,
      "step": 1060
    },
    {
      "epoch": 0.8359497645211931,
      "grad_norm": 214.7855502152401,
      "learning_rate": 3.6152475967482387e-07,
      "logits/chosen": -0.950244128704071,
      "logits/rejected": -1.0546875,
      "logps/chosen": -624.4000244140625,
      "logps/rejected": -1267.5999755859375,
      "loss": 0.7218,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.958203136920929,
      "rewards/margins": 2.7686524391174316,
      "rewards/rejected": -3.731250047683716,
      "step": 1065
    },
    {
      "epoch": 0.8398744113029827,
      "grad_norm": 238.23815649799593,
      "learning_rate": 3.5998940632567607e-07,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.7884765863418579,
      "logps/chosen": -626.0,
      "logps/rejected": -1158.4000244140625,
      "loss": 0.6607,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.979687511920929,
      "rewards/margins": 1.815820336341858,
      "rewards/rejected": -2.7945313453674316,
      "step": 1070
    },
    {
      "epoch": 0.8437990580847724,
      "grad_norm": 266.86205001136085,
      "learning_rate": 3.584488914186197e-07,
      "logits/chosen": -0.911328136920929,
      "logits/rejected": -0.8857421875,
      "logps/chosen": -596.4000244140625,
      "logps/rejected": -1204.0,
      "loss": 0.6739,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.783007800579071,
      "rewards/margins": 1.8820312023162842,
      "rewards/rejected": -2.6624999046325684,
      "step": 1075
    },
    {
      "epoch": 0.847723704866562,
      "grad_norm": 213.98573196680155,
      "learning_rate": 3.569032872465889e-07,
      "logits/chosen": -0.943164050579071,
      "logits/rejected": -0.89453125,
      "logps/chosen": -621.4000244140625,
      "logps/rejected": -1325.5999755859375,
      "loss": 0.6305,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.947265625,
      "rewards/margins": 3.0542969703674316,
      "rewards/rejected": -3.9945311546325684,
      "step": 1080
    },
    {
      "epoch": 0.8516483516483516,
      "grad_norm": 177.37711539490843,
      "learning_rate": 3.5535266634134554e-07,
      "logits/chosen": -1.082421898841858,
      "logits/rejected": -0.9554687738418579,
      "logps/chosen": -638.7999877929688,
      "logps/rejected": -959.2000122070312,
      "loss": 0.8293,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.017578125,
      "rewards/margins": 1.24609375,
      "rewards/rejected": -2.262500047683716,
      "step": 1085
    },
    {
      "epoch": 0.8555729984301413,
      "grad_norm": 211.40605824085952,
      "learning_rate": 3.537971014700756e-07,
      "logits/chosen": -1.0339844226837158,
      "logits/rejected": -0.949999988079071,
      "logps/chosen": -616.4000244140625,
      "logps/rejected": -949.2000122070312,
      "loss": 0.81,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4939941465854645,
      "rewards/margins": 1.3985474109649658,
      "rewards/rejected": -1.897070288658142,
      "step": 1090
    },
    {
      "epoch": 0.859497645211931,
      "grad_norm": 225.20159834312267,
      "learning_rate": 3.5223666563197473e-07,
      "logits/chosen": -1.2511718273162842,
      "logits/rejected": -1.015234351158142,
      "logps/chosen": -560.7999877929688,
      "logps/rejected": -1202.4000244140625,
      "loss": 0.756,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.21147461235523224,
      "rewards/margins": 2.1429505348205566,
      "rewards/rejected": -2.348828077316284,
      "step": 1095
    },
    {
      "epoch": 0.8634222919937206,
      "grad_norm": 220.2011220050631,
      "learning_rate": 3.5067143205482173e-07,
      "logits/chosen": -1.2374999523162842,
      "logits/rejected": -1.026953101158142,
      "logps/chosen": -799.5999755859375,
      "logps/rejected": -1158.4000244140625,
      "loss": 1.422,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.893908679485321,
      "rewards/margins": 2.7027344703674316,
      "rewards/rejected": -3.59375,
      "step": 1100
    },
    {
      "epoch": 0.8673469387755102,
      "grad_norm": 478.12666183864417,
      "learning_rate": 3.49101474191543e-07,
      "logits/chosen": -0.9429687261581421,
      "logits/rejected": -0.879687488079071,
      "logps/chosen": -612.2000122070312,
      "logps/rejected": -1489.199951171875,
      "loss": 0.7262,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9261718988418579,
      "rewards/margins": 1.873876929283142,
      "rewards/rejected": -2.7984375953674316,
      "step": 1105
    },
    {
      "epoch": 0.8712715855572999,
      "grad_norm": 250.37683787592368,
      "learning_rate": 3.475268657167649e-07,
      "logits/chosen": -0.92578125,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -707.2000122070312,
      "logps/rejected": -1034.0,
      "loss": 1.1686,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.9460937976837158,
      "rewards/margins": 0.6226562261581421,
      "rewards/rejected": -2.5679688453674316,
      "step": 1110
    },
    {
      "epoch": 0.8751962323390895,
      "grad_norm": 288.25786252863134,
      "learning_rate": 3.4594768052335667e-07,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.863476574420929,
      "logps/chosen": -691.5999755859375,
      "logps/rejected": -1489.199951171875,
      "loss": 0.8577,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.103515625,
      "rewards/margins": 2.1976561546325684,
      "rewards/rejected": -3.303906202316284,
      "step": 1115
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 199.67307563873212,
      "learning_rate": 3.4436399271896286e-07,
      "logits/chosen": -1.0509765148162842,
      "logits/rejected": -0.8734375238418579,
      "logps/chosen": -618.4000244140625,
      "logps/rejected": -1160.800048828125,
      "loss": 0.6758,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5218139886856079,
      "rewards/margins": 1.607812523841858,
      "rewards/rejected": -2.127734422683716,
      "step": 1120
    },
    {
      "epoch": 0.8830455259026687,
      "grad_norm": 1123.4119361708001,
      "learning_rate": 3.4277587662252567e-07,
      "logits/chosen": -1.090234398841858,
      "logits/rejected": -1.064843773841858,
      "logps/chosen": -602.5999755859375,
      "logps/rejected": -1423.5999755859375,
      "loss": 0.7875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.370849609375,
      "rewards/margins": 1.286718726158142,
      "rewards/rejected": -1.6583983898162842,
      "step": 1125
    },
    {
      "epoch": 0.8869701726844584,
      "grad_norm": 206.57825651295488,
      "learning_rate": 3.411834067607967e-07,
      "logits/chosen": -1.197265625,
      "logits/rejected": -1.119531273841858,
      "logps/chosen": -561.5999755859375,
      "logps/rejected": -1202.4000244140625,
      "loss": 0.7577,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.38945311307907104,
      "rewards/margins": 1.9915039539337158,
      "rewards/rejected": -2.381591796875,
      "step": 1130
    },
    {
      "epoch": 0.890894819466248,
      "grad_norm": 309.1407772630695,
      "learning_rate": 3.395866578648406e-07,
      "logits/chosen": -1.130859375,
      "logits/rejected": -1.0656249523162842,
      "logps/chosen": -601.7999877929688,
      "logps/rejected": -1029.5999755859375,
      "loss": 0.7834,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.544677734375,
      "rewards/margins": 1.382714867591858,
      "rewards/rejected": -1.926171898841858,
      "step": 1135
    },
    {
      "epoch": 0.8948194662480377,
      "grad_norm": 285.65500839764917,
      "learning_rate": 3.3798570486652726e-07,
      "logits/chosen": -1.189843773841858,
      "logits/rejected": -1.0574219226837158,
      "logps/chosen": -540.2000122070312,
      "logps/rejected": -1234.800048828125,
      "loss": 0.5914,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.2894531190395355,
      "rewards/margins": 3.0334229469299316,
      "rewards/rejected": -3.31884765625,
      "step": 1140
    },
    {
      "epoch": 0.8987441130298273,
      "grad_norm": 314.5151669481666,
      "learning_rate": 3.363806228950158e-07,
      "logits/chosen": -0.9515625238418579,
      "logits/rejected": -0.907031238079071,
      "logps/chosen": -592.5999755859375,
      "logps/rejected": -1350.0,
      "loss": 0.801,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8089843988418579,
      "rewards/margins": 3.384765625,
      "rewards/rejected": -4.192968845367432,
      "step": 1145
    },
    {
      "epoch": 0.902668759811617,
      "grad_norm": 338.22433992020797,
      "learning_rate": 3.3477148727322887e-07,
      "logits/chosen": -0.8412109613418579,
      "logits/rejected": -0.7461913824081421,
      "logps/chosen": -641.7999877929688,
      "logps/rejected": -1054.4000244140625,
      "loss": 1.1522,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.143457055091858,
      "rewards/margins": 1.562841773033142,
      "rewards/rejected": -2.7063231468200684,
      "step": 1150
    },
    {
      "epoch": 0.9065934065934066,
      "grad_norm": 302.0573923146702,
      "learning_rate": 3.3315837351431785e-07,
      "logits/chosen": -1.135156273841858,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -584.2000122070312,
      "logps/rejected": -1042.4000244140625,
      "loss": 0.7015,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5419921875,
      "rewards/margins": 1.915429711341858,
      "rewards/rejected": -2.4560546875,
      "step": 1155
    },
    {
      "epoch": 0.9105180533751962,
      "grad_norm": 263.90757906511146,
      "learning_rate": 3.315413573181193e-07,
      "logits/chosen": -0.962890625,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -590.7999877929688,
      "logps/rejected": -1049.5999755859375,
      "loss": 0.679,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.32512205839157104,
      "rewards/margins": 1.6982421875,
      "rewards/rejected": -2.024609327316284,
      "step": 1160
    },
    {
      "epoch": 0.9144427001569859,
      "grad_norm": 261.1770141332152,
      "learning_rate": 3.299205145676024e-07,
      "logits/chosen": -0.9292968511581421,
      "logits/rejected": -0.878125011920929,
      "logps/chosen": -500.20001220703125,
      "logps/rejected": -996.4000244140625,
      "loss": 0.7195,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.13481445610523224,
      "rewards/margins": 1.16552734375,
      "rewards/rejected": -1.2998535633087158,
      "step": 1165
    },
    {
      "epoch": 0.9183673469387755,
      "grad_norm": 244.1398117351398,
      "learning_rate": 3.2829592132530816e-07,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -1.028906226158142,
      "logps/chosen": -639.2000122070312,
      "logps/rejected": -1179.199951171875,
      "loss": 0.6767,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.514538586139679,
      "rewards/margins": 1.8740234375,
      "rewards/rejected": -2.3871092796325684,
      "step": 1170
    },
    {
      "epoch": 0.9222919937205651,
      "grad_norm": 222.95613828285838,
      "learning_rate": 3.266676538297797e-07,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.98828125,
      "logps/chosen": -603.7999877929688,
      "logps/rejected": -1016.7999877929688,
      "loss": 0.6312,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.628344714641571,
      "rewards/margins": 1.074804663658142,
      "rewards/rejected": -1.7039062976837158,
      "step": 1175
    },
    {
      "epoch": 0.9262166405023547,
      "grad_norm": 235.48561884362394,
      "learning_rate": 3.250357884919846e-07,
      "logits/chosen": -0.932421863079071,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -591.2000122070312,
      "logps/rejected": -1160.4000244140625,
      "loss": 0.6629,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8109375238418579,
      "rewards/margins": 1.766210913658142,
      "rewards/rejected": -2.575000047683716,
      "step": 1180
    },
    {
      "epoch": 0.9301412872841445,
      "grad_norm": 271.15610055524695,
      "learning_rate": 3.2340040189172926e-07,
      "logits/chosen": -1.084375023841858,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -579.4000244140625,
      "logps/rejected": -934.7999877929688,
      "loss": 0.6161,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7201172113418579,
      "rewards/margins": 1.4856445789337158,
      "rewards/rejected": -2.208203077316284,
      "step": 1185
    },
    {
      "epoch": 0.9340659340659341,
      "grad_norm": 495.266438717397,
      "learning_rate": 3.2176157077406505e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.96484375,
      "logps/chosen": -582.5999755859375,
      "logps/rejected": -933.5999755859375,
      "loss": 0.7798,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.563183605670929,
      "rewards/margins": 0.8550781011581421,
      "rewards/rejected": -1.41796875,
      "step": 1190
    },
    {
      "epoch": 0.9379905808477237,
      "grad_norm": 316.45417573937476,
      "learning_rate": 3.2011937204568693e-07,
      "logits/chosen": -1.125,
      "logits/rejected": -1.183203101158142,
      "logps/chosen": -567.5999755859375,
      "logps/rejected": -1078.4000244140625,
      "loss": 0.6703,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.42283934354782104,
      "rewards/margins": 1.0634765625,
      "rewards/rejected": -1.487890601158142,
      "step": 1195
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 476.4351289773554,
      "learning_rate": 3.1847388277132426e-07,
      "logits/chosen": -0.906054675579071,
      "logits/rejected": -0.873046875,
      "logps/chosen": -687.5999755859375,
      "logps/rejected": -1225.199951171875,
      "loss": 0.8809,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.32421875,
      "rewards/margins": 1.698339819908142,
      "rewards/rejected": -3.0234375,
      "step": 1200
    },
    {
      "epoch": 0.945839874411303,
      "grad_norm": 517.4397370746248,
      "learning_rate": 3.168251801701245e-07,
      "logits/chosen": -0.913281261920929,
      "logits/rejected": -0.8394531011581421,
      "logps/chosen": -651.0,
      "logps/rejected": -1340.800048828125,
      "loss": 0.6324,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2208983898162842,
      "rewards/margins": 1.76953125,
      "rewards/rejected": -2.9898438453674316,
      "step": 1205
    },
    {
      "epoch": 0.9497645211930926,
      "grad_norm": 304.12000021303965,
      "learning_rate": 3.1517334161202914e-07,
      "logits/chosen": -1.079687476158142,
      "logits/rejected": -0.9375,
      "logps/chosen": -639.2000122070312,
      "logps/rejected": -979.5999755859375,
      "loss": 0.9327,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.091796875,
      "rewards/margins": 1.004638671875,
      "rewards/rejected": -2.0953125953674316,
      "step": 1210
    },
    {
      "epoch": 0.9536891679748822,
      "grad_norm": 334.0239158245771,
      "learning_rate": 3.1351844461414345e-07,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -1.0089843273162842,
      "logps/chosen": -613.5999755859375,
      "logps/rejected": -1060.800048828125,
      "loss": 0.8772,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8257812261581421,
      "rewards/margins": 0.6606689691543579,
      "rewards/rejected": -1.485937476158142,
      "step": 1215
    },
    {
      "epoch": 0.957613814756672,
      "grad_norm": 278.38604365857987,
      "learning_rate": 3.118605668370983e-07,
      "logits/chosen": -1.203125,
      "logits/rejected": -1.2078125476837158,
      "logps/chosen": -577.2000122070312,
      "logps/rejected": -1177.199951171875,
      "loss": 0.7326,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4749511778354645,
      "rewards/margins": 2.088330030441284,
      "rewards/rejected": -2.565234422683716,
      "step": 1220
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 325.77563124535146,
      "learning_rate": 3.1019978608140577e-07,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -1.0910155773162842,
      "logps/chosen": -602.4000244140625,
      "logps/rejected": -1108.800048828125,
      "loss": 0.7811,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5126312375068665,
      "rewards/margins": 1.182714819908142,
      "rewards/rejected": -1.694238305091858,
      "step": 1225
    },
    {
      "epoch": 0.9654631083202512,
      "grad_norm": 293.13186983822675,
      "learning_rate": 3.085361802838086e-07,
      "logits/chosen": -0.9320312738418579,
      "logits/rejected": -1.0148437023162842,
      "logps/chosen": -663.0,
      "logps/rejected": -1197.199951171875,
      "loss": 0.83,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.141210913658142,
      "rewards/margins": 2.3662109375,
      "rewards/rejected": -3.506640672683716,
      "step": 1230
    },
    {
      "epoch": 0.9693877551020408,
      "grad_norm": 252.5929528657482,
      "learning_rate": 3.068698275136223e-07,
      "logits/chosen": -0.9703124761581421,
      "logits/rejected": -0.9039062261581421,
      "logps/chosen": -916.7999877929688,
      "logps/rejected": -1541.199951171875,
      "loss": 1.4042,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1468749046325684,
      "rewards/margins": 5.546875,
      "rewards/rejected": -7.707812309265137,
      "step": 1235
    },
    {
      "epoch": 0.9733124018838305,
      "grad_norm": 246.11746402027578,
      "learning_rate": 3.0520080596907146e-07,
      "logits/chosen": -0.9710937738418579,
      "logits/rejected": -0.8818359375,
      "logps/chosen": -712.4000244140625,
      "logps/rejected": -1332.4000244140625,
      "loss": 1.4745,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.745703101158142,
      "rewards/margins": 1.866796851158142,
      "rewards/rejected": -3.612499952316284,
      "step": 1240
    },
    {
      "epoch": 0.9772370486656201,
      "grad_norm": 190.0213172201279,
      "learning_rate": 3.035291939736206e-07,
      "logits/chosen": -1.105078101158142,
      "logits/rejected": -1.120703101158142,
      "logps/chosen": -591.4000244140625,
      "logps/rejected": -1166.800048828125,
      "loss": 0.7466,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5472656488418579,
      "rewards/margins": 1.26220703125,
      "rewards/rejected": -1.8074219226837158,
      "step": 1245
    },
    {
      "epoch": 0.9811616954474097,
      "grad_norm": 711.6477392126861,
      "learning_rate": 3.0185506997229825e-07,
      "logits/chosen": -1.122656226158142,
      "logits/rejected": -1.140234351158142,
      "logps/chosen": -809.7999877929688,
      "logps/rejected": -1531.5999755859375,
      "loss": 1.6856,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.076074242591858,
      "rewards/margins": 2.136425733566284,
      "rewards/rejected": -3.2125000953674316,
      "step": 1250
    },
    {
      "epoch": 0.9850863422291993,
      "grad_norm": 213.27786016208213,
      "learning_rate": 3.001785125280157e-07,
      "logits/chosen": -1.089453101158142,
      "logits/rejected": -1.0849609375,
      "logps/chosen": -558.5999755859375,
      "logps/rejected": -850.0,
      "loss": 0.6917,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5291992425918579,
      "rewards/margins": 0.9897705316543579,
      "rewards/rejected": -1.516992211341858,
      "step": 1255
    },
    {
      "epoch": 0.989010989010989,
      "grad_norm": 331.99543845036806,
      "learning_rate": 2.9849960031788036e-07,
      "logits/chosen": -1.171875,
      "logits/rejected": -1.0851562023162842,
      "logps/chosen": -656.4000244140625,
      "logps/rejected": -1604.4000244140625,
      "loss": 0.7309,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.162109375,
      "rewards/margins": 4.470117092132568,
      "rewards/rejected": -5.641406059265137,
      "step": 1260
    },
    {
      "epoch": 0.9929356357927787,
      "grad_norm": 393.0610896672158,
      "learning_rate": 2.9681841212950376e-07,
      "logits/chosen": -1.076171875,
      "logits/rejected": -0.9437500238418579,
      "logps/chosen": -726.4000244140625,
      "logps/rejected": -1034.4000244140625,
      "loss": 0.7536,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.318359375,
      "rewards/margins": 1.6257812976837158,
      "rewards/rejected": -2.94140625,
      "step": 1265
    },
    {
      "epoch": 0.9968602825745683,
      "grad_norm": 249.17999822811694,
      "learning_rate": 2.9513502685730353e-07,
      "logits/chosen": -1.1671874523162842,
      "logits/rejected": -0.928906261920929,
      "logps/chosen": -660.0,
      "logps/rejected": -1128.0,
      "loss": 0.7201,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.969433605670929,
      "rewards/margins": 1.193945288658142,
      "rewards/rejected": -2.160937547683716,
      "step": 1270
    },
    {
      "epoch": 1.000784929356358,
      "grad_norm": 98.41903881829174,
      "learning_rate": 2.9344952349880197e-07,
      "logits/chosen": -1.2109375,
      "logits/rejected": -1.1789062023162842,
      "logps/chosen": -578.0,
      "logps/rejected": -1147.199951171875,
      "loss": 0.6524,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.3814941346645355,
      "rewards/margins": 2.163281202316284,
      "rewards/rejected": -2.5439453125,
      "step": 1275
    },
    {
      "epoch": 1.0047095761381475,
      "grad_norm": 107.607979182684,
      "learning_rate": 2.917619811509184e-07,
      "logits/chosen": -1.0789062976837158,
      "logits/rejected": -1.058203101158142,
      "logps/chosen": -481.6000061035156,
      "logps/rejected": -1256.4000244140625,
      "loss": 0.1581,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.6390136480331421,
      "rewards/margins": 5.393750190734863,
      "rewards/rejected": -4.755468845367432,
      "step": 1280
    },
    {
      "epoch": 1.0086342229199372,
      "grad_norm": 118.02113372092037,
      "learning_rate": 2.900724790062571e-07,
      "logits/chosen": -1.01953125,
      "logits/rejected": -1.037500023841858,
      "logps/chosen": -533.7999877929688,
      "logps/rejected": -1654.4000244140625,
      "loss": 0.1361,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0027587891090661287,
      "rewards/margins": 5.876562595367432,
      "rewards/rejected": -5.865624904632568,
      "step": 1285
    },
    {
      "epoch": 1.012558869701727,
      "grad_norm": 39.996570400883755,
      "learning_rate": 2.8838109634939166e-07,
      "logits/chosen": -0.9136718511581421,
      "logits/rejected": -0.8541015386581421,
      "logps/chosen": -532.4000244140625,
      "logps/rejected": -1520.0,
      "loss": 0.1005,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.44013673067092896,
      "rewards/margins": 6.842187404632568,
      "rewards/rejected": -7.282812595367432,
      "step": 1290
    },
    {
      "epoch": 1.0164835164835164,
      "grad_norm": 67.43626486171631,
      "learning_rate": 2.866879125531437e-07,
      "logits/chosen": -0.927929699420929,
      "logits/rejected": -0.824023425579071,
      "logps/chosen": -577.4000244140625,
      "logps/rejected": -1557.5999755859375,
      "loss": 0.0776,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.432037353515625,
      "rewards/margins": 6.675000190734863,
      "rewards/rejected": -7.109375,
      "step": 1295
    },
    {
      "epoch": 1.0204081632653061,
      "grad_norm": 69.71909036492362,
      "learning_rate": 2.849930070748581e-07,
      "logits/chosen": -0.8447265625,
      "logits/rejected": -0.5718139410018921,
      "logps/chosen": -615.5999755859375,
      "logps/rejected": -1546.4000244140625,
      "loss": 0.1077,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.68212890625,
      "rewards/margins": 6.004687309265137,
      "rewards/rejected": -6.678124904632568,
      "step": 1300
    },
    {
      "epoch": 1.0243328100470959,
      "grad_norm": 84.65721114840902,
      "learning_rate": 2.8329645945267476e-07,
      "logits/chosen": -0.779296875,
      "logits/rejected": -0.6171875,
      "logps/chosen": -618.0,
      "logps/rejected": -1555.199951171875,
      "loss": 0.1185,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6380859613418579,
      "rewards/margins": 7.240624904632568,
      "rewards/rejected": -7.896874904632568,
      "step": 1305
    },
    {
      "epoch": 1.0282574568288854,
      "grad_norm": 173.95148349550433,
      "learning_rate": 2.815983493017957e-07,
      "logits/chosen": -0.891796886920929,
      "logits/rejected": -0.7198241949081421,
      "logps/chosen": -624.7999877929688,
      "logps/rejected": -1587.199951171875,
      "loss": 0.1312,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.606152355670929,
      "rewards/margins": 6.310937404632568,
      "rewards/rejected": -6.918749809265137,
      "step": 1310
    },
    {
      "epoch": 1.032182103610675,
      "grad_norm": 31.642253601362697,
      "learning_rate": 2.7989875631074875e-07,
      "logits/chosen": -0.900390625,
      "logits/rejected": -0.63623046875,
      "logps/chosen": -635.2000122070312,
      "logps/rejected": -1746.800048828125,
      "loss": 0.1517,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.4698730409145355,
      "rewards/margins": 7.428124904632568,
      "rewards/rejected": -7.912499904632568,
      "step": 1315
    },
    {
      "epoch": 1.0361067503924646,
      "grad_norm": 169.31431264295642,
      "learning_rate": 2.7819776023764835e-07,
      "logits/chosen": -0.91015625,
      "logits/rejected": -0.7789062261581421,
      "logps/chosen": -552.4000244140625,
      "logps/rejected": -1323.199951171875,
      "loss": 0.1137,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.33983153104782104,
      "rewards/margins": 5.418749809265137,
      "rewards/rejected": -5.764062404632568,
      "step": 1320
    },
    {
      "epoch": 1.0400313971742543,
      "grad_norm": 712.5619535813084,
      "learning_rate": 2.764954409064522e-07,
      "logits/chosen": -1.0007812976837158,
      "logits/rejected": -0.7450195550918579,
      "logps/chosen": -815.7999877929688,
      "logps/rejected": -1863.199951171875,
      "loss": 0.4991,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.02001953125,
      "rewards/margins": 7.852343559265137,
      "rewards/rejected": -8.875,
      "step": 1325
    },
    {
      "epoch": 1.043956043956044,
      "grad_norm": 336.74941435443867,
      "learning_rate": 2.7479187820321583e-07,
      "logits/chosen": -0.868359386920929,
      "logits/rejected": -0.7115722894668579,
      "logps/chosen": -625.5999755859375,
      "logps/rejected": -1667.199951171875,
      "loss": 0.22,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.545520007610321,
      "rewards/margins": 6.607812404632568,
      "rewards/rejected": -7.15625,
      "step": 1330
    },
    {
      "epoch": 1.0478806907378335,
      "grad_norm": 69.78251920352533,
      "learning_rate": 2.7308715207234324e-07,
      "logits/chosen": -1.115625023841858,
      "logits/rejected": -0.953906238079071,
      "logps/chosen": -598.2000122070312,
      "logps/rejected": -1597.199951171875,
      "loss": 0.4481,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5715576410293579,
      "rewards/margins": 5.154687404632568,
      "rewards/rejected": -5.723437309265137,
      "step": 1335
    },
    {
      "epoch": 1.0518053375196232,
      "grad_norm": 132.43995724701352,
      "learning_rate": 2.7138134251283555e-07,
      "logits/chosen": -1.11328125,
      "logits/rejected": -1.0496094226837158,
      "logps/chosen": -581.2000122070312,
      "logps/rejected": -1271.199951171875,
      "loss": 0.1268,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.02988281287252903,
      "rewards/margins": 4.9140625,
      "rewards/rejected": -4.885937690734863,
      "step": 1340
    },
    {
      "epoch": 1.055729984301413,
      "grad_norm": 190.4476673179279,
      "learning_rate": 2.6967452957453697e-07,
      "logits/chosen": -1.1062500476837158,
      "logits/rejected": -0.995312511920929,
      "logps/chosen": -483.20001220703125,
      "logps/rejected": -1460.0,
      "loss": 0.1699,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41552734375,
      "rewards/margins": 5.606249809265137,
      "rewards/rejected": -5.193749904632568,
      "step": 1345
    },
    {
      "epoch": 1.0596546310832025,
      "grad_norm": 116.65446374199605,
      "learning_rate": 2.679667933543776e-07,
      "logits/chosen": -1.0505859851837158,
      "logits/rejected": -0.9125000238418579,
      "logps/chosen": -575.4000244140625,
      "logps/rejected": -1306.800048828125,
      "loss": 0.2268,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.12412109225988388,
      "rewards/margins": 4.984375,
      "rewards/rejected": -5.106249809265137,
      "step": 1350
    },
    {
      "epoch": 1.0635792778649922,
      "grad_norm": 69.7882236167391,
      "learning_rate": 2.662582139926156e-07,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.693554699420929,
      "logps/chosen": -568.4000244140625,
      "logps/rejected": -1558.800048828125,
      "loss": 0.0748,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3160644471645355,
      "rewards/margins": 6.193749904632568,
      "rewards/rejected": -6.506249904632568,
      "step": 1355
    },
    {
      "epoch": 1.0675039246467817,
      "grad_norm": 162.80828970274823,
      "learning_rate": 2.645488716690756e-07,
      "logits/chosen": -0.932812511920929,
      "logits/rejected": -0.6470702886581421,
      "logps/chosen": -611.4000244140625,
      "logps/rejected": -1682.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.069433569908142,
      "rewards/margins": 6.0078125,
      "rewards/rejected": -7.074999809265137,
      "step": 1360
    },
    {
      "epoch": 1.0714285714285714,
      "grad_norm": 147.26479734589603,
      "learning_rate": 2.628388465993864e-07,
      "logits/chosen": -0.8623046875,
      "logits/rejected": -0.59912109375,
      "logps/chosen": -701.5999755859375,
      "logps/rejected": -1696.800048828125,
      "loss": 0.12,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.155664086341858,
      "rewards/margins": 7.34375,
      "rewards/rejected": -8.5,
      "step": 1365
    },
    {
      "epoch": 1.0753532182103611,
      "grad_norm": 107.21597245368804,
      "learning_rate": 2.611282190312164e-07,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.818359375,
      "logps/chosen": -578.7999877929688,
      "logps/rejected": -1656.800048828125,
      "loss": 0.1064,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4778686463832855,
      "rewards/margins": 6.967187404632568,
      "rewards/rejected": -7.443749904632568,
      "step": 1370
    },
    {
      "epoch": 1.0792778649921506,
      "grad_norm": 130.43821544121533,
      "learning_rate": 2.5941706924050827e-07,
      "logits/chosen": -1.122656226158142,
      "logits/rejected": -0.8330078125,
      "logps/chosen": -547.2000122070312,
      "logps/rejected": -1308.4000244140625,
      "loss": 0.1458,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.15585938096046448,
      "rewards/margins": 6.084374904632568,
      "rewards/rejected": -6.248437404632568,
      "step": 1375
    },
    {
      "epoch": 1.0832025117739403,
      "grad_norm": 152.78431846171227,
      "learning_rate": 2.577054775277111e-07,
      "logits/chosen": -1.137109398841858,
      "logits/rejected": -0.985546886920929,
      "logps/chosen": -591.4000244140625,
      "logps/rejected": -1476.0,
      "loss": 0.13,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.18574218451976776,
      "rewards/margins": 6.0625,
      "rewards/rejected": -6.25,
      "step": 1380
    },
    {
      "epoch": 1.08712715855573,
      "grad_norm": 384.70525911252696,
      "learning_rate": 2.559935242140125e-07,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.901171863079071,
      "logps/chosen": -586.0,
      "logps/rejected": -2040.0,
      "loss": 0.1292,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0331573486328125,
      "rewards/margins": 7.921875,
      "rewards/rejected": -7.887499809265137,
      "step": 1385
    },
    {
      "epoch": 1.0910518053375196,
      "grad_norm": 51.87380347397637,
      "learning_rate": 2.542812896375693e-07,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.8910156488418579,
      "logps/chosen": -579.7999877929688,
      "logps/rejected": -1701.5999755859375,
      "loss": 0.1094,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4830078184604645,
      "rewards/margins": 7.212500095367432,
      "rewards/rejected": -7.690625190734863,
      "step": 1390
    },
    {
      "epoch": 1.0949764521193093,
      "grad_norm": 266.1811375045263,
      "learning_rate": 2.525688541497371e-07,
      "logits/chosen": -0.7740234136581421,
      "logits/rejected": -0.6587890386581421,
      "logps/chosen": -676.7999877929688,
      "logps/rejected": -1713.199951171875,
      "loss": 0.1374,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6123046875,
      "rewards/margins": 6.245312690734863,
      "rewards/rejected": -7.846875190734863,
      "step": 1395
    },
    {
      "epoch": 1.098901098901099,
      "grad_norm": 67.3131529688098,
      "learning_rate": 2.5085629811130016e-07,
      "logits/chosen": -0.9195312261581421,
      "logits/rejected": -0.792187511920929,
      "logps/chosen": -714.4000244140625,
      "logps/rejected": -2083.199951171875,
      "loss": 0.1882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.146582007408142,
      "rewards/margins": 7.853125095367432,
      "rewards/rejected": -9.012499809265137,
      "step": 1400
    },
    {
      "epoch": 1.1028257456828885,
      "grad_norm": 56.842077887579954,
      "learning_rate": 2.4914370188869977e-07,
      "logits/chosen": -0.90283203125,
      "logits/rejected": -0.727246105670929,
      "logps/chosen": -539.7999877929688,
      "logps/rejected": -1471.199951171875,
      "loss": 0.1249,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.49580079317092896,
      "rewards/margins": 6.546875,
      "rewards/rejected": -7.040625095367432,
      "step": 1405
    },
    {
      "epoch": 1.1067503924646782,
      "grad_norm": 105.04148603255796,
      "learning_rate": 2.4743114585026285e-07,
      "logits/chosen": -1.0730469226837158,
      "logits/rejected": -0.985546886920929,
      "logps/chosen": -553.2000122070312,
      "logps/rejected": -1932.800048828125,
      "loss": 0.224,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3048339784145355,
      "rewards/margins": 6.321875095367432,
      "rewards/rejected": -6.625,
      "step": 1410
    },
    {
      "epoch": 1.110675039246468,
      "grad_norm": 125.78775938335988,
      "learning_rate": 2.4571871036243077e-07,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.8169921636581421,
      "logps/chosen": -566.5999755859375,
      "logps/rejected": -1528.800048828125,
      "loss": 0.0991,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.12841796875,
      "rewards/margins": 6.546875,
      "rewards/rejected": -6.684374809265137,
      "step": 1415
    },
    {
      "epoch": 1.1145996860282574,
      "grad_norm": 43.68314660123009,
      "learning_rate": 2.440064757859875e-07,
      "logits/chosen": -0.889843761920929,
      "logits/rejected": -0.6107422113418579,
      "logps/chosen": -583.0,
      "logps/rejected": -1446.4000244140625,
      "loss": 0.1232,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.44371336698532104,
      "rewards/margins": 6.521874904632568,
      "rewards/rejected": -6.96875,
      "step": 1420
    },
    {
      "epoch": 1.1185243328100472,
      "grad_norm": 74.94833562590675,
      "learning_rate": 2.422945224722889e-07,
      "logits/chosen": -1.030859351158142,
      "logits/rejected": -0.831250011920929,
      "logps/chosen": -549.0,
      "logps/rejected": -2214.800048828125,
      "loss": 0.1426,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4521728456020355,
      "rewards/margins": 7.660937309265137,
      "rewards/rejected": -8.110937118530273,
      "step": 1425
    },
    {
      "epoch": 1.1224489795918366,
      "grad_norm": 21.93435724493209,
      "learning_rate": 2.4058293075949166e-07,
      "logits/chosen": -0.9068359136581421,
      "logits/rejected": -0.550244152545929,
      "logps/chosen": -587.5999755859375,
      "logps/rejected": -1521.5999755859375,
      "loss": 0.0836,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.62799072265625,
      "rewards/margins": 6.756249904632568,
      "rewards/rejected": -7.384375095367432,
      "step": 1430
    },
    {
      "epoch": 1.1263736263736264,
      "grad_norm": 123.87391581807867,
      "learning_rate": 2.3887178096878363e-07,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.7354491949081421,
      "logps/chosen": -802.2000122070312,
      "logps/rejected": -1754.4000244140625,
      "loss": 0.5289,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.0666015148162842,
      "rewards/margins": 7.134375095367432,
      "rewards/rejected": -8.203125,
      "step": 1435
    },
    {
      "epoch": 1.130298273155416,
      "grad_norm": 103.30015258960564,
      "learning_rate": 2.3716115340061372e-07,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -608.4000244140625,
      "logps/rejected": -1502.4000244140625,
      "loss": 0.281,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5976928472518921,
      "rewards/margins": 5.640625,
      "rewards/rejected": -6.240624904632568,
      "step": 1440
    },
    {
      "epoch": 1.1342229199372056,
      "grad_norm": 68.8847942778874,
      "learning_rate": 2.3545112833092436e-07,
      "logits/chosen": -0.948046863079071,
      "logits/rejected": -0.8365234136581421,
      "logps/chosen": -552.4000244140625,
      "logps/rejected": -1809.199951171875,
      "loss": 0.083,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.24262695014476776,
      "rewards/margins": 7.996874809265137,
      "rewards/rejected": -8.243749618530273,
      "step": 1445
    },
    {
      "epoch": 1.1381475667189953,
      "grad_norm": 49.57618307672172,
      "learning_rate": 2.3374178600738437e-07,
      "logits/chosen": -1.158593773841858,
      "logits/rejected": -0.80810546875,
      "logps/chosen": -550.5999755859375,
      "logps/rejected": -1322.0,
      "loss": 0.0974,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.31635743379592896,
      "rewards/margins": 6.071875095367432,
      "rewards/rejected": -5.764062404632568,
      "step": 1450
    },
    {
      "epoch": 1.1420722135007848,
      "grad_norm": 88.23386656996844,
      "learning_rate": 2.3203320664562233e-07,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.8828125,
      "logps/chosen": -546.5999755859375,
      "logps/rejected": -1588.800048828125,
      "loss": 0.1391,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.04604492336511612,
      "rewards/margins": 7.199999809265137,
      "rewards/rejected": -7.25,
      "step": 1455
    },
    {
      "epoch": 1.1459968602825745,
      "grad_norm": 72.97888694374258,
      "learning_rate": 2.3032547042546306e-07,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.8912109136581421,
      "logps/chosen": -566.7999877929688,
      "logps/rejected": -1552.4000244140625,
      "loss": 0.1412,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.11048583686351776,
      "rewards/margins": 5.295312404632568,
      "rewards/rejected": -5.409375190734863,
      "step": 1460
    },
    {
      "epoch": 1.1499215070643642,
      "grad_norm": 372.93514069307156,
      "learning_rate": 2.2861865748716445e-07,
      "logits/chosen": -0.888671875,
      "logits/rejected": -0.664257824420929,
      "logps/chosen": -581.0,
      "logps/rejected": -1660.800048828125,
      "loss": 0.1551,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5367676019668579,
      "rewards/margins": 7.215624809265137,
      "rewards/rejected": -7.743750095367432,
      "step": 1465
    },
    {
      "epoch": 1.1538461538461537,
      "grad_norm": 222.5921504124036,
      "learning_rate": 2.2691284792765674e-07,
      "logits/chosen": -1.077734351158142,
      "logits/rejected": -0.6358398199081421,
      "logps/chosen": -584.2000122070312,
      "logps/rejected": -1524.800048828125,
      "loss": 0.1217,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.386474609375,
      "rewards/margins": 6.793749809265137,
      "rewards/rejected": -7.175000190734863,
      "step": 1470
    },
    {
      "epoch": 1.1577708006279435,
      "grad_norm": 173.83999045641218,
      "learning_rate": 2.252081217967842e-07,
      "logits/chosen": -1.0947265625,
      "logits/rejected": -0.8573242425918579,
      "logps/chosen": -531.2000122070312,
      "logps/rejected": -1484.800048828125,
      "loss": 0.1087,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.20942382514476776,
      "rewards/margins": 6.084374904632568,
      "rewards/rejected": -6.285937309265137,
      "step": 1475
    },
    {
      "epoch": 1.1616954474097332,
      "grad_norm": 303.1628185473354,
      "learning_rate": 2.2350455909354775e-07,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.875195324420929,
      "logps/chosen": -573.0,
      "logps/rejected": -1644.0,
      "loss": 0.1513,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5539795160293579,
      "rewards/margins": 6.53125,
      "rewards/rejected": -7.078125,
      "step": 1480
    },
    {
      "epoch": 1.1656200941915227,
      "grad_norm": 82.64056565455722,
      "learning_rate": 2.2180223976235169e-07,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.899218738079071,
      "logps/chosen": -518.4000244140625,
      "logps/rejected": -1760.800048828125,
      "loss": 0.0873,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.23056641221046448,
      "rewards/margins": 7.915625095367432,
      "rewards/rejected": -8.140625,
      "step": 1485
    },
    {
      "epoch": 1.1695447409733124,
      "grad_norm": 76.04814858731045,
      "learning_rate": 2.2010124368925125e-07,
      "logits/chosen": -1.0519530773162842,
      "logits/rejected": -0.891406238079071,
      "logps/chosen": -499.79998779296875,
      "logps/rejected": -1814.4000244140625,
      "loss": 0.1072,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0010528564453125,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.651562690734863,
      "step": 1490
    },
    {
      "epoch": 1.1734693877551021,
      "grad_norm": 71.26544236128251,
      "learning_rate": 2.184016506982043e-07,
      "logits/chosen": -0.9898437261581421,
      "logits/rejected": -0.7933593988418579,
      "logps/chosen": -571.7999877929688,
      "logps/rejected": -1554.4000244140625,
      "loss": 0.1234,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.18857422471046448,
      "rewards/margins": 6.509375095367432,
      "rewards/rejected": -6.693749904632568,
      "step": 1495
    },
    {
      "epoch": 1.1773940345368916,
      "grad_norm": 45.53432286866041,
      "learning_rate": 2.1670354054732525e-07,
      "logits/chosen": -1.136328101158142,
      "logits/rejected": -0.8871093988418579,
      "logps/chosen": -576.5999755859375,
      "logps/rejected": -1828.0,
      "loss": 0.0856,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3628906309604645,
      "rewards/margins": 7.034375190734863,
      "rewards/rejected": -7.396874904632568,
      "step": 1500
    },
    {
      "epoch": 1.1813186813186813,
      "grad_norm": 49.95219415799116,
      "learning_rate": 2.150069929251419e-07,
      "logits/chosen": -0.95947265625,
      "logits/rejected": -0.54736328125,
      "logps/chosen": -546.4000244140625,
      "logps/rejected": -1407.199951171875,
      "loss": 0.1014,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.22026367485523224,
      "rewards/margins": 6.912499904632568,
      "rewards/rejected": -7.134375095367432,
      "step": 1505
    },
    {
      "epoch": 1.185243328100471,
      "grad_norm": 157.93143816925812,
      "learning_rate": 2.1331208744685633e-07,
      "logits/chosen": -0.908203125,
      "logits/rejected": -0.744824230670929,
      "logps/chosen": -625.7999877929688,
      "logps/rejected": -1979.199951171875,
      "loss": 0.1036,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.294921875,
      "rewards/margins": 8.131250381469727,
      "rewards/rejected": -8.434374809265137,
      "step": 1510
    },
    {
      "epoch": 1.1891679748822606,
      "grad_norm": 82.36670029126445,
      "learning_rate": 2.1161890365060837e-07,
      "logits/chosen": -0.923828125,
      "logits/rejected": -0.8095703125,
      "logps/chosen": -591.0,
      "logps/rejected": -2082.39990234375,
      "loss": 0.2154,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.6912841796875,
      "rewards/margins": 9.131250381469727,
      "rewards/rejected": -9.834375381469727,
      "step": 1515
    },
    {
      "epoch": 1.1930926216640503,
      "grad_norm": 172.19564584167256,
      "learning_rate": 2.0992752099374284e-07,
      "logits/chosen": -0.9085937738418579,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -563.0,
      "logps/rejected": -1467.5999755859375,
      "loss": 0.1626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6578124761581421,
      "rewards/margins": 6.323437690734863,
      "rewards/rejected": -6.981249809265137,
      "step": 1520
    },
    {
      "epoch": 1.19701726844584,
      "grad_norm": 89.67310788853288,
      "learning_rate": 2.0823801884908166e-07,
      "logits/chosen": -0.793652355670929,
      "logits/rejected": -0.8236328363418579,
      "logps/chosen": -586.5999755859375,
      "logps/rejected": -1679.199951171875,
      "loss": 0.1118,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3154052793979645,
      "rewards/margins": 7.796875,
      "rewards/rejected": -8.109375,
      "step": 1525
    },
    {
      "epoch": 1.2009419152276295,
      "grad_norm": 188.41924165670733,
      "learning_rate": 2.0655047650119812e-07,
      "logits/chosen": -1.0050780773162842,
      "logits/rejected": -0.767822265625,
      "logps/chosen": -558.2000122070312,
      "logps/rejected": -1508.800048828125,
      "loss": 0.0904,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09091796725988388,
      "rewards/margins": 7.340624809265137,
      "rewards/rejected": -7.434374809265137,
      "step": 1530
    },
    {
      "epoch": 1.2048665620094192,
      "grad_norm": 77.2388538739631,
      "learning_rate": 2.048649731426965e-07,
      "logits/chosen": -0.9990234375,
      "logits/rejected": -0.845019519329071,
      "logps/chosen": -605.5999755859375,
      "logps/rejected": -1904.800048828125,
      "loss": 0.1112,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.41718751192092896,
      "rewards/margins": 8.565625190734863,
      "rewards/rejected": -8.984375,
      "step": 1535
    },
    {
      "epoch": 1.2087912087912087,
      "grad_norm": 42.94777930663939,
      "learning_rate": 2.0318158787049633e-07,
      "logits/chosen": -0.896484375,
      "logits/rejected": -0.7632812261581421,
      "logps/chosen": -534.5999755859375,
      "logps/rejected": -1813.5999755859375,
      "loss": 0.1017,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3238769471645355,
      "rewards/margins": 8.181249618530273,
      "rewards/rejected": -8.496874809265137,
      "step": 1540
    },
    {
      "epoch": 1.2127158555729984,
      "grad_norm": 36.798555665071184,
      "learning_rate": 2.0150039968211957e-07,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.5987548828125,
      "logps/chosen": -600.2000122070312,
      "logps/rejected": -1437.5999755859375,
      "loss": 0.1967,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.752636730670929,
      "rewards/margins": 6.474999904632568,
      "rewards/rejected": -7.231249809265137,
      "step": 1545
    },
    {
      "epoch": 1.2166405023547882,
      "grad_norm": 53.01405203699919,
      "learning_rate": 1.9982148747198433e-07,
      "logits/chosen": -0.9157959222793579,
      "logits/rejected": -0.7223144769668579,
      "logps/chosen": -536.2000122070312,
      "logps/rejected": -1613.5999755859375,
      "loss": 0.2403,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4642333984375,
      "rewards/margins": 7.410937309265137,
      "rewards/rejected": -7.8828125,
      "step": 1550
    },
    {
      "epoch": 1.2205651491365777,
      "grad_norm": 59.52158158130305,
      "learning_rate": 1.9814493002770183e-07,
      "logits/chosen": -1.107812523841858,
      "logits/rejected": -0.7925781011581421,
      "logps/chosen": -519.2000122070312,
      "logps/rejected": -1433.199951171875,
      "loss": 0.1317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02707214280962944,
      "rewards/margins": 6.504687309265137,
      "rewards/rejected": -6.485937595367432,
      "step": 1555
    },
    {
      "epoch": 1.2244897959183674,
      "grad_norm": 114.27696835695879,
      "learning_rate": 1.964708060263794e-07,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.88525390625,
      "logps/chosen": -516.5999755859375,
      "logps/rejected": -1718.4000244140625,
      "loss": 0.1104,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.13553467392921448,
      "rewards/margins": 6.675000190734863,
      "rewards/rejected": -6.537499904632568,
      "step": 1560
    },
    {
      "epoch": 1.2284144427001569,
      "grad_norm": 105.4407119554599,
      "learning_rate": 1.947991940309286e-07,
      "logits/chosen": -0.95263671875,
      "logits/rejected": -0.657421886920929,
      "logps/chosen": -568.0,
      "logps/rejected": -1569.5999755859375,
      "loss": 0.095,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.14287109673023224,
      "rewards/margins": 6.887499809265137,
      "rewards/rejected": -7.037499904632568,
      "step": 1565
    },
    {
      "epoch": 1.2323390894819466,
      "grad_norm": 64.92566614019948,
      "learning_rate": 1.9313017248637767e-07,
      "logits/chosen": -1.133203148841858,
      "logits/rejected": -0.947265625,
      "logps/chosen": -596.7999877929688,
      "logps/rejected": -1739.199951171875,
      "loss": 0.1277,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.40839844942092896,
      "rewards/margins": 7.490624904632568,
      "rewards/rejected": -7.903124809265137,
      "step": 1570
    },
    {
      "epoch": 1.2362637362637363,
      "grad_norm": 43.98753049798123,
      "learning_rate": 1.9146381971619138e-07,
      "logits/chosen": -0.970141589641571,
      "logits/rejected": -0.6372314691543579,
      "logps/chosen": -542.0,
      "logps/rejected": -1614.4000244140625,
      "loss": 0.0701,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.18754883110523224,
      "rewards/margins": 7.512499809265137,
      "rewards/rejected": -7.328125,
      "step": 1575
    },
    {
      "epoch": 1.2401883830455258,
      "grad_norm": 49.01872725261063,
      "learning_rate": 1.8980021391859423e-07,
      "logits/chosen": -0.920703113079071,
      "logits/rejected": -0.7352539300918579,
      "logps/chosen": -595.2000122070312,
      "logps/rejected": -1844.0,
      "loss": 0.1045,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.5687500238418579,
      "rewards/margins": 7.403124809265137,
      "rewards/rejected": -7.96875,
      "step": 1580
    },
    {
      "epoch": 1.2441130298273155,
      "grad_norm": 55.33372047862798,
      "learning_rate": 1.8813943316290174e-07,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.8138672113418579,
      "logps/chosen": -578.0,
      "logps/rejected": -1996.0,
      "loss": 0.1018,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.550000011920929,
      "rewards/margins": 8.318750381469727,
      "rewards/rejected": -8.878125190734863,
      "step": 1585
    },
    {
      "epoch": 1.2480376766091053,
      "grad_norm": 134.965386255027,
      "learning_rate": 1.8648155538585658e-07,
      "logits/chosen": -1.03515625,
      "logits/rejected": -0.818164050579071,
      "logps/chosen": -598.4000244140625,
      "logps/rejected": -1403.5999755859375,
      "loss": 0.1017,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3663085997104645,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.162499904632568,
      "step": 1590
    },
    {
      "epoch": 1.2519623233908947,
      "grad_norm": 452.8361236765151,
      "learning_rate": 1.8482665838797084e-07,
      "logits/chosen": -1.185156226158142,
      "logits/rejected": -1.0154297351837158,
      "logps/chosen": -622.7999877929688,
      "logps/rejected": -1829.5999755859375,
      "loss": 0.1353,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.39213865995407104,
      "rewards/margins": 7.737500190734863,
      "rewards/rejected": -8.128125190734863,
      "step": 1595
    },
    {
      "epoch": 1.2558869701726845,
      "grad_norm": 53.220418089301766,
      "learning_rate": 1.8317481982987553e-07,
      "logits/chosen": -1.058984398841858,
      "logits/rejected": -0.8685547113418579,
      "logps/chosen": -534.0,
      "logps/rejected": -1697.5999755859375,
      "loss": 0.3418,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.10637817531824112,
      "rewards/margins": 7.490624904632568,
      "rewards/rejected": -7.592187404632568,
      "step": 1600
    },
    {
      "epoch": 1.2598116169544742,
      "grad_norm": 40.22767193415629,
      "learning_rate": 1.8152611722867572e-07,
      "logits/chosen": -1.1218750476837158,
      "logits/rejected": -0.8653808832168579,
      "logps/chosen": -561.2000122070312,
      "logps/rejected": -1360.800048828125,
      "loss": 0.2577,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.13457031548023224,
      "rewards/margins": 5.540625095367432,
      "rewards/rejected": -5.689062595367432,
      "step": 1605
    },
    {
      "epoch": 1.2637362637362637,
      "grad_norm": 116.51510347929201,
      "learning_rate": 1.7988062795431302e-07,
      "logits/chosen": -1.1320312023162842,
      "logits/rejected": -0.8515625,
      "logps/chosen": -521.2000122070312,
      "logps/rejected": -1404.800048828125,
      "loss": 0.1711,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.21145018935203552,
      "rewards/margins": 5.217187404632568,
      "rewards/rejected": -5.428124904632568,
      "step": 1610
    },
    {
      "epoch": 1.2676609105180534,
      "grad_norm": 160.46467195348245,
      "learning_rate": 1.7823842922593495e-07,
      "logits/chosen": -0.94921875,
      "logits/rejected": -0.749804675579071,
      "logps/chosen": -576.4000244140625,
      "logps/rejected": -1481.5999755859375,
      "loss": 0.1005,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.10455322265625,
      "rewards/margins": 5.8671875,
      "rewards/rejected": -5.978125095367432,
      "step": 1615
    },
    {
      "epoch": 1.2715855572998431,
      "grad_norm": 1421.4771333424662,
      "learning_rate": 1.7659959810827075e-07,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.684521496295929,
      "logps/chosen": -564.5999755859375,
      "logps/rejected": -1478.800048828125,
      "loss": 0.6199,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.15185546875,
      "rewards/margins": 5.038232326507568,
      "rewards/rejected": -5.1962890625,
      "step": 1620
    },
    {
      "epoch": 1.2755102040816326,
      "grad_norm": 209.67351869699914,
      "learning_rate": 1.7496421150801542e-07,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.536914050579071,
      "logps/chosen": -568.7999877929688,
      "logps/rejected": -1657.5999755859375,
      "loss": 0.1602,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.32890623807907104,
      "rewards/margins": 6.3125,
      "rewards/rejected": -6.628125190734863,
      "step": 1625
    },
    {
      "epoch": 1.2794348508634223,
      "grad_norm": 1138.7234189289838,
      "learning_rate": 1.7333234617022037e-07,
      "logits/chosen": -0.90234375,
      "logits/rejected": -0.67333984375,
      "logps/chosen": -636.5999755859375,
      "logps/rejected": -1703.199951171875,
      "loss": 0.4421,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.74560546875,
      "rewards/margins": 6.719531059265137,
      "rewards/rejected": -7.471875190734863,
      "step": 1630
    },
    {
      "epoch": 1.283359497645212,
      "grad_norm": 39.53741246241464,
      "learning_rate": 1.7170407867469184e-07,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.9400390386581421,
      "logps/chosen": -536.2000122070312,
      "logps/rejected": -2034.4000244140625,
      "loss": 0.0882,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.08514404296875,
      "rewards/margins": 6.678124904632568,
      "rewards/rejected": -6.762499809265137,
      "step": 1635
    },
    {
      "epoch": 1.2872841444270016,
      "grad_norm": 56.584951841907625,
      "learning_rate": 1.7007948543239762e-07,
      "logits/chosen": -1.035546898841858,
      "logits/rejected": -0.758593738079071,
      "logps/chosen": -553.2000122070312,
      "logps/rejected": -1781.5999755859375,
      "loss": 0.0971,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.503125011920929,
      "rewards/margins": 6.621874809265137,
      "rewards/rejected": -7.118750095367432,
      "step": 1640
    },
    {
      "epoch": 1.2912087912087913,
      "grad_norm": 125.23830443077074,
      "learning_rate": 1.684586426818807e-07,
      "logits/chosen": -1.092187523841858,
      "logits/rejected": -0.9664062261581421,
      "logps/chosen": -617.7999877929688,
      "logps/rejected": -1930.4000244140625,
      "loss": 0.0963,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.28814697265625,
      "rewards/margins": 6.778124809265137,
      "rewards/rejected": -7.068749904632568,
      "step": 1645
    },
    {
      "epoch": 1.2951334379905808,
      "grad_norm": 74.36323746565634,
      "learning_rate": 1.6684162648568218e-07,
      "logits/chosen": -1.0984375476837158,
      "logits/rejected": -0.888867199420929,
      "logps/chosen": -578.2000122070312,
      "logps/rejected": -1592.0,
      "loss": 0.0896,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.26933592557907104,
      "rewards/margins": 7.324999809265137,
      "rewards/rejected": -7.606249809265137,
      "step": 1650
    },
    {
      "epoch": 1.2990580847723705,
      "grad_norm": 126.84790976414477,
      "learning_rate": 1.6522851272677124e-07,
      "logits/chosen": -1.073828101158142,
      "logits/rejected": -0.80078125,
      "logps/chosen": -593.2000122070312,
      "logps/rejected": -1559.5999755859375,
      "loss": 0.0965,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.45292967557907104,
      "rewards/margins": 7.371874809265137,
      "rewards/rejected": -7.828125,
      "step": 1655
    },
    {
      "epoch": 1.30298273155416,
      "grad_norm": 53.53475248498685,
      "learning_rate": 1.636193771049842e-07,
      "logits/chosen": -1.0499999523162842,
      "logits/rejected": -0.7251952886581421,
      "logps/chosen": -543.0,
      "logps/rejected": -1558.4000244140625,
      "loss": 0.1635,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3136962950229645,
      "rewards/margins": 6.251562595367432,
      "rewards/rejected": -6.567187309265137,
      "step": 1660
    },
    {
      "epoch": 1.3069073783359497,
      "grad_norm": 92.28547841615084,
      "learning_rate": 1.6201429513347272e-07,
      "logits/chosen": -0.92529296875,
      "logits/rejected": -0.659863293170929,
      "logps/chosen": -549.0,
      "logps/rejected": -1421.199951171875,
      "loss": 0.092,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.20087890326976776,
      "rewards/margins": 6.509375095367432,
      "rewards/rejected": -6.712500095367432,
      "step": 1665
    },
    {
      "epoch": 1.3108320251177394,
      "grad_norm": 124.83388991387946,
      "learning_rate": 1.6041334213515945e-07,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.584765613079071,
      "logps/chosen": -653.5999755859375,
      "logps/rejected": -1281.199951171875,
      "loss": 0.6706,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.936572253704071,
      "rewards/margins": 5.121874809265137,
      "rewards/rejected": -6.056250095367432,
      "step": 1670
    },
    {
      "epoch": 1.314756671899529,
      "grad_norm": 43.53630661821646,
      "learning_rate": 1.5881659323920326e-07,
      "logits/chosen": -1.2156250476837158,
      "logits/rejected": -0.9232422113418579,
      "logps/chosen": -556.4000244140625,
      "logps/rejected": -1678.4000244140625,
      "loss": 0.1445,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.34771728515625,
      "rewards/margins": 6.953125,
      "rewards/rejected": -7.303124904632568,
      "step": 1675
    },
    {
      "epoch": 1.3186813186813187,
      "grad_norm": 142.60223246330244,
      "learning_rate": 1.5722412337747436e-07,
      "logits/chosen": -1.3093750476837158,
      "logits/rejected": -1.002343773841858,
      "logps/chosen": -598.7999877929688,
      "logps/rejected": -1908.0,
      "loss": 0.0997,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.006823730655014515,
      "rewards/margins": 7.515625,
      "rewards/rejected": -7.5,
      "step": 1680
    },
    {
      "epoch": 1.3226059654631084,
      "grad_norm": 97.50104697278564,
      "learning_rate": 1.556360072810371e-07,
      "logits/chosen": -1.211328148841858,
      "logits/rejected": -0.874218761920929,
      "logps/chosen": -569.7999877929688,
      "logps/rejected": -1588.0,
      "loss": 0.0626,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.13740234076976776,
      "rewards/margins": 6.871874809265137,
      "rewards/rejected": -7.009375095367432,
      "step": 1685
    },
    {
      "epoch": 1.3265306122448979,
      "grad_norm": 46.83569238044282,
      "learning_rate": 1.540523194766434e-07,
      "logits/chosen": -1.169921875,
      "logits/rejected": -0.979296863079071,
      "logps/chosen": -484.20001220703125,
      "logps/rejected": -1260.0,
      "loss": 0.1006,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.16660156846046448,
      "rewards/margins": 5.614062309265137,
      "rewards/rejected": -5.449999809265137,
      "step": 1690
    },
    {
      "epoch": 1.3304552590266876,
      "grad_norm": 151.94898921859826,
      "learning_rate": 1.524731342832352e-07,
      "logits/chosen": -1.1671874523162842,
      "logits/rejected": -0.958984375,
      "logps/chosen": -498.3999938964844,
      "logps/rejected": -1582.800048828125,
      "loss": 0.134,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.17949219048023224,
      "rewards/margins": 6.415625095367432,
      "rewards/rejected": -6.595312595367432,
      "step": 1695
    },
    {
      "epoch": 1.3343799058084773,
      "grad_norm": 89.89629249445832,
      "learning_rate": 1.5089852580845703e-07,
      "logits/chosen": -1.130468726158142,
      "logits/rejected": -0.8792968988418579,
      "logps/chosen": -584.2000122070312,
      "logps/rejected": -1656.800048828125,
      "loss": 0.089,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.418701171875,
      "rewards/margins": 6.237500190734863,
      "rewards/rejected": -6.653124809265137,
      "step": 1700
    },
    {
      "epoch": 1.3383045525902668,
      "grad_norm": 60.938540378583355,
      "learning_rate": 1.4932856794517825e-07,
      "logits/chosen": -1.1906249523162842,
      "logits/rejected": -0.9701172113418579,
      "logps/chosen": -574.5999755859375,
      "logps/rejected": -1953.5999755859375,
      "loss": 0.0982,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3189453184604645,
      "rewards/margins": 7.696875095367432,
      "rewards/rejected": -8.006250381469727,
      "step": 1705
    },
    {
      "epoch": 1.3422291993720565,
      "grad_norm": 39.58704981842939,
      "learning_rate": 1.4776333436802522e-07,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.8041015863418579,
      "logps/chosen": -561.0,
      "logps/rejected": -1732.800048828125,
      "loss": 0.1495,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.626757800579071,
      "rewards/margins": 7.349999904632568,
      "rewards/rejected": -7.984375,
      "step": 1710
    },
    {
      "epoch": 1.3461538461538463,
      "grad_norm": 179.57029655745256,
      "learning_rate": 1.4620289852992436e-07,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.6786133050918579,
      "logps/chosen": -559.0,
      "logps/rejected": -1837.5999755859375,
      "loss": 0.1152,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5279785394668579,
      "rewards/margins": 7.181250095367432,
      "rewards/rejected": -7.709374904632568,
      "step": 1715
    },
    {
      "epoch": 1.3500784929356358,
      "grad_norm": 197.1830823289898,
      "learning_rate": 1.4464733365865454e-07,
      "logits/chosen": -1.0812499523162842,
      "logits/rejected": -0.822265625,
      "logps/chosen": -591.5999755859375,
      "logps/rejected": -1852.800048828125,
      "loss": 0.1027,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5708984136581421,
      "rewards/margins": 8.071874618530273,
      "rewards/rejected": -8.634374618530273,
      "step": 1720
    },
    {
      "epoch": 1.3540031397174255,
      "grad_norm": 93.98939898391808,
      "learning_rate": 1.4309671275341113e-07,
      "logits/chosen": -0.9632812738418579,
      "logits/rejected": -0.8202270269393921,
      "logps/chosen": -587.0,
      "logps/rejected": -1990.4000244140625,
      "loss": 0.0928,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.647412121295929,
      "rewards/margins": 7.743750095367432,
      "rewards/rejected": -8.375,
      "step": 1725
    },
    {
      "epoch": 1.3579277864992152,
      "grad_norm": 149.85846067691418,
      "learning_rate": 1.4155110858138024e-07,
      "logits/chosen": -1.0964844226837158,
      "logits/rejected": -1.0085937976837158,
      "logps/chosen": -646.4000244140625,
      "logps/rejected": -1685.5999755859375,
      "loss": 0.1195,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.11057128757238388,
      "rewards/margins": 7.003125190734863,
      "rewards/rejected": -7.118750095367432,
      "step": 1730
    },
    {
      "epoch": 1.3618524332810047,
      "grad_norm": 258.3087221569205,
      "learning_rate": 1.4001059367432386e-07,
      "logits/chosen": -1.241796851158142,
      "logits/rejected": -0.9769531488418579,
      "logps/chosen": -644.0,
      "logps/rejected": -1561.5999755859375,
      "loss": 0.2873,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.3869995176792145,
      "rewards/margins": 6.821875095367432,
      "rewards/rejected": -7.209374904632568,
      "step": 1735
    },
    {
      "epoch": 1.3657770800627944,
      "grad_norm": 444.22977191831643,
      "learning_rate": 1.3847524032517616e-07,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -1.010156273841858,
      "logps/chosen": -750.5999755859375,
      "logps/rejected": -1735.5999755859375,
      "loss": 0.3662,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.525738537311554,
      "rewards/margins": 5.904687404632568,
      "rewards/rejected": -6.432812690734863,
      "step": 1740
    },
    {
      "epoch": 1.369701726844584,
      "grad_norm": 27.04754855068082,
      "learning_rate": 1.3694512058465062e-07,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.746874988079071,
      "logps/chosen": -518.5999755859375,
      "logps/rejected": -1291.5999755859375,
      "loss": 0.0639,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.02384033240377903,
      "rewards/margins": 5.821875095367432,
      "rewards/rejected": -5.846875190734863,
      "step": 1745
    },
    {
      "epoch": 1.3736263736263736,
      "grad_norm": 3816.700121309222,
      "learning_rate": 1.3542030625785938e-07,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.7367187738418579,
      "logps/chosen": -570.4000244140625,
      "logps/rejected": -1552.0,
      "loss": 0.1194,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4248046875,
      "rewards/margins": 6.534375190734863,
      "rewards/rejected": -6.959374904632568,
      "step": 1750
    },
    {
      "epoch": 1.3775510204081631,
      "grad_norm": 335.16119720469965,
      "learning_rate": 1.3390086890094342e-07,
      "logits/chosen": -0.878125011920929,
      "logits/rejected": -0.48637694120407104,
      "logps/chosen": -658.4000244140625,
      "logps/rejected": -1505.5999755859375,
      "loss": 0.2213,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.085351586341858,
      "rewards/margins": 6.721875190734863,
      "rewards/rejected": -7.8125,
      "step": 1755
    },
    {
      "epoch": 1.3814756671899528,
      "grad_norm": 249.5343930594709,
      "learning_rate": 1.3238687981771398e-07,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.767578125,
      "logps/chosen": -635.4000244140625,
      "logps/rejected": -1765.5999755859375,
      "loss": 0.0776,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.8587402105331421,
      "rewards/margins": 6.449999809265137,
      "rewards/rejected": -7.309374809265137,
      "step": 1760
    },
    {
      "epoch": 1.3854003139717426,
      "grad_norm": 284.10914292705473,
      "learning_rate": 1.3087841005630773e-07,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -0.7724609375,
      "logps/chosen": -596.5999755859375,
      "logps/rejected": -1496.0,
      "loss": 0.1241,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.7421875,
      "rewards/margins": 5.715624809265137,
      "rewards/rejected": -6.459374904632568,
      "step": 1765
    },
    {
      "epoch": 1.389324960753532,
      "grad_norm": 273.854517730081,
      "learning_rate": 1.2937553040585123e-07,
      "logits/chosen": -0.999218761920929,
      "logits/rejected": -0.8340820074081421,
      "logps/chosen": -587.2000122070312,
      "logps/rejected": -1709.5999755859375,
      "loss": 0.2813,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.591931164264679,
      "rewards/margins": 5.706250190734863,
      "rewards/rejected": -6.303124904632568,
      "step": 1770
    },
    {
      "epoch": 1.3932496075353218,
      "grad_norm": 136.69590754184225,
      "learning_rate": 1.2787831139313956e-07,
      "logits/chosen": -0.9634765386581421,
      "logits/rejected": -0.588916003704071,
      "logps/chosen": -616.0,
      "logps/rejected": -1428.0,
      "loss": 0.1183,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.392578125,
      "rewards/margins": 6.012499809265137,
      "rewards/rejected": -6.40625,
      "step": 1775
    },
    {
      "epoch": 1.3971742543171115,
      "grad_norm": 237.37685365484376,
      "learning_rate": 1.2638682327932732e-07,
      "logits/chosen": -1.2000000476837158,
      "logits/rejected": -0.966796875,
      "logps/chosen": -568.7999877929688,
      "logps/rejected": -1525.5999755859375,
      "loss": 0.1015,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.04248046875,
      "rewards/margins": 6.025000095367432,
      "rewards/rejected": -6.071875095367432,
      "step": 1780
    },
    {
      "epoch": 1.401098901098901,
      "grad_norm": 190.85916974384313,
      "learning_rate": 1.2490113605663009e-07,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.814453125,
      "logps/chosen": -566.5999755859375,
      "logps/rejected": -1733.5999755859375,
      "loss": 0.1132,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.25322264432907104,
      "rewards/margins": 7.890625,
      "rewards/rejected": -8.140625,
      "step": 1785
    },
    {
      "epoch": 1.4050235478806907,
      "grad_norm": 40.620050138877914,
      "learning_rate": 1.234213194450409e-07,
      "logits/chosen": -1.087499976158142,
      "logits/rejected": -0.779296875,
      "logps/chosen": -570.7999877929688,
      "logps/rejected": -1627.199951171875,
      "loss": 0.1148,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.39643555879592896,
      "rewards/margins": 6.493750095367432,
      "rewards/rejected": -6.890625,
      "step": 1790
    },
    {
      "epoch": 1.4089481946624804,
      "grad_norm": 57.30036914543044,
      "learning_rate": 1.2194744288905805e-07,
      "logits/chosen": -1.174218773841858,
      "logits/rejected": -0.8304687738418579,
      "logps/chosen": -644.4000244140625,
      "logps/rejected": -1699.199951171875,
      "loss": 0.0746,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.6650390625,
      "rewards/margins": 7.887499809265137,
      "rewards/rejected": -8.540624618530273,
      "step": 1795
    },
    {
      "epoch": 1.41287284144427,
      "grad_norm": 77.48083662312791,
      "learning_rate": 1.2047957555442605e-07,
      "logits/chosen": -0.945019543170929,
      "logits/rejected": -0.6998046636581421,
      "logps/chosen": -625.2000122070312,
      "logps/rejected": -2116.800048828125,
      "loss": 0.0798,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.704418957233429,
      "rewards/margins": 8.021875381469727,
      "rewards/rejected": -8.725000381469727,
      "step": 1800
    },
    {
      "epoch": 1.4167974882260597,
      "grad_norm": 32.11611464664754,
      "learning_rate": 1.1901778632489034e-07,
      "logits/chosen": -0.9857422113418579,
      "logits/rejected": -0.626953125,
      "logps/chosen": -619.5999755859375,
      "logps/rejected": -1468.800048828125,
      "loss": 0.1164,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.503710925579071,
      "rewards/margins": 6.909375190734863,
      "rewards/rejected": -7.421875,
      "step": 1805
    },
    {
      "epoch": 1.4207221350078494,
      "grad_norm": 79.99303174913608,
      "learning_rate": 1.1756214379896428e-07,
      "logits/chosen": -0.949023425579071,
      "logits/rejected": -0.7669922113418579,
      "logps/chosen": -615.7999877929688,
      "logps/rejected": -1516.800048828125,
      "loss": 0.1624,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6151367425918579,
      "rewards/margins": 7.162499904632568,
      "rewards/rejected": -7.78125,
      "step": 1810
    },
    {
      "epoch": 1.4246467817896389,
      "grad_norm": 75.82751914207549,
      "learning_rate": 1.1611271628670999e-07,
      "logits/chosen": -0.9449218511581421,
      "logits/rejected": -0.8999999761581421,
      "logps/chosen": -615.5999755859375,
      "logps/rejected": -1944.0,
      "loss": 0.1074,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.4077697694301605,
      "rewards/margins": 8.034375190734863,
      "rewards/rejected": -8.440625190734863,
      "step": 1815
    },
    {
      "epoch": 1.4285714285714286,
      "grad_norm": 642.9534677496823,
      "learning_rate": 1.1466957180653297e-07,
      "logits/chosen": -1.15234375,
      "logits/rejected": -0.9609375,
      "logps/chosen": -525.4000244140625,
      "logps/rejected": -1570.4000244140625,
      "loss": 0.1565,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3372802734375,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.518750190734863,
      "step": 1820
    },
    {
      "epoch": 1.4324960753532183,
      "grad_norm": 125.22563927240078,
      "learning_rate": 1.1323277808199008e-07,
      "logits/chosen": -1.0578124523162842,
      "logits/rejected": -0.79345703125,
      "logps/chosen": -528.0,
      "logps/rejected": -1612.800048828125,
      "loss": 0.0957,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3062744140625,
      "rewards/margins": 6.606249809265137,
      "rewards/rejected": -6.912499904632568,
      "step": 1825
    },
    {
      "epoch": 1.4364207221350078,
      "grad_norm": 63.01786413080434,
      "learning_rate": 1.1180240253861134e-07,
      "logits/chosen": -1.087890625,
      "logits/rejected": -0.7371581792831421,
      "logps/chosen": -510.0,
      "logps/rejected": -1455.199951171875,
      "loss": 0.1045,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08364257961511612,
      "rewards/margins": 6.918749809265137,
      "rewards/rejected": -6.996874809265137,
      "step": 1830
    },
    {
      "epoch": 1.4403453689167975,
      "grad_norm": 91.34007916580279,
      "learning_rate": 1.1037851230073561e-07,
      "logits/chosen": -0.942187488079071,
      "logits/rejected": -0.859375,
      "logps/chosen": -628.7999877929688,
      "logps/rejected": -1632.0,
      "loss": 0.084,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.40678709745407104,
      "rewards/margins": 6.943749904632568,
      "rewards/rejected": -7.349999904632568,
      "step": 1835
    },
    {
      "epoch": 1.4442700156985873,
      "grad_norm": 66.80276923261975,
      "learning_rate": 1.0896117418836098e-07,
      "logits/chosen": -1.159765601158142,
      "logits/rejected": -0.8296874761581421,
      "logps/chosen": -545.4000244140625,
      "logps/rejected": -1471.199951171875,
      "loss": 0.0834,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.13554687798023224,
      "rewards/margins": 7.253125190734863,
      "rewards/rejected": -7.387499809265137,
      "step": 1840
    },
    {
      "epoch": 1.4481946624803768,
      "grad_norm": 171.76276501339746,
      "learning_rate": 1.0755045471400895e-07,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.7231200933456421,
      "logps/chosen": -561.5999755859375,
      "logps/rejected": -1707.199951171875,
      "loss": 0.1348,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.38350218534469604,
      "rewards/margins": 7.128125190734863,
      "rewards/rejected": -7.509375095367432,
      "step": 1845
    },
    {
      "epoch": 1.4521193092621665,
      "grad_norm": 87.21198027516775,
      "learning_rate": 1.0614642007960292e-07,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.8382812738418579,
      "logps/chosen": -584.4000244140625,
      "logps/rejected": -1774.0,
      "loss": 0.1249,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4532226622104645,
      "rewards/margins": 6.803124904632568,
      "rewards/rejected": -7.262499809265137,
      "step": 1850
    },
    {
      "epoch": 1.456043956043956,
      "grad_norm": 21.788847775540646,
      "learning_rate": 1.0474913617336185e-07,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.9453125,
      "logps/chosen": -578.2000122070312,
      "logps/rejected": -1832.0,
      "loss": 0.074,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3325561583042145,
      "rewards/margins": 7.465624809265137,
      "rewards/rejected": -7.806250095367432,
      "step": 1855
    },
    {
      "epoch": 1.4599686028257457,
      "grad_norm": 61.299634844420275,
      "learning_rate": 1.0335866856670766e-07,
      "logits/chosen": -0.938281238079071,
      "logits/rejected": -0.672119140625,
      "logps/chosen": -638.4000244140625,
      "logps/rejected": -1721.5999755859375,
      "loss": 0.0973,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.637988269329071,
      "rewards/margins": 7.84375,
      "rewards/rejected": -8.478124618530273,
      "step": 1860
    },
    {
      "epoch": 1.4638932496075352,
      "grad_norm": 71.26830250715827,
      "learning_rate": 1.0197508251118891e-07,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.708789050579071,
      "logps/chosen": -586.0,
      "logps/rejected": -1566.4000244140625,
      "loss": 0.1439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5977538824081421,
      "rewards/margins": 6.581250190734863,
      "rewards/rejected": -7.181250095367432,
      "step": 1865
    },
    {
      "epoch": 1.467817896389325,
      "grad_norm": 225.4623795066631,
      "learning_rate": 1.0059844293541822e-07,
      "logits/chosen": -0.869921863079071,
      "logits/rejected": -0.6263672113418579,
      "logps/chosen": -584.7999877929688,
      "logps/rejected": -1443.5999755859375,
      "loss": 0.1542,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.761425793170929,
      "rewards/margins": 6.645312309265137,
      "rewards/rejected": -7.409375190734863,
      "step": 1870
    },
    {
      "epoch": 1.4717425431711146,
      "grad_norm": 144.7334204905417,
      "learning_rate": 9.922881444202513e-08,
      "logits/chosen": -0.91796875,
      "logits/rejected": -0.60693359375,
      "logps/chosen": -560.0,
      "logps/rejected": -1634.4000244140625,
      "loss": 0.1085,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.8248291015625,
      "rewards/margins": 6.418749809265137,
      "rewards/rejected": -7.25,
      "step": 1875
    },
    {
      "epoch": 1.4756671899529041,
      "grad_norm": 79.01407486754518,
      "learning_rate": 9.786626130462494e-08,
      "logits/chosen": -0.8876953125,
      "logits/rejected": -0.857226550579071,
      "logps/chosen": -631.7999877929688,
      "logps/rejected": -2000.800048828125,
      "loss": 0.0942,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.811389148235321,
      "rewards/margins": 7.099999904632568,
      "rewards/rejected": -7.918749809265137,
      "step": 1880
    },
    {
      "epoch": 1.4795918367346939,
      "grad_norm": 416.9481388845517,
      "learning_rate": 9.651084746480229e-08,
      "logits/chosen": -0.9359375238418579,
      "logits/rejected": -0.649609386920929,
      "logps/chosen": -625.7999877929688,
      "logps/rejected": -2037.5999755859375,
      "loss": 0.1088,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.637622058391571,
      "rewards/margins": 7.653124809265137,
      "rewards/rejected": -8.287500381469727,
      "step": 1885
    },
    {
      "epoch": 1.4835164835164836,
      "grad_norm": 43.978417997401515,
      "learning_rate": 9.516263652911019e-08,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.6680663824081421,
      "logps/chosen": -568.7999877929688,
      "logps/rejected": -1836.800048828125,
      "loss": 0.0744,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.297607421875,
      "rewards/margins": 8.090624809265137,
      "rewards/rejected": -8.381250381469727,
      "step": 1890
    },
    {
      "epoch": 1.487441130298273,
      "grad_norm": 130.48765347083614,
      "learning_rate": 9.382169176608567e-08,
      "logits/chosen": -1.0089843273162842,
      "logits/rejected": -0.749804675579071,
      "logps/chosen": -564.0,
      "logps/rejected": -1682.4000244140625,
      "loss": 0.0823,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.22475585341453552,
      "rewards/margins": 7.412499904632568,
      "rewards/rejected": -7.637499809265137,
      "step": 1895
    },
    {
      "epoch": 1.4913657770800628,
      "grad_norm": 53.55466981004728,
      "learning_rate": 9.24880761032804e-08,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.5263671875,
      "logps/chosen": -589.4000244140625,
      "logps/rejected": -1651.199951171875,
      "loss": 0.0668,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3369140625,
      "rewards/margins": 7.362500190734863,
      "rewards/rejected": -7.696875095367432,
      "step": 1900
    },
    {
      "epoch": 1.4952904238618525,
      "grad_norm": 70.07263332403784,
      "learning_rate": 9.116185212430763e-08,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.845898449420929,
      "logps/chosen": -605.2000122070312,
      "logps/rejected": -1838.0,
      "loss": 0.2346,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.8384765386581421,
      "rewards/margins": 7.296875,
      "rewards/rejected": -8.137499809265137,
      "step": 1905
    },
    {
      "epoch": 1.499215070643642,
      "grad_norm": 141.1815585785424,
      "learning_rate": 8.984308206590551e-08,
      "logits/chosen": -1.137109398841858,
      "logits/rejected": -0.899609386920929,
      "logps/chosen": -586.2000122070312,
      "logps/rejected": -1729.199951171875,
      "loss": 0.1149,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5799804925918579,
      "rewards/margins": 6.715624809265137,
      "rewards/rejected": -7.296875,
      "step": 1910
    },
    {
      "epoch": 1.5031397174254317,
      "grad_norm": 63.821147777388276,
      "learning_rate": 8.853182781501597e-08,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -541.2000122070312,
      "logps/rejected": -1332.800048828125,
      "loss": 0.0895,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.23560181260108948,
      "rewards/margins": 6.078125,
      "rewards/rejected": -6.318749904632568,
      "step": 1915
    },
    {
      "epoch": 1.5070643642072215,
      "grad_norm": 143.47295527115722,
      "learning_rate": 8.722815090588112e-08,
      "logits/chosen": -1.029296875,
      "logits/rejected": -0.657275378704071,
      "logps/chosen": -582.4000244140625,
      "logps/rejected": -1807.199951171875,
      "loss": 0.1036,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": 0.05687255784869194,
      "rewards/margins": 7.400000095367432,
      "rewards/rejected": -7.340624809265137,
      "step": 1920
    },
    {
      "epoch": 1.510989010989011,
      "grad_norm": 173.80427569754428,
      "learning_rate": 8.593211251715532e-08,
      "logits/chosen": -1.008154273033142,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -600.0,
      "logps/rejected": -1554.4000244140625,
      "loss": 0.1858,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.4128662049770355,
      "rewards/margins": 6.1953125,
      "rewards/rejected": -6.609375,
      "step": 1925
    },
    {
      "epoch": 1.5149136577708007,
      "grad_norm": 83.39172301778038,
      "learning_rate": 8.464377346903386e-08,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.9087890386581421,
      "logps/chosen": -626.0,
      "logps/rejected": -1874.4000244140625,
      "loss": 0.2357,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.559252917766571,
      "rewards/margins": 7.704687595367432,
      "rewards/rejected": -8.271875381469727,
      "step": 1930
    },
    {
      "epoch": 1.5188383045525904,
      "grad_norm": 227.16658277663473,
      "learning_rate": 8.336319422039945e-08,
      "logits/chosen": -1.069921851158142,
      "logits/rejected": -0.7955077886581421,
      "logps/chosen": -656.7999877929688,
      "logps/rejected": -1556.800048828125,
      "loss": 0.1458,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6567627191543579,
      "rewards/margins": 6.381249904632568,
      "rewards/rejected": -7.034375190734863,
      "step": 1935
    },
    {
      "epoch": 1.5227629513343799,
      "grad_norm": 222.2934977077575,
      "learning_rate": 8.209043486598452e-08,
      "logits/chosen": -0.8871093988418579,
      "logits/rejected": -0.5166015625,
      "logps/chosen": -622.2000122070312,
      "logps/rejected": -1797.5999755859375,
      "loss": 0.1503,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.548046886920929,
      "rewards/margins": 7.625,
      "rewards/rejected": -8.162500381469727,
      "step": 1940
    },
    {
      "epoch": 1.5266875981161694,
      "grad_norm": 107.44357540630351,
      "learning_rate": 8.082555513355146e-08,
      "logits/chosen": -0.9624999761581421,
      "logits/rejected": -0.753125011920929,
      "logps/chosen": -622.4000244140625,
      "logps/rejected": -1608.800048828125,
      "loss": 0.1831,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.612988293170929,
      "rewards/margins": 6.059374809265137,
      "rewards/rejected": -6.671875,
      "step": 1945
    },
    {
      "epoch": 1.5306122448979593,
      "grad_norm": 30.773714054687158,
      "learning_rate": 7.956861438108922e-08,
      "logits/chosen": -0.9273437261581421,
      "logits/rejected": -0.8294922113418579,
      "logps/chosen": -581.0,
      "logps/rejected": -1802.800048828125,
      "loss": 0.1063,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.5713866949081421,
      "rewards/margins": 8.140625,
      "rewards/rejected": -8.709375381469727,
      "step": 1950
    },
    {
      "epoch": 1.5345368916797488,
      "grad_norm": 149.96027706541733,
      "learning_rate": 7.831967159402836e-08,
      "logits/chosen": -0.945117175579071,
      "logits/rejected": -0.879101574420929,
      "logps/chosen": -549.5999755859375,
      "logps/rejected": -2305.60009765625,
      "loss": 0.1017,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.40068358182907104,
      "rewards/margins": 8.290624618530273,
      "rewards/rejected": -8.696874618530273,
      "step": 1955
    },
    {
      "epoch": 1.5384615384615383,
      "grad_norm": 574.0023457171804,
      "learning_rate": 7.707878538247275e-08,
      "logits/chosen": -0.904296875,
      "logits/rejected": -0.576171875,
      "logps/chosen": -566.4000244140625,
      "logps/rejected": -1431.199951171875,
      "loss": 0.1076,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.17336425185203552,
      "rewards/margins": 6.828125,
      "rewards/rejected": -7.006249904632568,
      "step": 1960
    },
    {
      "epoch": 1.542386185243328,
      "grad_norm": 34.935796551191814,
      "learning_rate": 7.584601397844881e-08,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.5306640863418579,
      "logps/chosen": -560.5999755859375,
      "logps/rejected": -1421.5999755859375,
      "loss": 0.0755,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.255575567483902,
      "rewards/margins": 6.775000095367432,
      "rewards/rejected": -7.03125,
      "step": 1965
    },
    {
      "epoch": 1.5463108320251178,
      "grad_norm": 154.82540127977018,
      "learning_rate": 7.462141523317361e-08,
      "logits/chosen": -0.9574218988418579,
      "logits/rejected": -0.6907714605331421,
      "logps/chosen": -562.2000122070312,
      "logps/rejected": -1520.800048828125,
      "loss": 0.1233,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.2597717344760895,
      "rewards/margins": 6.8125,
      "rewards/rejected": -7.074999809265137,
      "step": 1970
    },
    {
      "epoch": 1.5502354788069073,
      "grad_norm": 84.81926399505227,
      "learning_rate": 7.340504661433916e-08,
      "logits/chosen": -1.0441405773162842,
      "logits/rejected": -0.802539050579071,
      "logps/chosen": -527.0,
      "logps/rejected": -1436.800048828125,
      "loss": 0.0894,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.12929686903953552,
      "rewards/margins": 6.839062690734863,
      "rewards/rejected": -6.965624809265137,
      "step": 1975
    },
    {
      "epoch": 1.554160125588697,
      "grad_norm": 26.71245641124269,
      "learning_rate": 7.219696520341586e-08,
      "logits/chosen": -1.0597655773162842,
      "logits/rejected": -1.0078125,
      "logps/chosen": -527.7999877929688,
      "logps/rejected": -1532.0,
      "loss": 0.0775,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.18408203125,
      "rewards/margins": 6.671875,
      "rewards/rejected": -6.856249809265137,
      "step": 1980
    },
    {
      "epoch": 1.5580847723704867,
      "grad_norm": 162.36938242862726,
      "learning_rate": 7.099722769297445e-08,
      "logits/chosen": -1.097265601158142,
      "logits/rejected": -0.8197265863418579,
      "logps/chosen": -578.7999877929688,
      "logps/rejected": -1550.0,
      "loss": 0.1425,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5315917730331421,
      "rewards/margins": 6.543749809265137,
      "rewards/rejected": -7.084374904632568,
      "step": 1985
    },
    {
      "epoch": 1.5620094191522762,
      "grad_norm": 113.94630417852768,
      "learning_rate": 6.980589038402438e-08,
      "logits/chosen": -0.776806652545929,
      "logits/rejected": -0.560742199420929,
      "logps/chosen": -636.4000244140625,
      "logps/rejected": -1552.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.077539086341858,
      "rewards/margins": 6.971875190734863,
      "rewards/rejected": -8.050000190734863,
      "step": 1990
    },
    {
      "epoch": 1.565934065934066,
      "grad_norm": 47.481599481923325,
      "learning_rate": 6.862300918337274e-08,
      "logits/chosen": -0.8779296875,
      "logits/rejected": -0.748364269733429,
      "logps/chosen": -552.5999755859375,
      "logps/rejected": -1636.800048828125,
      "loss": 0.1222,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.44072264432907104,
      "rewards/margins": 6.990624904632568,
      "rewards/rejected": -7.431250095367432,
      "step": 1995
    },
    {
      "epoch": 1.5698587127158556,
      "grad_norm": 209.21359846160317,
      "learning_rate": 6.744863960100025e-08,
      "logits/chosen": -0.8238769769668579,
      "logits/rejected": -0.555859386920929,
      "logps/chosen": -558.7999877929688,
      "logps/rejected": -1741.5999755859375,
      "loss": 0.1264,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.521899402141571,
      "rewards/margins": 7.865624904632568,
      "rewards/rejected": -8.393750190734863,
      "step": 2000
    },
    {
      "epoch": 1.5737833594976451,
      "grad_norm": 66.80031355752362,
      "learning_rate": 6.628283674745608e-08,
      "logits/chosen": -0.893359363079071,
      "logits/rejected": -0.5850585699081421,
      "logps/chosen": -586.5999755859375,
      "logps/rejected": -1752.800048828125,
      "loss": 0.1263,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.36369627714157104,
      "rewards/margins": 8.046875,
      "rewards/rejected": -8.412500381469727,
      "step": 2005
    },
    {
      "epoch": 1.5777080062794349,
      "grad_norm": 49.6873279559207,
      "learning_rate": 6.512565533127218e-08,
      "logits/chosen": -0.984375,
      "logits/rejected": -0.646289050579071,
      "logps/chosen": -602.5999755859375,
      "logps/rejected": -1706.4000244140625,
      "loss": 0.1299,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.53466796875,
      "rewards/margins": 7.537499904632568,
      "rewards/rejected": -8.065625190734863,
      "step": 2010
    },
    {
      "epoch": 1.5816326530612246,
      "grad_norm": 723.7078038891599,
      "learning_rate": 6.397714965639556e-08,
      "logits/chosen": -0.934765636920929,
      "logits/rejected": -0.49677735567092896,
      "logps/chosen": -644.0,
      "logps/rejected": -1552.0,
      "loss": 0.2778,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.922943115234375,
      "rewards/margins": 6.298437595367432,
      "rewards/rejected": -7.209374904632568,
      "step": 2015
    },
    {
      "epoch": 1.585557299843014,
      "grad_norm": 153.63129256337177,
      "learning_rate": 6.283737361963998e-08,
      "logits/chosen": -0.926953136920929,
      "logits/rejected": -0.731249988079071,
      "logps/chosen": -691.2000122070312,
      "logps/rejected": -2369.60009765625,
      "loss": 0.1082,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.2208983898162842,
      "rewards/margins": 8.943750381469727,
      "rewards/rejected": -10.165624618530273,
      "step": 2020
    },
    {
      "epoch": 1.5894819466248038,
      "grad_norm": 117.23538777944779,
      "learning_rate": 6.170638070815668e-08,
      "logits/chosen": -1.0285155773162842,
      "logits/rejected": -0.77490234375,
      "logps/chosen": -652.4000244140625,
      "logps/rejected": -1656.800048828125,
      "loss": 0.1085,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.926464855670929,
      "rewards/margins": 7.125,
      "rewards/rejected": -8.053125381469727,
      "step": 2025
    },
    {
      "epoch": 1.5934065934065935,
      "grad_norm": 58.48792496209738,
      "learning_rate": 6.058422399692442e-08,
      "logits/chosen": -0.8949218988418579,
      "logits/rejected": -0.8984375,
      "logps/chosen": -602.5999755859375,
      "logps/rejected": -1560.0,
      "loss": 0.0707,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.9017578363418579,
      "rewards/margins": 6.553124904632568,
      "rewards/rejected": -7.453125,
      "step": 2030
    },
    {
      "epoch": 1.597331240188383,
      "grad_norm": 51.39919138550725,
      "learning_rate": 5.9470956146258935e-08,
      "logits/chosen": -0.8363281488418579,
      "logits/rejected": -0.6483398675918579,
      "logps/chosen": -661.2000122070312,
      "logps/rejected": -1592.0,
      "loss": 0.2294,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0994141101837158,
      "rewards/margins": 7.275000095367432,
      "rewards/rejected": -8.378125190734863,
      "step": 2035
    },
    {
      "epoch": 1.6012558869701727,
      "grad_norm": 123.16255743838592,
      "learning_rate": 5.8366629399341174e-08,
      "logits/chosen": -0.760546863079071,
      "logits/rejected": -0.5171874761581421,
      "logps/chosen": -613.5999755859375,
      "logps/rejected": -1707.199951171875,
      "loss": 0.1156,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.698046863079071,
      "rewards/margins": 7.224999904632568,
      "rewards/rejected": -7.928124904632568,
      "step": 2040
    },
    {
      "epoch": 1.6051805337519625,
      "grad_norm": 203.50204136987284,
      "learning_rate": 5.7271295579766314e-08,
      "logits/chosen": -1.062402367591858,
      "logits/rejected": -0.853515625,
      "logps/chosen": -614.0,
      "logps/rejected": -1484.4000244140625,
      "loss": 0.1637,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.6107422113418579,
      "rewards/margins": 6.724999904632568,
      "rewards/rejected": -7.331250190734863,
      "step": 2045
    },
    {
      "epoch": 1.609105180533752,
      "grad_norm": 51.16261126213157,
      "learning_rate": 5.618500608911139e-08,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.6998535394668579,
      "logps/chosen": -571.2000122070312,
      "logps/rejected": -1578.4000244140625,
      "loss": 0.1445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.31820374727249146,
      "rewards/margins": 6.768750190734863,
      "rewards/rejected": -7.090624809265137,
      "step": 2050
    },
    {
      "epoch": 1.6130298273155415,
      "grad_norm": 47.549995159725626,
      "learning_rate": 5.510781190452299e-08,
      "logits/chosen": -1.123437523841858,
      "logits/rejected": -0.9271484613418579,
      "logps/chosen": -589.0,
      "logps/rejected": -1753.199951171875,
      "loss": 0.0902,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4491210877895355,
      "rewards/margins": 6.598437309265137,
      "rewards/rejected": -7.0390625,
      "step": 2055
    },
    {
      "epoch": 1.6169544740973314,
      "grad_norm": 141.4206769872315,
      "learning_rate": 5.4039763576325724e-08,
      "logits/chosen": -1.134765625,
      "logits/rejected": -0.881640613079071,
      "logps/chosen": -554.7999877929688,
      "logps/rejected": -1236.800048828125,
      "loss": 0.1243,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.16269531846046448,
      "rewards/margins": 5.482031345367432,
      "rewards/rejected": -5.643750190734863,
      "step": 2060
    },
    {
      "epoch": 1.620879120879121,
      "grad_norm": 49.39706256979363,
      "learning_rate": 5.29809112256491e-08,
      "logits/chosen": -1.1105468273162842,
      "logits/rejected": -1.0109374523162842,
      "logps/chosen": -501.0,
      "logps/rejected": -1668.800048828125,
      "loss": 0.076,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.010180664248764515,
      "rewards/margins": 6.199999809265137,
      "rewards/rejected": -6.203125,
      "step": 2065
    },
    {
      "epoch": 1.6248037676609104,
      "grad_norm": 262.589635706207,
      "learning_rate": 5.193130454207592e-08,
      "logits/chosen": -1.141015648841858,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -1602.4000244140625,
      "loss": 0.1808,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.13920898735523224,
      "rewards/margins": 6.662499904632568,
      "rewards/rejected": -6.807812690734863,
      "step": 2070
    },
    {
      "epoch": 1.6287284144427001,
      "grad_norm": 106.13978065650966,
      "learning_rate": 5.089099278131079e-08,
      "logits/chosen": -1.0652344226837158,
      "logits/rejected": -0.896289050579071,
      "logps/chosen": -590.2000122070312,
      "logps/rejected": -1954.800048828125,
      "loss": 0.132,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 7.556250095367432,
      "rewards/rejected": -8.0625,
      "step": 2075
    },
    {
      "epoch": 1.6326530612244898,
      "grad_norm": 47.1351601672999,
      "learning_rate": 4.986002476286777e-08,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.775390625,
      "logps/chosen": -551.4000244140625,
      "logps/rejected": -1502.4000244140625,
      "loss": 0.142,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.26191407442092896,
      "rewards/margins": 6.184374809265137,
      "rewards/rejected": -6.449999809265137,
      "step": 2080
    },
    {
      "epoch": 1.6365777080062793,
      "grad_norm": 414.386237272122,
      "learning_rate": 4.8838448867780266e-08,
      "logits/chosen": -1.094140648841858,
      "logits/rejected": -0.8046875,
      "logps/chosen": -592.5999755859375,
      "logps/rejected": -1562.4000244140625,
      "loss": 0.1387,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.20673827826976776,
      "rewards/margins": 6.340624809265137,
      "rewards/rejected": -6.543749809265137,
      "step": 2085
    },
    {
      "epoch": 1.640502354788069,
      "grad_norm": 219.22071868371683,
      "learning_rate": 4.7826313036330194e-08,
      "logits/chosen": -1.103906273841858,
      "logits/rejected": -0.895312488079071,
      "logps/chosen": -707.2000122070312,
      "logps/rejected": -1827.199951171875,
      "loss": 0.3286,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.30029296875,
      "rewards/margins": 6.2890625,
      "rewards/rejected": -6.59375,
      "step": 2090
    },
    {
      "epoch": 1.6444270015698588,
      "grad_norm": 91.40334197427316,
      "learning_rate": 4.6823664765798135e-08,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.6957031488418579,
      "logps/chosen": -493.0,
      "logps/rejected": -1284.0,
      "loss": 0.1388,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.16954346001148224,
      "rewards/margins": 5.421875,
      "rewards/rejected": -5.59375,
      "step": 2095
    },
    {
      "epoch": 1.6483516483516483,
      "grad_norm": 307.88957074684777,
      "learning_rate": 4.5830551108234665e-08,
      "logits/chosen": -1.048828125,
      "logits/rejected": -0.8628906011581421,
      "logps/chosen": -573.0,
      "logps/rejected": -1629.5999755859375,
      "loss": 0.1422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.35075682401657104,
      "rewards/margins": 6.590624809265137,
      "rewards/rejected": -6.940625190734863,
      "step": 2100
    },
    {
      "epoch": 1.652276295133438,
      "grad_norm": 47.62430278757741,
      "learning_rate": 4.48470186682523e-08,
      "logits/chosen": -0.8841797113418579,
      "logits/rejected": -0.6097656488418579,
      "logps/chosen": -521.7999877929688,
      "logps/rejected": -1524.800048828125,
      "loss": 0.0658,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.38005369901657104,
      "rewards/margins": 6.4375,
      "rewards/rejected": -6.8125,
      "step": 2105
    },
    {
      "epoch": 1.6562009419152277,
      "grad_norm": 74.23348928861374,
      "learning_rate": 4.387311360083826e-08,
      "logits/chosen": -0.9976562261581421,
      "logits/rejected": -0.803906261920929,
      "logps/chosen": -543.5999755859375,
      "logps/rejected": -1456.0,
      "loss": 0.092,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.2717651426792145,
      "rewards/margins": 6.712500095367432,
      "rewards/rejected": -6.987500190734863,
      "step": 2110
    },
    {
      "epoch": 1.6601255886970172,
      "grad_norm": 45.37046404252626,
      "learning_rate": 4.290888160918854e-08,
      "logits/chosen": -1.0867187976837158,
      "logits/rejected": -0.7220703363418579,
      "logps/chosen": -564.2000122070312,
      "logps/rejected": -1798.4000244140625,
      "loss": 0.0596,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.10408325493335724,
      "rewards/margins": 7.115624904632568,
      "rewards/rejected": -7.221875190734863,
      "step": 2115
    },
    {
      "epoch": 1.664050235478807,
      "grad_norm": 57.06547763915654,
      "learning_rate": 4.195436794256335e-08,
      "logits/chosen": -1.094140648841858,
      "logits/rejected": -0.6987670660018921,
      "logps/chosen": -554.5999755859375,
      "logps/rejected": -1590.4000244140625,
      "loss": 0.12,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.11810912936925888,
      "rewards/margins": 6.912499904632568,
      "rewards/rejected": -7.034375190734863,
      "step": 2120
    },
    {
      "epoch": 1.6679748822605966,
      "grad_norm": 128.71298643767503,
      "learning_rate": 4.100961739416356e-08,
      "logits/chosen": -1.085546851158142,
      "logits/rejected": -0.8193359375,
      "logps/chosen": -550.2000122070312,
      "logps/rejected": -1565.5999755859375,
      "loss": 0.1282,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.25859373807907104,
      "rewards/margins": 6.265625,
      "rewards/rejected": -6.528124809265137,
      "step": 2125
    },
    {
      "epoch": 1.6718995290423861,
      "grad_norm": 65.74408290737922,
      "learning_rate": 4.0074674299028414e-08,
      "logits/chosen": -1.0625,
      "logits/rejected": -0.7395995855331421,
      "logps/chosen": -519.7999877929688,
      "logps/rejected": -1318.4000244140625,
      "loss": 0.097,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.19385986030101776,
      "rewards/margins": 5.96875,
      "rewards/rejected": -6.159375190734863,
      "step": 2130
    },
    {
      "epoch": 1.6758241758241759,
      "grad_norm": 39.18852436246344,
      "learning_rate": 3.9149582531955606e-08,
      "logits/chosen": -1.087890625,
      "logits/rejected": -0.773144543170929,
      "logps/chosen": -605.7999877929688,
      "logps/rejected": -1782.4000244140625,
      "loss": 0.0625,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2792114317417145,
      "rewards/margins": 6.993750095367432,
      "rewards/rejected": -7.271874904632568,
      "step": 2135
    },
    {
      "epoch": 1.6797488226059656,
      "grad_norm": 205.9391888907699,
      "learning_rate": 3.823438550544156e-08,
      "logits/chosen": -0.9581054449081421,
      "logits/rejected": -0.5308593511581421,
      "logps/chosen": -630.4000244140625,
      "logps/rejected": -1593.5999755859375,
      "loss": 0.4803,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.528124988079071,
      "rewards/margins": 6.504687309265137,
      "rewards/rejected": -7.040625095367432,
      "step": 2140
    },
    {
      "epoch": 1.683673469387755,
      "grad_norm": 66.93276095617634,
      "learning_rate": 3.732912616764478e-08,
      "logits/chosen": -0.934374988079071,
      "logits/rejected": -0.783203125,
      "logps/chosen": -533.5999755859375,
      "logps/rejected": -1607.5999755859375,
      "loss": 0.1495,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.677197277545929,
      "rewards/margins": 6.1875,
      "rewards/rejected": -6.868750095367432,
      "step": 2145
    },
    {
      "epoch": 1.6875981161695446,
      "grad_norm": 326.23774993138244,
      "learning_rate": 3.6433847000370135e-08,
      "logits/chosen": -1.138671875,
      "logits/rejected": -0.681933581829071,
      "logps/chosen": -585.7999877929688,
      "logps/rejected": -1718.4000244140625,
      "loss": 0.1402,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4579101502895355,
      "rewards/margins": 6.509375095367432,
      "rewards/rejected": -6.962500095367432,
      "step": 2150
    },
    {
      "epoch": 1.6915227629513345,
      "grad_norm": 203.28163488461175,
      "learning_rate": 3.554859001707522e-08,
      "logits/chosen": -1.0390625,
      "logits/rejected": -0.7459961175918579,
      "logps/chosen": -631.7999877929688,
      "logps/rejected": -1716.4000244140625,
      "loss": 0.1794,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.588671863079071,
      "rewards/margins": 6.537499904632568,
      "rewards/rejected": -7.121874809265137,
      "step": 2155
    },
    {
      "epoch": 1.695447409733124,
      "grad_norm": 67.25284517698998,
      "learning_rate": 3.467339676089898e-08,
      "logits/chosen": -0.9398437738418579,
      "logits/rejected": -0.7040039300918579,
      "logps/chosen": -598.0,
      "logps/rejected": -1915.199951171875,
      "loss": 0.0926,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4092040956020355,
      "rewards/margins": 7.743750095367432,
      "rewards/rejected": -8.15625,
      "step": 2160
    },
    {
      "epoch": 1.6993720565149135,
      "grad_norm": 103.50881782702439,
      "learning_rate": 3.3808308302712e-08,
      "logits/chosen": -1.036718726158142,
      "logits/rejected": -0.8785156011581421,
      "logps/chosen": -644.7999877929688,
      "logps/rejected": -1913.199951171875,
      "loss": 0.101,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6089843511581421,
      "rewards/margins": 6.896874904632568,
      "rewards/rejected": -7.509375095367432,
      "step": 2165
    },
    {
      "epoch": 1.7032967032967035,
      "grad_norm": 39.42742349533509,
      "learning_rate": 3.295336523918915e-08,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.7442382574081421,
      "logps/chosen": -601.5999755859375,
      "logps/rejected": -1659.199951171875,
      "loss": 0.0963,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.25507813692092896,
      "rewards/margins": 7.184374809265137,
      "rewards/rejected": -7.440625190734863,
      "step": 2170
    },
    {
      "epoch": 1.707221350078493,
      "grad_norm": 25.629197047230104,
      "learning_rate": 3.2108607690904595e-08,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -0.683300793170929,
      "logps/chosen": -603.5999755859375,
      "logps/rejected": -1539.199951171875,
      "loss": 0.2932,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.6676025390625,
      "rewards/margins": 6.067187309265137,
      "rewards/rejected": -6.731249809265137,
      "step": 2175
    },
    {
      "epoch": 1.7111459968602825,
      "grad_norm": 58.91123260423505,
      "learning_rate": 3.127407530044898e-08,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.775585949420929,
      "logps/chosen": -590.2000122070312,
      "logps/rejected": -1517.5999755859375,
      "loss": 0.0671,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.4860595762729645,
      "rewards/margins": 6.540625095367432,
      "rewards/rejected": -7.015625,
      "step": 2180
    },
    {
      "epoch": 1.7150706436420722,
      "grad_norm": 189.9173504826597,
      "learning_rate": 3.044980723056883e-08,
      "logits/chosen": -1.0261719226837158,
      "logits/rejected": -0.7215820550918579,
      "logps/chosen": -571.7999877929688,
      "logps/rejected": -1661.5999755859375,
      "loss": 0.1062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.5221191644668579,
      "rewards/margins": 6.712500095367432,
      "rewards/rejected": -7.234375,
      "step": 2185
    },
    {
      "epoch": 1.718995290423862,
      "grad_norm": 110.82247542194176,
      "learning_rate": 2.9635842162329395e-08,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.7798827886581421,
      "logps/chosen": -594.0,
      "logps/rejected": -1816.0,
      "loss": 0.0704,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.508959949016571,
      "rewards/margins": 8.009374618530273,
      "rewards/rejected": -8.518750190734863,
      "step": 2190
    },
    {
      "epoch": 1.7229199372056514,
      "grad_norm": 803.0336934216188,
      "learning_rate": 2.883221829329857e-08,
      "logits/chosen": -0.815234363079071,
      "logits/rejected": -0.648144543170929,
      "logps/chosen": -606.4000244140625,
      "logps/rejected": -1616.800048828125,
      "loss": 0.1291,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.743847668170929,
      "rewards/margins": 6.495312690734863,
      "rewards/rejected": -7.234375,
      "step": 2195
    },
    {
      "epoch": 1.7268445839874411,
      "grad_norm": 98.20718847554392,
      "learning_rate": 2.8038973335755e-08,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.675097644329071,
      "logps/chosen": -571.2000122070312,
      "logps/rejected": -1729.5999755859375,
      "loss": 0.108,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4634765684604645,
      "rewards/margins": 7.737500190734863,
      "rewards/rejected": -8.196874618530273,
      "step": 2200
    },
    {
      "epoch": 1.7307692307692308,
      "grad_norm": 185.91323462572925,
      "learning_rate": 2.72561445149182e-08,
      "logits/chosen": -0.869921863079071,
      "logits/rejected": -0.7730468511581421,
      "logps/chosen": -660.4000244140625,
      "logps/rejected": -1740.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8910156488418579,
      "rewards/margins": 6.746874809265137,
      "rewards/rejected": -7.628125190734863,
      "step": 2205
    },
    {
      "epoch": 1.7346938775510203,
      "grad_norm": 80.8590609898365,
      "learning_rate": 2.648376856720136e-08,
      "logits/chosen": -1.030664086341858,
      "logits/rejected": -0.7105468511581421,
      "logps/chosen": -600.7999877929688,
      "logps/rejected": -1668.0,
      "loss": 0.0856,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3208984434604645,
      "rewards/margins": 7.703125,
      "rewards/rejected": -8.024999618530273,
      "step": 2210
    },
    {
      "epoch": 1.73861852433281,
      "grad_norm": 137.45389642214167,
      "learning_rate": 2.572188173848777e-08,
      "logits/chosen": -1.040429711341858,
      "logits/rejected": -0.885546863079071,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -1626.4000244140625,
      "loss": 0.0973,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3145507872104645,
      "rewards/margins": 5.984375,
      "rewards/rejected": -6.290625095367432,
      "step": 2215
    },
    {
      "epoch": 1.7425431711145998,
      "grad_norm": 29.408513489374826,
      "learning_rate": 2.497051978242973e-08,
      "logits/chosen": -0.968945324420929,
      "logits/rejected": -0.815625011920929,
      "logps/chosen": -552.2000122070312,
      "logps/rejected": -1695.5999755859375,
      "loss": 0.0666,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.6156250238418579,
      "rewards/margins": 7.176562309265137,
      "rewards/rejected": -7.792187690734863,
      "step": 2220
    },
    {
      "epoch": 1.7464678178963893,
      "grad_norm": 110.95465099847296,
      "learning_rate": 2.4229717958770683e-08,
      "logits/chosen": -0.979687511920929,
      "logits/rejected": -0.833251953125,
      "logps/chosen": -754.0,
      "logps/rejected": -1563.199951171875,
      "loss": 0.283,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.578442394733429,
      "rewards/margins": 5.550000190734863,
      "rewards/rejected": -6.129687309265137,
      "step": 2225
    },
    {
      "epoch": 1.750392464678179,
      "grad_norm": 51.23109035571722,
      "learning_rate": 2.3499511031690482e-08,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.719531238079071,
      "logps/chosen": -570.5999755859375,
      "logps/rejected": -1700.0,
      "loss": 0.0526,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2916015684604645,
      "rewards/margins": 7.971875190734863,
      "rewards/rejected": -8.262499809265137,
      "step": 2230
    },
    {
      "epoch": 1.7543171114599687,
      "grad_norm": 50.72038215890505,
      "learning_rate": 2.2779933268174206e-08,
      "logits/chosen": -1.062890648841858,
      "logits/rejected": -0.853710949420929,
      "logps/chosen": -577.5999755859375,
      "logps/rejected": -1673.5999755859375,
      "loss": 0.074,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.39521485567092896,
      "rewards/margins": 5.984375,
      "rewards/rejected": -6.375,
      "step": 2235
    },
    {
      "epoch": 1.7582417582417582,
      "grad_norm": 111.99965172494834,
      "learning_rate": 2.207101843640391e-08,
      "logits/chosen": -1.045312523841858,
      "logits/rejected": -0.8578125238418579,
      "logps/chosen": -605.0,
      "logps/rejected": -1514.4000244140625,
      "loss": 0.0984,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5815063714981079,
      "rewards/margins": 6.5234375,
      "rewards/rejected": -7.096875190734863,
      "step": 2240
    },
    {
      "epoch": 1.762166405023548,
      "grad_norm": 80.24201176793242,
      "learning_rate": 2.1372799804173892e-08,
      "logits/chosen": -0.840624988079071,
      "logits/rejected": -0.600830078125,
      "logps/chosen": -609.2000122070312,
      "logps/rejected": -1492.800048828125,
      "loss": 0.1134,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.822265625,
      "rewards/margins": 6.765625,
      "rewards/rejected": -7.584374904632568,
      "step": 2245
    },
    {
      "epoch": 1.7660910518053377,
      "grad_norm": 177.8540577671086,
      "learning_rate": 2.0685310137329908e-08,
      "logits/chosen": -0.871289074420929,
      "logits/rejected": -0.670117199420929,
      "logps/chosen": -601.4000244140625,
      "logps/rejected": -1599.199951171875,
      "loss": 0.1172,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6729980707168579,
      "rewards/margins": 6.368750095367432,
      "rewards/rejected": -7.040625095367432,
      "step": 2250
    },
    {
      "epoch": 1.7700156985871272,
      "grad_norm": 46.91290297206502,
      "learning_rate": 2.0008581698230996e-08,
      "logits/chosen": -0.954296886920929,
      "logits/rejected": -0.609375,
      "logps/chosen": -582.4000244140625,
      "logps/rejected": -1386.800048828125,
      "loss": 0.1138,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.45747071504592896,
      "rewards/margins": 6.550000190734863,
      "rewards/rejected": -7.003125190734863,
      "step": 2255
    },
    {
      "epoch": 1.7739403453689166,
      "grad_norm": 12.234727750334118,
      "learning_rate": 1.9342646244235733e-08,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.7855468988418579,
      "logps/chosen": -556.7999877929688,
      "logps/rejected": -1700.800048828125,
      "loss": 0.0864,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4144531190395355,
      "rewards/margins": 7.528124809265137,
      "rewards/rejected": -7.943749904632568,
      "step": 2260
    },
    {
      "epoch": 1.7778649921507066,
      "grad_norm": 140.92916360299415,
      "learning_rate": 1.8687535026212204e-08,
      "logits/chosen": -0.9476562738418579,
      "logits/rejected": -0.6029297113418579,
      "logps/chosen": -592.7999877929688,
      "logps/rejected": -1675.199951171875,
      "loss": 0.1443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6392883062362671,
      "rewards/margins": 6.912499904632568,
      "rewards/rejected": -7.559374809265137,
      "step": 2265
    },
    {
      "epoch": 1.781789638932496,
      "grad_norm": 74.85200328838532,
      "learning_rate": 1.8043278787070903e-08,
      "logits/chosen": -0.8392578363418579,
      "logits/rejected": -0.6485870480537415,
      "logps/chosen": -587.2000122070312,
      "logps/rejected": -1282.4000244140625,
      "loss": 0.0956,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.3482299745082855,
      "rewards/margins": 6.115624904632568,
      "rewards/rejected": -6.462500095367432,
      "step": 2270
    },
    {
      "epoch": 1.7857142857142856,
      "grad_norm": 76.5861376291362,
      "learning_rate": 1.7409907760322557e-08,
      "logits/chosen": -0.923046886920929,
      "logits/rejected": -0.702929675579071,
      "logps/chosen": -769.2000122070312,
      "logps/rejected": -1648.0,
      "loss": 0.1295,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6021484136581421,
      "rewards/margins": 7.650000095367432,
      "rewards/rejected": -8.256250381469727,
      "step": 2275
    },
    {
      "epoch": 1.7896389324960753,
      "grad_norm": 162.11969374135484,
      "learning_rate": 1.6787451668658997e-08,
      "logits/chosen": -0.867968738079071,
      "logits/rejected": -0.7447265386581421,
      "logps/chosen": -601.2000122070312,
      "logps/rejected": -1770.4000244140625,
      "loss": 0.0995,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6324218511581421,
      "rewards/margins": 6.940625190734863,
      "rewards/rejected": -7.571875095367432,
      "step": 2280
    },
    {
      "epoch": 1.793563579277865,
      "grad_norm": 112.88104415242795,
      "learning_rate": 1.6175939722558457e-08,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.5547851324081421,
      "logps/chosen": -596.5999755859375,
      "logps/rejected": -1578.0,
      "loss": 0.0971,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6710449457168579,
      "rewards/margins": 6.268750190734863,
      "rewards/rejected": -6.9375,
      "step": 2285
    },
    {
      "epoch": 1.7974882260596545,
      "grad_norm": 115.94723971373939,
      "learning_rate": 1.5575400618914852e-08,
      "logits/chosen": -0.9560546875,
      "logits/rejected": -0.622631847858429,
      "logps/chosen": -619.5999755859375,
      "logps/rejected": -1669.5999755859375,
      "loss": 0.1008,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.582409679889679,
      "rewards/margins": 7.421875,
      "rewards/rejected": -8.006250381469727,
      "step": 2290
    },
    {
      "epoch": 1.8014128728414442,
      "grad_norm": 90.86819079777715,
      "learning_rate": 1.498586253969103e-08,
      "logits/chosen": -0.9664062261581421,
      "logits/rejected": -0.8544921875,
      "logps/chosen": -586.5999755859375,
      "logps/rejected": -1563.199951171875,
      "loss": 0.0805,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5706787109375,
      "rewards/margins": 7.060937404632568,
      "rewards/rejected": -7.634375095367432,
      "step": 2295
    },
    {
      "epoch": 1.805337519623234,
      "grad_norm": 58.3184315487747,
      "learning_rate": 1.4407353150596158e-08,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.853515625,
      "logps/chosen": -575.0,
      "logps/rejected": -2228.0,
      "loss": 0.0803,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4580078125,
      "rewards/margins": 7.724999904632568,
      "rewards/rejected": -8.181249618530273,
      "step": 2300
    },
    {
      "epoch": 1.8092621664050235,
      "grad_norm": 190.84782502864022,
      "learning_rate": 1.3839899599787652e-08,
      "logits/chosen": -1.0177733898162842,
      "logits/rejected": -0.67138671875,
      "logps/chosen": -616.4000244140625,
      "logps/rejected": -1592.800048828125,
      "loss": 0.0879,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.66796875,
      "rewards/margins": 7.209374904632568,
      "rewards/rejected": -7.878125190734863,
      "step": 2305
    },
    {
      "epoch": 1.8131868131868132,
      "grad_norm": 116.80379458428796,
      "learning_rate": 1.3283528516597004e-08,
      "logits/chosen": -0.957812488079071,
      "logits/rejected": -0.8011718988418579,
      "logps/chosen": -554.0,
      "logps/rejected": -1738.800048828125,
      "loss": 0.1426,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3089965879917145,
      "rewards/margins": 6.634375095367432,
      "rewards/rejected": -6.949999809265137,
      "step": 2310
    },
    {
      "epoch": 1.817111459968603,
      "grad_norm": 51.04109288895478,
      "learning_rate": 1.2738266010280274e-08,
      "logits/chosen": -0.953906238079071,
      "logits/rejected": -0.5542968511581421,
      "logps/chosen": -557.7999877929688,
      "logps/rejected": -1542.4000244140625,
      "loss": 0.0881,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.4448486268520355,
      "rewards/margins": 6.584374904632568,
      "rewards/rejected": -7.034375190734863,
      "step": 2315
    },
    {
      "epoch": 1.8210361067503924,
      "grad_norm": 127.27463454483292,
      "learning_rate": 1.2204137668792524e-08,
      "logits/chosen": -1.070703148841858,
      "logits/rejected": -0.982421875,
      "logps/chosen": -527.7999877929688,
      "logps/rejected": -1577.5999755859375,
      "loss": 0.1148,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.3863769471645355,
      "rewards/margins": 6.314062595367432,
      "rewards/rejected": -6.703125,
      "step": 2320
    },
    {
      "epoch": 1.8249607535321821,
      "grad_norm": 98.7333243945462,
      "learning_rate": 1.1681168557587467e-08,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.7378906011581421,
      "logps/chosen": -571.2000122070312,
      "logps/rejected": -1749.5999755859375,
      "loss": 0.0794,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.36445313692092896,
      "rewards/margins": 7.090624809265137,
      "rewards/rejected": -7.453125,
      "step": 2325
    },
    {
      "epoch": 1.8288854003139718,
      "grad_norm": 37.045377976053004,
      "learning_rate": 1.1169383218440965e-08,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.715527355670929,
      "logps/chosen": -612.4000244140625,
      "logps/rejected": -1623.199951171875,
      "loss": 0.0837,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.3622680604457855,
      "rewards/margins": 6.548437595367432,
      "rewards/rejected": -6.90625,
      "step": 2330
    },
    {
      "epoch": 1.8328100470957613,
      "grad_norm": 75.37663169062202,
      "learning_rate": 1.0668805668299169e-08,
      "logits/chosen": -1.1730468273162842,
      "logits/rejected": -0.741992175579071,
      "logps/chosen": -563.4000244140625,
      "logps/rejected": -1584.800048828125,
      "loss": 0.1115,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4791503846645355,
      "rewards/margins": 5.981249809265137,
      "rewards/rejected": -6.449999809265137,
      "step": 2335
    },
    {
      "epoch": 1.836734693877551,
      "grad_norm": 117.22793879414833,
      "learning_rate": 1.0179459398151985e-08,
      "logits/chosen": -0.851367175579071,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -614.7999877929688,
      "logps/rejected": -1606.0,
      "loss": 0.1329,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.3626464903354645,
      "rewards/margins": 7.078125,
      "rewards/rejected": -7.443749904632568,
      "step": 2340
    },
    {
      "epoch": 1.8406593406593408,
      "grad_norm": 39.982367545382075,
      "learning_rate": 9.701367371930058e-09,
      "logits/chosen": -0.9267578125,
      "logits/rejected": -0.579907238483429,
      "logps/chosen": -557.7999877929688,
      "logps/rejected": -1473.5999755859375,
      "loss": 0.0847,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.26665037870407104,
      "rewards/margins": 7.059374809265137,
      "rewards/rejected": -7.324999809265137,
      "step": 2345
    },
    {
      "epoch": 1.8445839874411303,
      "grad_norm": 154.68654471110162,
      "learning_rate": 9.234552025427472e-09,
      "logits/chosen": -0.8890625238418579,
      "logits/rejected": -0.621874988079071,
      "logps/chosen": -567.4000244140625,
      "logps/rejected": -1484.0,
      "loss": 0.1307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.536694347858429,
      "rewards/margins": 6.599999904632568,
      "rewards/rejected": -7.140625,
      "step": 2350
    },
    {
      "epoch": 1.8485086342229198,
      "grad_norm": 112.81243425575937,
      "learning_rate": 8.779035265249062e-09,
      "logits/chosen": -1.032812476158142,
      "logits/rejected": -0.7806640863418579,
      "logps/chosen": -556.0,
      "logps/rejected": -1365.199951171875,
      "loss": 0.086,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.24960938096046448,
      "rewards/margins": 6.184374809265137,
      "rewards/rejected": -6.435937404632568,
      "step": 2355
    },
    {
      "epoch": 1.8524332810047097,
      "grad_norm": 52.81836764460795,
      "learning_rate": 8.334838467781907e-09,
      "logits/chosen": -1.0691406726837158,
      "logits/rejected": -0.8626953363418579,
      "logps/chosen": -562.5999755859375,
      "logps/rejected": -1667.199951171875,
      "loss": 0.0577,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.6070312261581421,
      "rewards/margins": 7.328125,
      "rewards/rejected": -7.940625190734863,
      "step": 2360
    },
    {
      "epoch": 1.8563579277864992,
      "grad_norm": 47.3782690958797,
      "learning_rate": 7.90198247819257e-09,
      "logits/chosen": -0.937695324420929,
      "logits/rejected": -0.622973620891571,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -1664.800048828125,
      "loss": 0.116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.627734363079071,
      "rewards/margins": 7.009375095367432,
      "rewards/rejected": -7.629687309265137,
      "step": 2365
    },
    {
      "epoch": 1.8602825745682887,
      "grad_norm": 59.50339359128698,
      "learning_rate": 7.480487609448804e-09,
      "logits/chosen": -1.0125000476837158,
      "logits/rejected": -0.6320556402206421,
      "logps/chosen": -557.5999755859375,
      "logps/rejected": -1512.0,
      "loss": 0.1036,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.47392576932907104,
      "rewards/margins": 6.987500190734863,
      "rewards/rejected": -7.462500095367432,
      "step": 2370
    },
    {
      "epoch": 1.8642072213500787,
      "grad_norm": 95.92129770189777,
      "learning_rate": 7.0703736413660895e-09,
      "logits/chosen": -1.183203101158142,
      "logits/rejected": -0.7156738042831421,
      "logps/chosen": -630.2000122070312,
      "logps/rejected": -1482.800048828125,
      "loss": 0.1184,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.427490234375,
      "rewards/margins": 6.65625,
      "rewards/rejected": -7.084374904632568,
      "step": 2375
    },
    {
      "epoch": 1.8681318681318682,
      "grad_norm": 95.21374464708528,
      "learning_rate": 6.671659819679737e-09,
      "logits/chosen": -0.87890625,
      "logits/rejected": -0.804443359375,
      "logps/chosen": -594.7999877929688,
      "logps/rejected": -1558.4000244140625,
      "loss": 0.1169,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.605273425579071,
      "rewards/margins": 6.778124809265137,
      "rewards/rejected": -7.375,
      "step": 2380
    },
    {
      "epoch": 1.8720565149136577,
      "grad_norm": 275.3410676021277,
      "learning_rate": 6.2843648551415e-09,
      "logits/chosen": -0.9833984375,
      "logits/rejected": -0.74609375,
      "logps/chosen": -599.2000122070312,
      "logps/rejected": -1794.4000244140625,
      "loss": 0.0976,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.37714844942092896,
      "rewards/margins": 7.728125095367432,
      "rewards/rejected": -8.100000381469727,
      "step": 2385
    },
    {
      "epoch": 1.8759811616954474,
      "grad_norm": 40.24562538170489,
      "learning_rate": 5.908506922641665e-09,
      "logits/chosen": -1.021093726158142,
      "logits/rejected": -0.828320324420929,
      "logps/chosen": -517.2000122070312,
      "logps/rejected": -1665.5999755859375,
      "loss": 0.0782,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.47297364473342896,
      "rewards/margins": 7.001562595367432,
      "rewards/rejected": -7.478125095367432,
      "step": 2390
    },
    {
      "epoch": 1.879905808477237,
      "grad_norm": 79.04734482690033,
      "learning_rate": 5.544103660355987e-09,
      "logits/chosen": -0.962695300579071,
      "logits/rejected": -0.74169921875,
      "logps/chosen": -596.4000244140625,
      "logps/rejected": -1612.800048828125,
      "loss": 0.119,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.48359376192092896,
      "rewards/margins": 7.084374904632568,
      "rewards/rejected": -7.565625190734863,
      "step": 2395
    },
    {
      "epoch": 1.8838304552590266,
      "grad_norm": 152.55192544987463,
      "learning_rate": 5.1911721689180675e-09,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.806640625,
      "logps/chosen": -618.0,
      "logps/rejected": -1348.0,
      "loss": 0.1235,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.47929686307907104,
      "rewards/margins": 5.984375,
      "rewards/rejected": -6.465624809265137,
      "step": 2400
    },
    {
      "epoch": 1.8877551020408163,
      "grad_norm": 158.98534534580628,
      "learning_rate": 4.8497290106169484e-09,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.7066406011581421,
      "logps/chosen": -558.5999755859375,
      "logps/rejected": -1399.5999755859375,
      "loss": 0.1919,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.34978026151657104,
      "rewards/margins": 5.890625,
      "rewards/rejected": -6.237500190734863,
      "step": 2405
    },
    {
      "epoch": 1.891679748822606,
      "grad_norm": 186.04074879061463,
      "learning_rate": 4.5197902086195895e-09,
      "logits/chosen": -0.9261718988418579,
      "logits/rejected": -0.715624988079071,
      "logps/chosen": -596.0,
      "logps/rejected": -1768.0,
      "loss": 0.0906,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.30937498807907104,
      "rewards/margins": 8.103124618530273,
      "rewards/rejected": -8.421875,
      "step": 2410
    },
    {
      "epoch": 1.8956043956043955,
      "grad_norm": 78.8018010138376,
      "learning_rate": 4.2013712462193875e-09,
      "logits/chosen": -1.0048828125,
      "logits/rejected": -0.49688720703125,
      "logps/chosen": -548.2000122070312,
      "logps/rejected": -1380.0,
      "loss": 0.0882,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2893310487270355,
      "rewards/margins": 6.556250095367432,
      "rewards/rejected": -6.84375,
      "step": 2415
    },
    {
      "epoch": 1.8995290423861853,
      "grad_norm": 66.22499681860901,
      "learning_rate": 3.894487066109037e-09,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.843554675579071,
      "logps/chosen": -581.5999755859375,
      "logps/rejected": -1514.4000244140625,
      "loss": 0.1032,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.33720701932907104,
      "rewards/margins": 6.724999904632568,
      "rewards/rejected": -7.059374809265137,
      "step": 2420
    },
    {
      "epoch": 1.903453689167975,
      "grad_norm": 55.97977500504951,
      "learning_rate": 3.599152069679812e-09,
      "logits/chosen": -1.0285155773162842,
      "logits/rejected": -0.849414050579071,
      "logps/chosen": -588.7999877929688,
      "logps/rejected": -1664.800048828125,
      "loss": 0.1659,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.673828125,
      "rewards/margins": 6.209374904632568,
      "rewards/rejected": -6.893750190734863,
      "step": 2425
    },
    {
      "epoch": 1.9073783359497645,
      "grad_norm": 73.90201374433167,
      "learning_rate": 3.315380116345412e-09,
      "logits/chosen": -0.9273437261581421,
      "logits/rejected": -0.5963134765625,
      "logps/chosen": -603.2000122070312,
      "logps/rejected": -1901.5999755859375,
      "loss": 0.1386,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.5654296875,
      "rewards/margins": 7.807812690734863,
      "rewards/rejected": -8.375,
      "step": 2430
    },
    {
      "epoch": 1.9113029827315542,
      "grad_norm": 44.101123060176214,
      "learning_rate": 3.0431845228916788e-09,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.767578125,
      "logps/chosen": -613.0,
      "logps/rejected": -1684.800048828125,
      "loss": 0.1149,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7491210699081421,
      "rewards/margins": 6.284375190734863,
      "rewards/rejected": -7.03125,
      "step": 2435
    },
    {
      "epoch": 1.915227629513344,
      "grad_norm": 38.62853484923705,
      "learning_rate": 2.7825780628516483e-09,
      "logits/chosen": -1.0671875476837158,
      "logits/rejected": -0.8564453125,
      "logps/chosen": -567.5999755859375,
      "logps/rejected": -1678.4000244140625,
      "loss": 0.0982,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.4397644102573395,
      "rewards/margins": 6.21875,
      "rewards/rejected": -6.659375190734863,
      "step": 2440
    },
    {
      "epoch": 1.9191522762951334,
      "grad_norm": 151.8088925410376,
      "learning_rate": 2.5335729659061998e-09,
      "logits/chosen": -0.9375,
      "logits/rejected": -0.788281261920929,
      "logps/chosen": -591.7999877929688,
      "logps/rejected": -1593.5999755859375,
      "loss": 0.0994,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.330322265625,
      "rewards/margins": 7.09375,
      "rewards/rejected": -7.421875,
      "step": 2445
    },
    {
      "epoch": 1.9230769230769231,
      "grad_norm": 168.84971746851934,
      "learning_rate": 2.2961809173100142e-09,
      "logits/chosen": -0.9966796636581421,
      "logits/rejected": -0.67529296875,
      "logps/chosen": -593.4000244140625,
      "logps/rejected": -1577.5999755859375,
      "loss": 0.1283,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.555957019329071,
      "rewards/margins": 7.051562309265137,
      "rewards/rejected": -7.615624904632568,
      "step": 2450
    },
    {
      "epoch": 1.9270015698587128,
      "grad_norm": 34.7951067739959,
      "learning_rate": 2.07041305734329e-09,
      "logits/chosen": -1.0265624523162842,
      "logits/rejected": -0.701855480670929,
      "logps/chosen": -555.5999755859375,
      "logps/rejected": -1610.800048828125,
      "loss": 0.0741,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.525683581829071,
      "rewards/margins": 6.859375,
      "rewards/rejected": -7.378125190734863,
      "step": 2455
    },
    {
      "epoch": 1.9309262166405023,
      "grad_norm": 70.43516004155998,
      "learning_rate": 1.8562799807889939e-09,
      "logits/chosen": -0.912304699420929,
      "logits/rejected": -0.586718738079071,
      "logps/chosen": -558.2000122070312,
      "logps/rejected": -1291.5999755859375,
      "loss": 0.1186,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.419921875,
      "rewards/margins": 5.951562404632568,
      "rewards/rejected": -6.368750095367432,
      "step": 2460
    },
    {
      "epoch": 1.9348508634222918,
      "grad_norm": 59.61333765184036,
      "learning_rate": 1.6537917364354836e-09,
      "logits/chosen": -1.0251953601837158,
      "logits/rejected": -0.7958984375,
      "logps/chosen": -587.4000244140625,
      "logps/rejected": -1404.800048828125,
      "loss": 0.1017,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.6341308355331421,
      "rewards/margins": 5.984375,
      "rewards/rejected": -6.615624904632568,
      "step": 2465
    },
    {
      "epoch": 1.9387755102040818,
      "grad_norm": 212.15027328543792,
      "learning_rate": 1.4629578266051878e-09,
      "logits/chosen": -1.033593773841858,
      "logits/rejected": -0.6666015386581421,
      "logps/chosen": -629.4000244140625,
      "logps/rejected": -1437.199951171875,
      "loss": 0.0896,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.575732409954071,
      "rewards/margins": 6.042187690734863,
      "rewards/rejected": -6.606249809265137,
      "step": 2470
    },
    {
      "epoch": 1.9427001569858713,
      "grad_norm": 32.1551678842623,
      "learning_rate": 1.2837872067085476e-09,
      "logits/chosen": -1.013281226158142,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -755.2000122070312,
      "logps/rejected": -1596.800048828125,
      "loss": 0.1739,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.4320068359375,
      "rewards/margins": 7.298437595367432,
      "rewards/rejected": -7.731249809265137,
      "step": 2475
    },
    {
      "epoch": 1.9466248037676608,
      "grad_norm": 148.2209988034083,
      "learning_rate": 1.1162882848237687e-09,
      "logits/chosen": -1.103124976158142,
      "logits/rejected": -0.715136706829071,
      "logps/chosen": -598.4000244140625,
      "logps/rejected": -1496.800048828125,
      "loss": 0.0918,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.30488282442092896,
      "rewards/margins": 6.925000190734863,
      "rewards/rejected": -7.231249809265137,
      "step": 2480
    },
    {
      "epoch": 1.9505494505494505,
      "grad_norm": 87.04197351648007,
      "learning_rate": 9.604689213022766e-10,
      "logits/chosen": -0.966796875,
      "logits/rejected": -0.749804675579071,
      "logps/chosen": -630.4000244140625,
      "logps/rejected": -1541.5999755859375,
      "loss": 0.1103,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.6706298589706421,
      "rewards/margins": 7.046875,
      "rewards/rejected": -7.715624809265137,
      "step": 2485
    },
    {
      "epoch": 1.9544740973312402,
      "grad_norm": 247.01913249101872,
      "learning_rate": 8.163364283997886e-10,
      "logits/chosen": -0.9117187261581421,
      "logits/rejected": -0.7362304925918579,
      "logps/chosen": -629.5999755859375,
      "logps/rejected": -2020.800048828125,
      "loss": 0.1056,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.706250011920929,
      "rewards/margins": 7.884375095367432,
      "rewards/rejected": -8.587499618530273,
      "step": 2490
    },
    {
      "epoch": 1.9583987441130297,
      "grad_norm": 415.2895817383117,
      "learning_rate": 6.838975699332005e-10,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.7853027582168579,
      "logps/chosen": -589.2000122070312,
      "logps/rejected": -1806.800048828125,
      "loss": 0.1197,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.435791015625,
      "rewards/margins": 6.7734375,
      "rewards/rejected": -7.214062690734863,
      "step": 2495
    },
    {
      "epoch": 1.9623233908948194,
      "grad_norm": 60.12615212543445,
      "learning_rate": 5.631585609632838e-10,
      "logits/chosen": -1.005859375,
      "logits/rejected": -0.92578125,
      "logps/chosen": -600.2000122070312,
      "logps/rejected": -1504.800048828125,
      "loss": 0.1002,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.4160400331020355,
      "rewards/margins": 7.190625190734863,
      "rewards/rejected": -7.606249809265137,
      "step": 2500
    },
    {
      "epoch": 1.9662480376766092,
      "grad_norm": 36.804557045608654,
      "learning_rate": 4.541250675028363e-10,
      "logits/chosen": -0.9105468988418579,
      "logits/rejected": -0.69873046875,
      "logps/chosen": -561.4000244140625,
      "logps/rejected": -1799.199951171875,
      "loss": 0.052,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.701708972454071,
      "rewards/margins": 7.596875190734863,
      "rewards/rejected": -8.290624618530273,
      "step": 2505
    },
    {
      "epoch": 1.9701726844583987,
      "grad_norm": 134.36811732018265,
      "learning_rate": 3.568022062508946e-10,
      "logits/chosen": -0.981249988079071,
      "logits/rejected": -0.7754882574081421,
      "logps/chosen": -567.4000244140625,
      "logps/rejected": -1789.5999755859375,
      "loss": 0.1252,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5733398199081421,
      "rewards/margins": 7.412499904632568,
      "rewards/rejected": -7.996874809265137,
      "step": 2510
    },
    {
      "epoch": 1.9740973312401884,
      "grad_norm": 194.02019778546745,
      "learning_rate": 2.711945443526209e-10,
      "logits/chosen": -1.041406273841858,
      "logits/rejected": -0.8609374761581421,
      "logps/chosen": -547.7999877929688,
      "logps/rejected": -1604.800048828125,
      "loss": 0.0804,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.22504882514476776,
      "rewards/margins": 7.0625,
      "rewards/rejected": -7.287499904632568,
      "step": 2515
    },
    {
      "epoch": 1.978021978021978,
      "grad_norm": 34.21069366981161,
      "learning_rate": 1.97306099184974e-10,
      "logits/chosen": -0.9126952886581421,
      "logits/rejected": -0.5447753667831421,
      "logps/chosen": -537.0,
      "logps/rejected": -1759.199951171875,
      "loss": 0.0874,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.25346678495407104,
      "rewards/margins": 7.435937404632568,
      "rewards/rejected": -7.690625190734863,
      "step": 2520
    },
    {
      "epoch": 1.9819466248037676,
      "grad_norm": 70.1224292449981,
      "learning_rate": 1.3514033816808267e-10,
      "logits/chosen": -0.889843761920929,
      "logits/rejected": -0.674609363079071,
      "logps/chosen": -590.0,
      "logps/rejected": -1676.800048828125,
      "loss": 0.0843,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.24578857421875,
      "rewards/margins": 7.34375,
      "rewards/rejected": -7.59375,
      "step": 2525
    },
    {
      "epoch": 1.9858712715855573,
      "grad_norm": 100.87133660475966,
      "learning_rate": 8.470017860273681e-11,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.88671875,
      "logps/chosen": -618.4000244140625,
      "logps/rejected": -1793.5999755859375,
      "loss": 0.1041,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.7432616949081421,
      "rewards/margins": 6.987500190734863,
      "rewards/rejected": -7.737500190734863,
      "step": 2530
    },
    {
      "epoch": 1.989795918367347,
      "grad_norm": 104.79968508879631,
      "learning_rate": 4.5987987533302465e-11,
      "logits/chosen": -0.960742175579071,
      "logits/rejected": -0.634570300579071,
      "logps/chosen": -650.7999877929688,
      "logps/rejected": -1952.800048828125,
      "loss": 0.3212,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.928515613079071,
      "rewards/margins": 6.487500190734863,
      "rewards/rejected": -7.418749809265137,
      "step": 2535
    },
    {
      "epoch": 1.9937205651491365,
      "grad_norm": 34.669436808864916,
      "learning_rate": 1.9005581636671876e-11,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.936328113079071,
      "logps/chosen": -666.5999755859375,
      "logps/rejected": -1478.4000244140625,
      "loss": 0.0628,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3313964903354645,
      "rewards/margins": 6.590624809265137,
      "rewards/rejected": -6.918749809265137,
      "step": 2540
    },
    {
      "epoch": 1.9976452119309263,
      "grad_norm": 38.023729870866894,
      "learning_rate": 3.754227137081623e-12,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.75244140625,
      "logps/chosen": -580.5999755859375,
      "logps/rejected": -1900.0,
      "loss": 0.0682,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.4244628846645355,
      "rewards/margins": 7.287499904632568,
      "rewards/rejected": -7.715624809265137,
      "step": 2545
    },
    {
      "epoch": 2.0,
      "step": 2548,
      "total_flos": 0.0,
      "train_loss": 0.4729907219554042,
      "train_runtime": 6750.8442,
      "train_samples_per_second": 12.074,
      "train_steps_per_second": 0.377
    }
  ],
  "logging_steps": 5,
  "max_steps": 2548,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}