{
  "best_metric": 0.21195410192012787,
  "best_model_checkpoint": "models/qwen2.5-3b-dpo-finegrained/checkpoint-10000",
  "epoch": 1.0,
  "eval_steps": 5000,
  "global_step": 13853,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 7.218652999350322e-05,
      "grad_norm": 1.852354340577762,
      "learning_rate": 3.6075036075036073e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -1.6015625,
      "logps/chosen": -146.0,
      "logps/rejected": -166.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0007218652999350321,
      "grad_norm": 1.998199384338708,
      "learning_rate": 3.6075036075036073e-09,
      "logits/chosen": -2.265625,
      "logits/rejected": -1.953125,
      "logps/chosen": -157.0,
      "logps/rejected": -143.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.2638888955116272,
      "rewards/chosen": 0.000766754150390625,
      "rewards/margins": 0.000606536865234375,
      "rewards/rejected": 0.000156402587890625,
      "step": 10
    },
    {
      "epoch": 0.0014437305998700643,
      "grad_norm": 1.859902408963682,
      "learning_rate": 7.215007215007215e-09,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.984375,
      "logps/chosen": -157.0,
      "logps/rejected": -139.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.001129150390625,
      "rewards/margins": 0.00061798095703125,
      "rewards/rejected": 0.000507354736328125,
      "step": 20
    },
    {
      "epoch": 0.0021655958998050965,
      "grad_norm": 1.800945636929482,
      "learning_rate": 1.0822510822510822e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.015625,
      "logps/chosen": -146.0,
      "logps/rejected": -148.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.000640869140625,
      "rewards/margins": -0.0003757476806640625,
      "rewards/rejected": 0.00101470947265625,
      "step": 30
    },
    {
      "epoch": 0.0028874611997401285,
      "grad_norm": 1.9512697515276394,
      "learning_rate": 1.443001443001443e-08,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.0,
      "logps/chosen": -165.0,
      "logps/rejected": -161.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 0.00010967254638671875,
      "rewards/margins": 0.0003910064697265625,
      "rewards/rejected": -0.00028228759765625,
      "step": 40
    },
    {
      "epoch": 0.0036093264996751606,
      "grad_norm": 1.9685077450218404,
      "learning_rate": 1.8037518037518035e-08,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.015625,
      "logps/chosen": -165.0,
      "logps/rejected": -143.0,
      "loss": 0.6929,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": 9.441375732421875e-05,
      "rewards/margins": 0.0001239776611328125,
      "rewards/rejected": -3.0159950256347656e-05,
      "step": 50
    },
    {
      "epoch": 0.004331191799610193,
      "grad_norm": 1.9944209672205195,
      "learning_rate": 2.1645021645021644e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.984375,
      "logps/chosen": -160.0,
      "logps/rejected": -136.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.000232696533203125,
      "rewards/margins": 0.00102996826171875,
      "rewards/rejected": -0.0012664794921875,
      "step": 60
    },
    {
      "epoch": 0.005053057099545225,
      "grad_norm": 1.8602055997261977,
      "learning_rate": 2.525252525252525e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -1.8984375,
      "logps/chosen": -150.0,
      "logps/rejected": -154.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.000438690185546875,
      "rewards/margins": 0.00054931640625,
      "rewards/rejected": -0.00010919570922851562,
      "step": 70
    },
    {
      "epoch": 0.005774922399480257,
      "grad_norm": 1.7129972696287459,
      "learning_rate": 2.886002886002886e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.046875,
      "logps/chosen": -170.0,
      "logps/rejected": -145.0,
      "loss": 0.6928,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.000579833984375,
      "rewards/margins": 0.0004520416259765625,
      "rewards/rejected": 0.0001239776611328125,
      "step": 80
    },
    {
      "epoch": 0.006496787699415289,
      "grad_norm": 1.8246478253492482,
      "learning_rate": 3.246753246753246e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.0625,
      "logps/chosen": -178.0,
      "logps/rejected": -158.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.000469207763671875,
      "rewards/margins": 0.0001888275146484375,
      "rewards/rejected": 0.0002803802490234375,
      "step": 90
    },
    {
      "epoch": 0.007218652999350321,
      "grad_norm": 1.7657315202986226,
      "learning_rate": 3.607503607503607e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -1.984375,
      "logps/chosen": -172.0,
      "logps/rejected": -163.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 0.000797271728515625,
      "rewards/margins": -0.0003910064697265625,
      "rewards/rejected": 0.001190185546875,
      "step": 100
    },
    {
      "epoch": 0.007940518299285354,
      "grad_norm": 2.0123631368082844,
      "learning_rate": 3.968253968253968e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -1.9296875,
      "logps/chosen": -153.0,
      "logps/rejected": -140.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": 0.000186920166015625,
      "rewards/margins": 0.00018787384033203125,
      "rewards/rejected": 1.1473894119262695e-06,
      "step": 110
    },
    {
      "epoch": 0.008662383599220386,
      "grad_norm": 1.9667654175700489,
      "learning_rate": 4.329004329004329e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.0,
      "logps/chosen": -146.0,
      "logps/rejected": -137.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.00154876708984375,
      "rewards/margins": -0.00103759765625,
      "rewards/rejected": 0.0025787353515625,
      "step": 120
    },
    {
      "epoch": 0.009384248899155418,
      "grad_norm": 1.7516436962452608,
      "learning_rate": 4.68975468975469e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.1875,
      "logps/chosen": -152.0,
      "logps/rejected": -151.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.0027313232421875,
      "rewards/margins": 0.00164031982421875,
      "rewards/rejected": 0.0010986328125,
      "step": 130
    },
    {
      "epoch": 0.01010611419909045,
      "grad_norm": 2.0629971445302435,
      "learning_rate": 5.05050505050505e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.046875,
      "logps/chosen": -167.0,
      "logps/rejected": -152.0,
      "loss": 0.6921,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.0018768310546875,
      "rewards/margins": 0.000438690185546875,
      "rewards/rejected": 0.00144195556640625,
      "step": 140
    },
    {
      "epoch": 0.010827979499025482,
      "grad_norm": 2.0455409873804205,
      "learning_rate": 5.411255411255411e-08,
      "logits/chosen": -2.375,
      "logits/rejected": -2.09375,
      "logps/chosen": -165.0,
      "logps/rejected": -147.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.001739501953125,
      "rewards/margins": 5.435943603515625e-05,
      "rewards/rejected": 0.001678466796875,
      "step": 150
    },
    {
      "epoch": 0.011549844798960514,
      "grad_norm": 2.003008491192784,
      "learning_rate": 5.772005772005772e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -166.0,
      "logps/rejected": -161.0,
      "loss": 0.6919,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0026702880859375,
      "rewards/margins": 0.00093841552734375,
      "rewards/rejected": 0.00173187255859375,
      "step": 160
    },
    {
      "epoch": 0.012271710098895546,
      "grad_norm": 1.8970983946398985,
      "learning_rate": 6.132756132756133e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.9921875,
      "logps/chosen": -163.0,
      "logps/rejected": -142.0,
      "loss": 0.6915,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00445556640625,
      "rewards/margins": 0.00396728515625,
      "rewards/rejected": 0.0004787445068359375,
      "step": 170
    },
    {
      "epoch": 0.012993575398830578,
      "grad_norm": 1.9000812663628288,
      "learning_rate": 6.493506493506492e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -1.9140625,
      "logps/chosen": -184.0,
      "logps/rejected": -166.0,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004791259765625,
      "rewards/margins": 0.00274658203125,
      "rewards/rejected": 0.0020294189453125,
      "step": 180
    },
    {
      "epoch": 0.01371544069876561,
      "grad_norm": 1.7694192875136483,
      "learning_rate": 6.854256854256854e-08,
      "logits/chosen": -2.375,
      "logits/rejected": -2.015625,
      "logps/chosen": -168.0,
      "logps/rejected": -147.0,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007598876953125,
      "rewards/margins": 0.003387451171875,
      "rewards/rejected": 0.00421142578125,
      "step": 190
    },
    {
      "epoch": 0.014437305998700642,
      "grad_norm": 1.8022914272788901,
      "learning_rate": 7.215007215007214e-08,
      "logits/chosen": -2.296875,
      "logits/rejected": -1.890625,
      "logps/chosen": -150.0,
      "logps/rejected": -137.0,
      "loss": 0.6907,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.007354736328125,
      "rewards/margins": 0.005584716796875,
      "rewards/rejected": 0.0017547607421875,
      "step": 200
    },
    {
      "epoch": 0.015159171298635674,
      "grad_norm": 1.7754509638333658,
      "learning_rate": 7.575757575757576e-08,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.0,
      "logps/chosen": -155.0,
      "logps/rejected": -151.0,
      "loss": 0.6908,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.007537841796875,
      "rewards/margins": 0.0034942626953125,
      "rewards/rejected": 0.0040283203125,
      "step": 210
    },
    {
      "epoch": 0.015881036598570708,
      "grad_norm": 1.8713087698233093,
      "learning_rate": 7.936507936507936e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -1.8984375,
      "logps/chosen": -156.0,
      "logps/rejected": -168.0,
      "loss": 0.6903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00860595703125,
      "rewards/margins": 0.00469970703125,
      "rewards/rejected": 0.00390625,
      "step": 220
    },
    {
      "epoch": 0.01660290189850574,
      "grad_norm": 1.7352700913418746,
      "learning_rate": 8.297258297258297e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.0,
      "logps/chosen": -171.0,
      "logps/rejected": -150.0,
      "loss": 0.6897,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.01153564453125,
      "rewards/margins": 0.00640869140625,
      "rewards/rejected": 0.005157470703125,
      "step": 230
    },
    {
      "epoch": 0.017324767198440772,
      "grad_norm": 1.923679325098434,
      "learning_rate": 8.658008658008658e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.171875,
      "logps/chosen": -168.0,
      "logps/rejected": -153.0,
      "loss": 0.6889,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01507568359375,
      "rewards/margins": 0.00933837890625,
      "rewards/rejected": 0.005706787109375,
      "step": 240
    },
    {
      "epoch": 0.018046632498375802,
      "grad_norm": 1.9336109183415302,
      "learning_rate": 9.018759018759018e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.140625,
      "logps/chosen": -198.0,
      "logps/rejected": -190.0,
      "loss": 0.6883,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.015625,
      "rewards/margins": 0.01055908203125,
      "rewards/rejected": 0.005035400390625,
      "step": 250
    },
    {
      "epoch": 0.018768497798310836,
      "grad_norm": 1.7118991587292718,
      "learning_rate": 9.37950937950938e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -1.9921875,
      "logps/chosen": -148.0,
      "logps/rejected": -164.0,
      "loss": 0.6873,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.012451171875,
      "rewards/rejected": 0.00469970703125,
      "step": 260
    },
    {
      "epoch": 0.019490363098245866,
      "grad_norm": 2.0498889647791634,
      "learning_rate": 9.74025974025974e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.125,
      "logps/chosen": -168.0,
      "logps/rejected": -150.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.0159912109375,
      "rewards/margins": 0.00982666015625,
      "rewards/rejected": 0.006195068359375,
      "step": 270
    },
    {
      "epoch": 0.0202122283981809,
      "grad_norm": 1.7948780770940682,
      "learning_rate": 1.01010101010101e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -1.9140625,
      "logps/chosen": -162.0,
      "logps/rejected": -170.0,
      "loss": 0.6865,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.018310546875,
      "rewards/margins": 0.01513671875,
      "rewards/rejected": 0.0032196044921875,
      "step": 280
    },
    {
      "epoch": 0.02093409369811593,
      "grad_norm": 1.7485728753360685,
      "learning_rate": 1.0461760461760462e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.09375,
      "logps/chosen": -173.0,
      "logps/rejected": -152.0,
      "loss": 0.6865,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.0089111328125,
      "rewards/rejected": 0.0084228515625,
      "step": 290
    },
    {
      "epoch": 0.021655958998050964,
      "grad_norm": 1.7752446010354384,
      "learning_rate": 1.0822510822510822e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.15625,
      "logps/chosen": -162.0,
      "logps/rejected": -165.0,
      "loss": 0.6857,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0238037109375,
      "rewards/margins": 0.015625,
      "rewards/rejected": 0.00823974609375,
      "step": 300
    },
    {
      "epoch": 0.022377824297985995,
      "grad_norm": 1.9527079542055057,
      "learning_rate": 1.1183261183261184e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.078125,
      "logps/chosen": -148.0,
      "logps/rejected": -153.0,
      "loss": 0.6845,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.0203857421875,
      "rewards/margins": 0.0184326171875,
      "rewards/rejected": 0.00201416015625,
      "step": 310
    },
    {
      "epoch": 0.02309968959792103,
      "grad_norm": 1.798683396295616,
      "learning_rate": 1.1544011544011543e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.09375,
      "logps/chosen": -163.0,
      "logps/rejected": -141.0,
      "loss": 0.6844,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.02197265625,
      "rewards/margins": 0.0184326171875,
      "rewards/rejected": 0.003570556640625,
      "step": 320
    },
    {
      "epoch": 0.02382155489785606,
      "grad_norm": 1.822086025310402,
      "learning_rate": 1.1904761904761903e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -1.9765625,
      "logps/chosen": -159.0,
      "logps/rejected": -147.0,
      "loss": 0.6816,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02490234375,
      "rewards/margins": 0.0225830078125,
      "rewards/rejected": 0.002349853515625,
      "step": 330
    },
    {
      "epoch": 0.024543420197791092,
      "grad_norm": 1.8205804931965275,
      "learning_rate": 1.2265512265512265e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.125,
      "logps/chosen": -163.0,
      "logps/rejected": -168.0,
      "loss": 0.6809,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0240478515625,
      "rewards/margins": 0.02392578125,
      "rewards/rejected": 4.839897155761719e-05,
      "step": 340
    },
    {
      "epoch": 0.025265285497726123,
      "grad_norm": 2.015225469187424,
      "learning_rate": 1.2626262626262626e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.1875,
      "logps/chosen": -168.0,
      "logps/rejected": -138.0,
      "loss": 0.6787,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.023193359375,
      "rewards/margins": 0.02880859375,
      "rewards/rejected": -0.005706787109375,
      "step": 350
    },
    {
      "epoch": 0.025987150797661156,
      "grad_norm": 1.8960338628946363,
      "learning_rate": 1.2987012987012984e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.0625,
      "logps/chosen": -171.0,
      "logps/rejected": -155.0,
      "loss": 0.679,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.024169921875,
      "rewards/margins": 0.029052734375,
      "rewards/rejected": -0.004791259765625,
      "step": 360
    },
    {
      "epoch": 0.02670901609759619,
      "grad_norm": 1.9289106441512014,
      "learning_rate": 1.3347763347763348e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.125,
      "logps/chosen": -143.0,
      "logps/rejected": -147.0,
      "loss": 0.6755,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0277099609375,
      "rewards/margins": 0.036376953125,
      "rewards/rejected": -0.00848388671875,
      "step": 370
    },
    {
      "epoch": 0.02743088139753122,
      "grad_norm": 2.4318763159683168,
      "learning_rate": 1.370851370851371e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.078125,
      "logps/chosen": -161.0,
      "logps/rejected": -154.0,
      "loss": 0.6738,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.016845703125,
      "rewards/margins": 0.0390625,
      "rewards/rejected": -0.022216796875,
      "step": 380
    },
    {
      "epoch": 0.028152746697466254,
      "grad_norm": 1.727518768188907,
      "learning_rate": 1.406926406926407e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.125,
      "logps/chosen": -151.0,
      "logps/rejected": -145.0,
      "loss": 0.6737,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.0238037109375,
      "rewards/margins": 0.04541015625,
      "rewards/rejected": -0.0216064453125,
      "step": 390
    },
    {
      "epoch": 0.028874611997401285,
      "grad_norm": 1.8565593414895172,
      "learning_rate": 1.4430014430014428e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.046875,
      "logps/chosen": -169.0,
      "logps/rejected": -162.0,
      "loss": 0.6703,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.01324462890625,
      "rewards/margins": 0.0458984375,
      "rewards/rejected": -0.03271484375,
      "step": 400
    },
    {
      "epoch": 0.02959647729733632,
      "grad_norm": 1.9588571879597823,
      "learning_rate": 1.479076479076479e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.296875,
      "logps/chosen": -152.0,
      "logps/rejected": -137.0,
      "loss": 0.6681,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.006378173828125,
      "rewards/margins": 0.052734375,
      "rewards/rejected": -0.04638671875,
      "step": 410
    },
    {
      "epoch": 0.03031834259727135,
      "grad_norm": 1.93069247030128,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.203125,
      "logps/chosen": -172.0,
      "logps/rejected": -144.0,
      "loss": 0.6667,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0025787353515625,
      "rewards/margins": 0.044189453125,
      "rewards/rejected": -0.046875,
      "step": 420
    },
    {
      "epoch": 0.031040207897206382,
      "grad_norm": 1.819742407006859,
      "learning_rate": 1.5512265512265513e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.125,
      "logps/chosen": -172.0,
      "logps/rejected": -152.0,
      "loss": 0.6625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.002685546875,
      "rewards/margins": 0.06396484375,
      "rewards/rejected": -0.06640625,
      "step": 430
    },
    {
      "epoch": 0.031762073197141416,
      "grad_norm": 2.3055219231256108,
      "learning_rate": 1.5873015873015872e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.1875,
      "logps/chosen": -155.0,
      "logps/rejected": -158.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.01123046875,
      "rewards/margins": 0.078125,
      "rewards/rejected": -0.08935546875,
      "step": 440
    },
    {
      "epoch": 0.032483938497076446,
      "grad_norm": 1.9129712763765747,
      "learning_rate": 1.6233766233766232e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.140625,
      "logps/chosen": -166.0,
      "logps/rejected": -154.0,
      "loss": 0.6547,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.031494140625,
      "rewards/margins": 0.07666015625,
      "rewards/rejected": -0.10791015625,
      "step": 450
    },
    {
      "epoch": 0.03320580379701148,
      "grad_norm": 1.936887902580248,
      "learning_rate": 1.6594516594516593e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.25,
      "logps/chosen": -166.0,
      "logps/rejected": -163.0,
      "loss": 0.65,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0478515625,
      "rewards/margins": 0.08740234375,
      "rewards/rejected": -0.1357421875,
      "step": 460
    },
    {
      "epoch": 0.03392766909694651,
      "grad_norm": 2.1168551608123725,
      "learning_rate": 1.6955266955266957e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.234375,
      "logps/chosen": -187.0,
      "logps/rejected": -190.0,
      "loss": 0.6399,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": -0.17578125,
      "step": 470
    },
    {
      "epoch": 0.034649534396881544,
      "grad_norm": 2.4110075564522533,
      "learning_rate": 1.7316017316017315e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.296875,
      "logps/chosen": -171.0,
      "logps/rejected": -167.0,
      "loss": 0.6345,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.09375,
      "rewards/margins": 0.1376953125,
      "rewards/rejected": -0.2314453125,
      "step": 480
    },
    {
      "epoch": 0.035371399696816574,
      "grad_norm": 2.561479324237141,
      "learning_rate": 1.7676767676767676e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -165.0,
      "logps/rejected": -169.0,
      "loss": 0.6272,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1552734375,
      "rewards/margins": 0.1328125,
      "rewards/rejected": -0.287109375,
      "step": 490
    },
    {
      "epoch": 0.036093264996751605,
      "grad_norm": 2.2009284942320004,
      "learning_rate": 1.8037518037518037e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.1875,
      "logps/chosen": -180.0,
      "logps/rejected": -198.0,
      "loss": 0.6213,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1826171875,
      "rewards/margins": 0.1806640625,
      "rewards/rejected": -0.36328125,
      "step": 500
    },
    {
      "epoch": 0.036815130296686635,
      "grad_norm": 2.433624086599741,
      "learning_rate": 1.8398268398268395e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.203125,
      "logps/chosen": -182.0,
      "logps/rejected": -181.0,
      "loss": 0.6137,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2294921875,
      "rewards/margins": 0.1787109375,
      "rewards/rejected": -0.408203125,
      "step": 510
    },
    {
      "epoch": 0.03753699559662167,
      "grad_norm": 2.3303081056576396,
      "learning_rate": 1.875901875901876e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.3125,
      "logps/chosen": -183.0,
      "logps/rejected": -198.0,
      "loss": 0.6089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28125,
      "rewards/margins": 0.1953125,
      "rewards/rejected": -0.4765625,
      "step": 520
    },
    {
      "epoch": 0.0382588608965567,
      "grad_norm": 7.529255626517495,
      "learning_rate": 1.911976911976912e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.203125,
      "logps/chosen": -188.0,
      "logps/rejected": -206.0,
      "loss": 0.6063,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3046875,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.5390625,
      "step": 530
    },
    {
      "epoch": 0.03898072619649173,
      "grad_norm": 2.2072494405878578,
      "learning_rate": 1.948051948051948e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.3125,
      "logps/chosen": -203.0,
      "logps/rejected": -224.0,
      "loss": 0.5956,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3515625,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.6328125,
      "step": 540
    },
    {
      "epoch": 0.03970259149642677,
      "grad_norm": 2.310476086986689,
      "learning_rate": 1.984126984126984e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -220.0,
      "logps/rejected": -209.0,
      "loss": 0.5867,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.416015625,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.65234375,
      "step": 550
    },
    {
      "epoch": 0.0404244567963618,
      "grad_norm": 2.6001658124806286,
      "learning_rate": 2.02020202020202e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.359375,
      "logps/chosen": -208.0,
      "logps/rejected": -239.0,
      "loss": 0.5754,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.447265625,
      "rewards/margins": 0.31640625,
      "rewards/rejected": -0.76171875,
      "step": 560
    },
    {
      "epoch": 0.04114632209629683,
      "grad_norm": 2.635075269431316,
      "learning_rate": 2.0562770562770563e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -217.0,
      "logps/rejected": -227.0,
      "loss": 0.5719,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.494140625,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -0.83203125,
      "step": 570
    },
    {
      "epoch": 0.04186818739623186,
      "grad_norm": 2.5380650062224066,
      "learning_rate": 2.0923520923520924e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.453125,
      "logps/chosen": -216.0,
      "logps/rejected": -237.0,
      "loss": 0.5794,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 0.349609375,
      "rewards/rejected": -0.8515625,
      "step": 580
    },
    {
      "epoch": 0.0425900526961669,
      "grad_norm": 2.808866971835476,
      "learning_rate": 2.1284271284271282e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -216.0,
      "logps/rejected": -264.0,
      "loss": 0.5557,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.50390625,
      "rewards/rejected": -1.046875,
      "step": 590
    },
    {
      "epoch": 0.04331191799610193,
      "grad_norm": 3.660790591856929,
      "learning_rate": 2.1645021645021643e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.734375,
      "logps/chosen": -215.0,
      "logps/rejected": -226.0,
      "loss": 0.5374,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.60546875,
      "rewards/margins": 0.3203125,
      "rewards/rejected": -0.92578125,
      "step": 600
    },
    {
      "epoch": 0.04403378329603696,
      "grad_norm": 3.4012692341173842,
      "learning_rate": 2.2005772005772004e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.671875,
      "logps/chosen": -243.0,
      "logps/rejected": -288.0,
      "loss": 0.5302,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.51953125,
      "rewards/rejected": -1.2734375,
      "step": 610
    },
    {
      "epoch": 0.04475564859597199,
      "grad_norm": 3.1213723367337276,
      "learning_rate": 2.2366522366522368e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -2.796875,
      "logps/chosen": -252.0,
      "logps/rejected": -290.0,
      "loss": 0.5349,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.82421875,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -1.4375,
      "step": 620
    },
    {
      "epoch": 0.045477513895907026,
      "grad_norm": 3.4139219834989825,
      "learning_rate": 2.2727272727272726e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.8125,
      "logps/chosen": -241.0,
      "logps/rejected": -290.0,
      "loss": 0.5313,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.390625,
      "step": 630
    },
    {
      "epoch": 0.04619937919584206,
      "grad_norm": 3.545601457816913,
      "learning_rate": 2.3088023088023087e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.796875,
      "logps/chosen": -254.0,
      "logps/rejected": -306.0,
      "loss": 0.5163,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.796875,
      "rewards/margins": 0.671875,
      "rewards/rejected": -1.46875,
      "step": 640
    },
    {
      "epoch": 0.04692124449577709,
      "grad_norm": 6.320535114176617,
      "learning_rate": 2.3448773448773448e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.9375,
      "logps/chosen": -258.0,
      "logps/rejected": -312.0,
      "loss": 0.5168,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.5546875,
      "step": 650
    },
    {
      "epoch": 0.04764310979571212,
      "grad_norm": 4.40606882426551,
      "learning_rate": 2.3809523809523806e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -2.984375,
      "logps/chosen": -258.0,
      "logps/rejected": -300.0,
      "loss": 0.506,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9375,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -1.5546875,
      "step": 660
    },
    {
      "epoch": 0.048364975095647154,
      "grad_norm": 7.228420353041346,
      "learning_rate": 2.4170274170274167e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.8125,
      "logps/chosen": -230.0,
      "logps/rejected": -308.0,
      "loss": 0.5231,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 0.7421875,
      "rewards/rejected": -1.578125,
      "step": 670
    },
    {
      "epoch": 0.049086840395582185,
      "grad_norm": 4.810449590822843,
      "learning_rate": 2.453102453102453e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.75,
      "logps/chosen": -246.0,
      "logps/rejected": -280.0,
      "loss": 0.5107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 0.498046875,
      "rewards/rejected": -1.3828125,
      "step": 680
    },
    {
      "epoch": 0.049808705695517215,
      "grad_norm": 6.916956622472092,
      "learning_rate": 2.4891774891774894e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.9375,
      "logps/chosen": -270.0,
      "logps/rejected": -322.0,
      "loss": 0.5105,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -1.625,
      "step": 690
    },
    {
      "epoch": 0.050530570995452245,
      "grad_norm": 5.072218871042774,
      "learning_rate": 2.525252525252525e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.84375,
      "logps/chosen": -256.0,
      "logps/rejected": -326.0,
      "loss": 0.5065,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.76953125,
      "rewards/rejected": -1.6953125,
      "step": 700
    },
    {
      "epoch": 0.05125243629538728,
      "grad_norm": 8.875662849487487,
      "learning_rate": 2.5613275613275616e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.84375,
      "logps/chosen": -254.0,
      "logps/rejected": -306.0,
      "loss": 0.5112,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.94921875,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.5625,
      "step": 710
    },
    {
      "epoch": 0.05197430159532231,
      "grad_norm": 5.595849818392523,
      "learning_rate": 2.597402597402597e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.96875,
      "logps/chosen": -264.0,
      "logps/rejected": -340.0,
      "loss": 0.5107,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 0.8125,
      "rewards/rejected": -1.8203125,
      "step": 720
    },
    {
      "epoch": 0.05269616689525734,
      "grad_norm": 5.404147285645744,
      "learning_rate": 2.633477633477633e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.65625,
      "logps/chosen": -268.0,
      "logps/rejected": -368.0,
      "loss": 0.5035,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.984375,
      "rewards/margins": 0.94140625,
      "rewards/rejected": -1.9296875,
      "step": 730
    },
    {
      "epoch": 0.05341803219519238,
      "grad_norm": 8.76041109611346,
      "learning_rate": 2.6695526695526696e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.828125,
      "logps/chosen": -286.0,
      "logps/rejected": -342.0,
      "loss": 0.4887,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.75390625,
      "rewards/rejected": -1.8828125,
      "step": 740
    },
    {
      "epoch": 0.05413989749512741,
      "grad_norm": 7.025589443972428,
      "learning_rate": 2.7056277056277054e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.78125,
      "logps/chosen": -262.0,
      "logps/rejected": -348.0,
      "loss": 0.4823,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -1.953125,
      "step": 750
    },
    {
      "epoch": 0.05486176279506244,
      "grad_norm": 11.975302977639574,
      "learning_rate": 2.741702741702742e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.75,
      "logps/chosen": -276.0,
      "logps/rejected": -322.0,
      "loss": 0.4766,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.140625,
      "rewards/margins": 0.6953125,
      "rewards/rejected": -1.8359375,
      "step": 760
    },
    {
      "epoch": 0.05558362809499747,
      "grad_norm": 6.5118457359444,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.734375,
      "logps/chosen": -280.0,
      "logps/rejected": -346.0,
      "loss": 0.4846,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1875,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -1.9921875,
      "step": 770
    },
    {
      "epoch": 0.05630549339493251,
      "grad_norm": 6.106781577833728,
      "learning_rate": 2.813852813852814e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.6875,
      "logps/chosen": -260.0,
      "logps/rejected": -356.0,
      "loss": 0.4665,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -1.875,
      "step": 780
    },
    {
      "epoch": 0.05702735869486754,
      "grad_norm": 10.75495308386994,
      "learning_rate": 2.8499278499278503e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -284.0,
      "logps/rejected": -408.0,
      "loss": 0.4695,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.390625,
      "step": 790
    },
    {
      "epoch": 0.05774922399480257,
      "grad_norm": 5.788421207967755,
      "learning_rate": 2.8860028860028856e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -274.0,
      "logps/rejected": -334.0,
      "loss": 0.4767,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.84765625,
      "rewards/rejected": -1.9765625,
      "step": 800
    },
    {
      "epoch": 0.0584710892947376,
      "grad_norm": 8.78614857490401,
      "learning_rate": 2.922077922077922e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.71875,
      "logps/chosen": -262.0,
      "logps/rejected": -362.0,
      "loss": 0.4722,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.125,
      "step": 810
    },
    {
      "epoch": 0.05919295459467264,
      "grad_norm": 7.3161771388355294,
      "learning_rate": 2.958152958152958e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -272.0,
      "logps/rejected": -376.0,
      "loss": 0.4996,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.109375,
      "step": 820
    },
    {
      "epoch": 0.05991481989460767,
      "grad_norm": 8.266553993044619,
      "learning_rate": 2.994227994227994e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -282.0,
      "logps/rejected": -376.0,
      "loss": 0.4545,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.15625,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.171875,
      "step": 830
    },
    {
      "epoch": 0.0606366851945427,
      "grad_norm": 8.218438777679667,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -272.0,
      "logps/rejected": -382.0,
      "loss": 0.4639,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.25,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.28125,
      "step": 840
    },
    {
      "epoch": 0.06135855049447773,
      "grad_norm": 8.61378341353772,
      "learning_rate": 3.0663780663780663e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.640625,
      "logps/chosen": -284.0,
      "logps/rejected": -408.0,
      "loss": 0.4536,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.359375,
      "step": 850
    },
    {
      "epoch": 0.062080415794412765,
      "grad_norm": 12.398406848670698,
      "learning_rate": 3.1024531024531027e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -286.0,
      "logps/rejected": -372.0,
      "loss": 0.476,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -2.265625,
      "step": 860
    },
    {
      "epoch": 0.0628022810943478,
      "grad_norm": 12.244121744087602,
      "learning_rate": 3.138528138528138e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -294.0,
      "logps/rejected": -388.0,
      "loss": 0.4623,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.359375,
      "step": 870
    },
    {
      "epoch": 0.06352414639428283,
      "grad_norm": 10.812869810884447,
      "learning_rate": 3.1746031746031743e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.796875,
      "logps/chosen": -268.0,
      "logps/rejected": -366.0,
      "loss": 0.4498,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.234375,
      "step": 880
    },
    {
      "epoch": 0.06424601169421786,
      "grad_norm": 11.229212598361977,
      "learning_rate": 3.2106782106782107e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -298.0,
      "logps/rejected": -394.0,
      "loss": 0.4373,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.46875,
      "step": 890
    },
    {
      "epoch": 0.06496787699415289,
      "grad_norm": 7.414765021326351,
      "learning_rate": 3.2467532467532465e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.75,
      "logps/chosen": -294.0,
      "logps/rejected": -394.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.34375,
      "rewards/margins": 1.140625,
      "rewards/rejected": -2.484375,
      "step": 900
    },
    {
      "epoch": 0.06568974229408793,
      "grad_norm": 6.656174306105041,
      "learning_rate": 3.282828282828283e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -322.0,
      "logps/rejected": -440.0,
      "loss": 0.4366,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -2.78125,
      "step": 910
    },
    {
      "epoch": 0.06641160759402295,
      "grad_norm": 6.986955189515223,
      "learning_rate": 3.3189033189033187e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -328.0,
      "logps/rejected": -422.0,
      "loss": 0.4335,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.515625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.75,
      "step": 920
    },
    {
      "epoch": 0.06713347289395799,
      "grad_norm": 8.704691417598838,
      "learning_rate": 3.354978354978355e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -322.0,
      "logps/rejected": -406.0,
      "loss": 0.4048,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.640625,
      "step": 930
    },
    {
      "epoch": 0.06785533819389301,
      "grad_norm": 19.52535270358171,
      "learning_rate": 3.3910533910533914e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -476.0,
      "loss": 0.434,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -3.21875,
      "step": 940
    },
    {
      "epoch": 0.06857720349382805,
      "grad_norm": 11.339737953685198,
      "learning_rate": 3.4271284271284267e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.375,
      "logps/chosen": -316.0,
      "logps/rejected": -398.0,
      "loss": 0.4334,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.59375,
      "step": 950
    },
    {
      "epoch": 0.06929906879376309,
      "grad_norm": 10.31037341226185,
      "learning_rate": 3.463203463203463e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.4375,
      "logps/chosen": -322.0,
      "logps/rejected": -456.0,
      "loss": 0.4161,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.046875,
      "step": 960
    },
    {
      "epoch": 0.07002093409369811,
      "grad_norm": 11.130584733664083,
      "learning_rate": 3.499278499278499e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -298.0,
      "logps/rejected": -424.0,
      "loss": 0.4078,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -2.796875,
      "step": 970
    },
    {
      "epoch": 0.07074279939363315,
      "grad_norm": 9.06083640139615,
      "learning_rate": 3.535353535353535e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -308.0,
      "logps/rejected": -442.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -2.890625,
      "step": 980
    },
    {
      "epoch": 0.07146466469356819,
      "grad_norm": 9.541422651232944,
      "learning_rate": 3.5714285714285716e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -338.0,
      "logps/rejected": -450.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.078125,
      "step": 990
    },
    {
      "epoch": 0.07218652999350321,
      "grad_norm": 9.110551563711448,
      "learning_rate": 3.6075036075036074e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -326.0,
      "logps/rejected": -464.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -2.984375,
      "step": 1000
    },
    {
      "epoch": 0.07290839529343825,
      "grad_norm": 15.071837561832831,
      "learning_rate": 3.643578643578644e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -314.0,
      "logps/rejected": -452.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -2.921875,
      "step": 1010
    },
    {
      "epoch": 0.07363026059337327,
      "grad_norm": 13.920977378141336,
      "learning_rate": 3.679653679653679e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -334.0,
      "logps/rejected": -492.0,
      "loss": 0.431,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -3.28125,
      "step": 1020
    },
    {
      "epoch": 0.07435212589330831,
      "grad_norm": 11.085289038392293,
      "learning_rate": 3.7157287157287154e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -298.0,
      "logps/rejected": -456.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -2.96875,
      "step": 1030
    },
    {
      "epoch": 0.07507399119324334,
      "grad_norm": 10.088971897286424,
      "learning_rate": 3.751803751803752e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -322.0,
      "logps/rejected": -456.0,
      "loss": 0.4073,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -3.0625,
      "step": 1040
    },
    {
      "epoch": 0.07579585649317837,
      "grad_norm": 43.24163673888106,
      "learning_rate": 3.7878787878787876e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.375,
      "logps/chosen": -348.0,
      "logps/rejected": -496.0,
      "loss": 0.4152,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.140625,
      "step": 1050
    },
    {
      "epoch": 0.0765177217931134,
      "grad_norm": 8.537892452168746,
      "learning_rate": 3.823953823953824e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -338.0,
      "logps/rejected": -480.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -3.296875,
      "step": 1060
    },
    {
      "epoch": 0.07723958709304844,
      "grad_norm": 9.485771716655114,
      "learning_rate": 3.86002886002886e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.375,
      "logps/chosen": -326.0,
      "logps/rejected": -462.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -3.09375,
      "step": 1070
    },
    {
      "epoch": 0.07796145239298347,
      "grad_norm": 9.46365785011617,
      "learning_rate": 3.896103896103896e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.328125,
      "logps/chosen": -386.0,
      "logps/rejected": -536.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -3.9375,
      "step": 1080
    },
    {
      "epoch": 0.0786833176929185,
      "grad_norm": 13.461346342787172,
      "learning_rate": 3.9321789321789325e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -360.0,
      "logps/rejected": -520.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.765625,
      "step": 1090
    },
    {
      "epoch": 0.07940518299285354,
      "grad_norm": 12.44634151581957,
      "learning_rate": 3.968253968253968e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -356.0,
      "logps/rejected": -508.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.5,
      "rewards/rejected": -3.515625,
      "step": 1100
    },
    {
      "epoch": 0.08012704829278856,
      "grad_norm": 11.687822831294243,
      "learning_rate": 4.004329004329004e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.4375,
      "logps/chosen": -346.0,
      "logps/rejected": -568.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.0625,
      "step": 1110
    },
    {
      "epoch": 0.0808489135927236,
      "grad_norm": 16.193115895860167,
      "learning_rate": 4.04040404040404e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -356.0,
      "logps/rejected": -532.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -3.65625,
      "step": 1120
    },
    {
      "epoch": 0.08157077889265862,
      "grad_norm": 10.454280347519187,
      "learning_rate": 4.0764790764790763e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -350.0,
      "logps/rejected": -496.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -3.296875,
      "step": 1130
    },
    {
      "epoch": 0.08229264419259366,
      "grad_norm": 10.033847939954763,
      "learning_rate": 4.1125541125541127e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -334.0,
      "logps/rejected": -462.0,
      "loss": 0.355,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -3.1875,
      "step": 1140
    },
    {
      "epoch": 0.0830145094925287,
      "grad_norm": 13.538065077526934,
      "learning_rate": 4.1486291486291485e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -398.0,
      "logps/rejected": -588.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.28125,
      "step": 1150
    },
    {
      "epoch": 0.08373637479246372,
      "grad_norm": 12.06165274825125,
      "learning_rate": 4.184704184704185e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -314.0,
      "logps/rejected": -500.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -3.4375,
      "step": 1160
    },
    {
      "epoch": 0.08445824009239876,
      "grad_norm": 14.245004620424362,
      "learning_rate": 4.22077922077922e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -376.0,
      "logps/rejected": -612.0,
      "loss": 0.3599,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.21875,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.59375,
      "step": 1170
    },
    {
      "epoch": 0.0851801053923338,
      "grad_norm": 12.93467746542696,
      "learning_rate": 4.2568542568542565e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -330.0,
      "logps/rejected": -476.0,
      "loss": 0.3635,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -3.34375,
      "step": 1180
    },
    {
      "epoch": 0.08590197069226882,
      "grad_norm": 11.202101570401776,
      "learning_rate": 4.292929292929293e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.546875,
      "logps/chosen": -364.0,
      "logps/rejected": -536.0,
      "loss": 0.3583,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -3.953125,
      "step": 1190
    },
    {
      "epoch": 0.08662383599220386,
      "grad_norm": 10.771994466129188,
      "learning_rate": 4.3290043290043287e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -358.0,
      "logps/rejected": -510.0,
      "loss": 0.3465,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -3.46875,
      "step": 1200
    },
    {
      "epoch": 0.08734570129213888,
      "grad_norm": 14.841268368598595,
      "learning_rate": 4.365079365079365e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -364.0,
      "logps/rejected": -584.0,
      "loss": 0.3477,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.40625,
      "step": 1210
    },
    {
      "epoch": 0.08806756659207392,
      "grad_norm": 9.504108427187644,
      "learning_rate": 4.401154401154401e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -394.0,
      "logps/rejected": -648.0,
      "loss": 0.3755,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.9375,
      "step": 1220
    },
    {
      "epoch": 0.08878943189200895,
      "grad_norm": 12.115514229718103,
      "learning_rate": 4.437229437229437e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -304.0,
      "logps/rejected": -520.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -3.71875,
      "step": 1230
    },
    {
      "epoch": 0.08951129719194398,
      "grad_norm": 10.317228267259349,
      "learning_rate": 4.4733044733044736e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -420.0,
      "logps/rejected": -612.0,
      "loss": 0.3559,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -4.46875,
      "step": 1240
    },
    {
      "epoch": 0.09023316249187902,
      "grad_norm": 11.197522859178363,
      "learning_rate": 4.509379509379509e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -428.0,
      "logps/rejected": -604.0,
      "loss": 0.3219,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -4.59375,
      "step": 1250
    },
    {
      "epoch": 0.09095502779181405,
      "grad_norm": 13.497331724381818,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -412.0,
      "logps/rejected": -604.0,
      "loss": 0.3388,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -4.53125,
      "step": 1260
    },
    {
      "epoch": 0.09167689309174908,
      "grad_norm": 11.433124284340494,
      "learning_rate": 4.581529581529581e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -390.0,
      "logps/rejected": -560.0,
      "loss": 0.3458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -4.0,
      "step": 1270
    },
    {
      "epoch": 0.09239875839168411,
      "grad_norm": 12.324384516756012,
      "learning_rate": 4.6176046176046174e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -410.0,
      "logps/rejected": -580.0,
      "loss": 0.3426,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -4.21875,
      "step": 1280
    },
    {
      "epoch": 0.09312062369161915,
      "grad_norm": 10.730987022263216,
      "learning_rate": 4.6536796536796537e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -442.0,
      "logps/rejected": -612.0,
      "loss": 0.3593,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -4.625,
      "step": 1290
    },
    {
      "epoch": 0.09384248899155417,
      "grad_norm": 10.132938191746307,
      "learning_rate": 4.6897546897546896e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -408.0,
      "logps/rejected": -564.0,
      "loss": 0.3632,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.09375,
      "step": 1300
    },
    {
      "epoch": 0.09456435429148921,
      "grad_norm": 11.988621181927288,
      "learning_rate": 4.725829725829726e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -452.0,
      "logps/rejected": -624.0,
      "loss": 0.3609,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.71875,
      "step": 1310
    },
    {
      "epoch": 0.09528621959142423,
      "grad_norm": 8.802084815882347,
      "learning_rate": 4.761904761904761e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -458.0,
      "logps/rejected": -604.0,
      "loss": 0.3197,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.53125,
      "step": 1320
    },
    {
      "epoch": 0.09600808489135927,
      "grad_norm": 13.978035740824382,
      "learning_rate": 4.797979797979798e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -420.0,
      "logps/rejected": -660.0,
      "loss": 0.355,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.15625,
      "step": 1330
    },
    {
      "epoch": 0.09672995019129431,
      "grad_norm": 9.329953252240916,
      "learning_rate": 4.834054834054833e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -426.0,
      "logps/rejected": -612.0,
      "loss": 0.3382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -4.53125,
      "step": 1340
    },
    {
      "epoch": 0.09745181549122933,
      "grad_norm": 14.354032725201476,
      "learning_rate": 4.87012987012987e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -438.0,
      "logps/rejected": -628.0,
      "loss": 0.3375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.15625,
      "rewards/rejected": -4.6875,
      "step": 1350
    },
    {
      "epoch": 0.09817368079116437,
      "grad_norm": 11.184165035806638,
      "learning_rate": 4.906204906204906e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -444.0,
      "logps/rejected": -640.0,
      "loss": 0.3528,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.015625,
      "rewards/rejected": -4.6875,
      "step": 1360
    },
    {
      "epoch": 0.0988955460910994,
      "grad_norm": 9.32730305340813,
      "learning_rate": 4.942279942279942e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -420.0,
      "logps/rejected": -592.0,
      "loss": 0.3343,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.9375,
      "rewards/rejected": -4.53125,
      "step": 1370
    },
    {
      "epoch": 0.09961741139103443,
      "grad_norm": 17.291834755334516,
      "learning_rate": 4.978354978354979e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -450.0,
      "logps/rejected": -636.0,
      "loss": 0.3246,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -4.6875,
      "step": 1380
    },
    {
      "epoch": 0.10033927669096947,
      "grad_norm": 10.743034154219572,
      "learning_rate": 4.999998729993963e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -430.0,
      "logps/rejected": -604.0,
      "loss": 0.3335,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -4.59375,
      "step": 1390
    },
    {
      "epoch": 0.10106114199090449,
      "grad_norm": 17.190340388139088,
      "learning_rate": 4.999984442440868e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.703125,
      "logps/chosen": -438.0,
      "logps/rejected": -680.0,
      "loss": 0.3204,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.984375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.4375,
      "step": 1400
    },
    {
      "epoch": 0.10178300729083953,
      "grad_norm": 9.641498367561857,
      "learning_rate": 4.99995427991816e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -442.0,
      "logps/rejected": -648.0,
      "loss": 0.3379,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.875,
      "step": 1410
    },
    {
      "epoch": 0.10250487259077457,
      "grad_norm": 9.768069678095703,
      "learning_rate": 4.999908242617371e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -452.0,
      "logps/rejected": -648.0,
      "loss": 0.34,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -4.59375,
      "step": 1420
    },
    {
      "epoch": 0.10322673789070959,
      "grad_norm": 13.37422488078675,
      "learning_rate": 4.99984633083084e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -416.0,
      "logps/rejected": -616.0,
      "loss": 0.331,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -4.5625,
      "step": 1430
    },
    {
      "epoch": 0.10394860319064463,
      "grad_norm": 11.253293027609532,
      "learning_rate": 4.99976854495171e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -488.0,
      "logps/rejected": -696.0,
      "loss": 0.3218,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 2.09375,
      "rewards/rejected": -5.53125,
      "step": 1440
    },
    {
      "epoch": 0.10467046849057966,
      "grad_norm": 10.82613372416563,
      "learning_rate": 4.999674885473922e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -458.0,
      "logps/rejected": -656.0,
      "loss": 0.3213,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.65625,
      "step": 1450
    },
    {
      "epoch": 0.10539233379051469,
      "grad_norm": 15.794446739298891,
      "learning_rate": 4.999565352992216e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -502.0,
      "logps/rejected": -712.0,
      "loss": 0.3289,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.296875,
      "rewards/rejected": -5.5,
      "step": 1460
    },
    {
      "epoch": 0.10611419909044972,
      "grad_norm": 10.361664841175285,
      "learning_rate": 4.999439948202127e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -450.0,
      "logps/rejected": -596.0,
      "loss": 0.3261,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -4.71875,
      "step": 1470
    },
    {
      "epoch": 0.10683606439038476,
      "grad_norm": 10.555951221100015,
      "learning_rate": 4.999298671899977e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -644.0,
      "loss": 0.3399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.9375,
      "step": 1480
    },
    {
      "epoch": 0.10755792969031978,
      "grad_norm": 9.802204192641314,
      "learning_rate": 4.999141524982877e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -444.0,
      "logps/rejected": -700.0,
      "loss": 0.309,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.53125,
      "step": 1490
    },
    {
      "epoch": 0.10827979499025482,
      "grad_norm": 12.72303480502336,
      "learning_rate": 4.998968508448714e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -432.0,
      "logps/rejected": -632.0,
      "loss": 0.3276,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.15625,
      "rewards/rejected": -4.78125,
      "step": 1500
    },
    {
      "epoch": 0.10900166029018984,
      "grad_norm": 14.570251314538908,
      "learning_rate": 4.998779623396146e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -502.0,
      "logps/rejected": -720.0,
      "loss": 0.2995,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.65625,
      "step": 1510
    },
    {
      "epoch": 0.10972352559012488,
      "grad_norm": 11.169758259343533,
      "learning_rate": 4.9985748710246e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -476.0,
      "logps/rejected": -656.0,
      "loss": 0.3344,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.0625,
      "step": 1520
    },
    {
      "epoch": 0.11044539089005992,
      "grad_norm": 10.504554948014457,
      "learning_rate": 4.998354252634257e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -486.0,
      "logps/rejected": -668.0,
      "loss": 0.2853,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.078125,
      "rewards/rejected": -5.28125,
      "step": 1530
    },
    {
      "epoch": 0.11116725618999494,
      "grad_norm": 15.073212833972551,
      "learning_rate": 4.998117769626051e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -456.0,
      "logps/rejected": -676.0,
      "loss": 0.309,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 2.25,
      "rewards/rejected": -5.21875,
      "step": 1540
    },
    {
      "epoch": 0.11188912148992998,
      "grad_norm": 10.298569511142073,
      "learning_rate": 4.997865423501657e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -380.0,
      "logps/rejected": -564.0,
      "loss": 0.3261,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -4.21875,
      "step": 1550
    },
    {
      "epoch": 0.11261098678986502,
      "grad_norm": 11.022262328486809,
      "learning_rate": 4.997597215863477e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -716.0,
      "loss": 0.2878,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.6875,
      "step": 1560
    },
    {
      "epoch": 0.11333285208980004,
      "grad_norm": 12.425423187064853,
      "learning_rate": 4.99731314841464e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -500.0,
      "logps/rejected": -704.0,
      "loss": 0.3008,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -5.5625,
      "step": 1570
    },
    {
      "epoch": 0.11405471738973508,
      "grad_norm": 10.996930809431353,
      "learning_rate": 4.997013222958978e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -496.0,
      "logps/rejected": -732.0,
      "loss": 0.3105,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 2.578125,
      "rewards/rejected": -5.8125,
      "step": 1580
    },
    {
      "epoch": 0.1147765826896701,
      "grad_norm": 11.11449736666811,
      "learning_rate": 4.99669744140103e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -512.0,
      "logps/rejected": -716.0,
      "loss": 0.3043,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 2.09375,
      "rewards/rejected": -5.84375,
      "step": 1590
    },
    {
      "epoch": 0.11549844798960514,
      "grad_norm": 10.155961204890554,
      "learning_rate": 4.996365805746015e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -474.0,
      "logps/rejected": -752.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.03125,
      "step": 1600
    },
    {
      "epoch": 0.11622031328954018,
      "grad_norm": 9.079844890608808,
      "learning_rate": 4.996018318099829e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.28125,
      "logps/chosen": -506.0,
      "logps/rejected": -712.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -5.65625,
      "step": 1610
    },
    {
      "epoch": 0.1169421785894752,
      "grad_norm": 13.337729397169802,
      "learning_rate": 4.995654980669028e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -476.0,
      "logps/rejected": -748.0,
      "loss": 0.2937,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.90625,
      "step": 1620
    },
    {
      "epoch": 0.11766404388941024,
      "grad_norm": 9.987891055540013,
      "learning_rate": 4.995275795760816e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -454.0,
      "logps/rejected": -660.0,
      "loss": 0.2991,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.109375,
      "rewards/rejected": -5.0625,
      "step": 1630
    },
    {
      "epoch": 0.11838590918934527,
      "grad_norm": 12.804317711855225,
      "learning_rate": 4.994880765783026e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -484.0,
      "logps/rejected": -672.0,
      "loss": 0.2883,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.125,
      "step": 1640
    },
    {
      "epoch": 0.1191077744892803,
      "grad_norm": 11.589955875632223,
      "learning_rate": 4.99446989324411e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -470.0,
      "logps/rejected": -756.0,
      "loss": 0.2847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.0,
      "step": 1650
    },
    {
      "epoch": 0.11982963978921533,
      "grad_norm": 14.992092750756044,
      "learning_rate": 4.99404318075312e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -672.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.125,
      "rewards/rejected": -5.34375,
      "step": 1660
    },
    {
      "epoch": 0.12055150508915037,
      "grad_norm": 14.63652475825442,
      "learning_rate": 4.993600631019689e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -484.0,
      "logps/rejected": -680.0,
      "loss": 0.3115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.984375,
      "rewards/rejected": -5.3125,
      "step": 1670
    },
    {
      "epoch": 0.1212733703890854,
      "grad_norm": 10.01896442134856,
      "learning_rate": 4.993142246854024e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -434.0,
      "logps/rejected": -656.0,
      "loss": 0.2917,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.15625,
      "rewards/rejected": -5.0,
      "step": 1680
    },
    {
      "epoch": 0.12199523568902043,
      "grad_norm": 10.480137264657502,
      "learning_rate": 4.992668031166876e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.703125,
      "logps/chosen": -444.0,
      "logps/rejected": -672.0,
      "loss": 0.3114,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -5.34375,
      "step": 1690
    },
    {
      "epoch": 0.12271710098895545,
      "grad_norm": 9.543166172183264,
      "learning_rate": 4.992177986969526e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -430.0,
      "logps/rejected": -624.0,
      "loss": 0.3235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.890625,
      "rewards/rejected": -4.59375,
      "step": 1700
    },
    {
      "epoch": 0.12343896628889049,
      "grad_norm": 9.39264085167921,
      "learning_rate": 4.991672117373769e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -712.0,
      "loss": 0.2971,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.625,
      "rewards/rejected": -5.4375,
      "step": 1710
    },
    {
      "epoch": 0.12416083158882553,
      "grad_norm": 9.35509206624716,
      "learning_rate": 4.991150425591891e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -448.0,
      "logps/rejected": -700.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.53125,
      "step": 1720
    },
    {
      "epoch": 0.12488269688876055,
      "grad_norm": 9.262201280713281,
      "learning_rate": 4.99061291493665e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -444.0,
      "logps/rejected": -676.0,
      "loss": 0.2925,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.3125,
      "step": 1730
    },
    {
      "epoch": 0.1256045621886956,
      "grad_norm": 10.008148122591312,
      "learning_rate": 4.99005958882125e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -450.0,
      "logps/rejected": -684.0,
      "loss": 0.3133,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -5.3125,
      "step": 1740
    },
    {
      "epoch": 0.12632642748863063,
      "grad_norm": 8.554136981806407,
      "learning_rate": 4.989490450759331e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -482.0,
      "logps/rejected": -696.0,
      "loss": 0.2649,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.3125,
      "rewards/rejected": -5.4375,
      "step": 1750
    },
    {
      "epoch": 0.12704829278856566,
      "grad_norm": 8.929491173417231,
      "learning_rate": 4.988905504364933e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -768.0,
      "loss": 0.2715,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.09375,
      "step": 1760
    },
    {
      "epoch": 0.12777015808850067,
      "grad_norm": 19.85583075822656,
      "learning_rate": 4.988304753352482e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -454.0,
      "logps/rejected": -744.0,
      "loss": 0.2838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.046875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.875,
      "step": 1770
    },
    {
      "epoch": 0.1284920233884357,
      "grad_norm": 11.573857511888404,
      "learning_rate": 4.987688201536764e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -404.0,
      "logps/rejected": -636.0,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -4.84375,
      "step": 1780
    },
    {
      "epoch": 0.12921388868837075,
      "grad_norm": 10.767926076980798,
      "learning_rate": 4.987055852832899e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -504.0,
      "logps/rejected": -784.0,
      "loss": 0.2768,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -6.15625,
      "step": 1790
    },
    {
      "epoch": 0.12993575398830579,
      "grad_norm": 9.480524665881354,
      "learning_rate": 4.986407711256319e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -434.0,
      "logps/rejected": -656.0,
      "loss": 0.2996,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.203125,
      "rewards/rejected": -5.0,
      "step": 1800
    },
    {
      "epoch": 0.13065761928824082,
      "grad_norm": 11.059595953397832,
      "learning_rate": 4.98574378092274e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.6875,
      "logps/chosen": -506.0,
      "logps/rejected": -760.0,
      "loss": 0.2706,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.21875,
      "step": 1810
    },
    {
      "epoch": 0.13137948458817586,
      "grad_norm": 12.559652259820835,
      "learning_rate": 4.985064066048139e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.40625,
      "logps/chosen": -424.0,
      "logps/rejected": -660.0,
      "loss": 0.3016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.375,
      "rewards/rejected": -5.0,
      "step": 1820
    },
    {
      "epoch": 0.13210134988811087,
      "grad_norm": 16.780300866871737,
      "learning_rate": 4.984368570948724e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -444.0,
      "logps/rejected": -704.0,
      "loss": 0.2943,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.34375,
      "step": 1830
    },
    {
      "epoch": 0.1328232151880459,
      "grad_norm": 10.65248837076819,
      "learning_rate": 4.983657300040907e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -472.0,
      "logps/rejected": -704.0,
      "loss": 0.2699,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 2.484375,
      "rewards/rejected": -5.75,
      "step": 1840
    },
    {
      "epoch": 0.13354508048798094,
      "grad_norm": 10.948003648298677,
      "learning_rate": 4.982930257841278e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.453125,
      "logps/chosen": -456.0,
      "logps/rejected": -728.0,
      "loss": 0.2941,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.75,
      "step": 1850
    },
    {
      "epoch": 0.13426694578791598,
      "grad_norm": 12.047287233162004,
      "learning_rate": 4.982187448966575e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -768.0,
      "loss": 0.2898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.640625,
      "rewards/margins": 2.390625,
      "rewards/rejected": -6.03125,
      "step": 1860
    },
    {
      "epoch": 0.13498881108785102,
      "grad_norm": 14.9941388186048,
      "learning_rate": 4.981428878133656e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.4375,
      "logps/chosen": -468.0,
      "logps/rejected": -772.0,
      "loss": 0.2811,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.1875,
      "step": 1870
    },
    {
      "epoch": 0.13571067638778603,
      "grad_norm": 9.951835424657778,
      "learning_rate": 4.980654550159463e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -492.0,
      "logps/rejected": -708.0,
      "loss": 0.2732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.1875,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.6875,
      "step": 1880
    },
    {
      "epoch": 0.13643254168772107,
      "grad_norm": 8.861361984310934,
      "learning_rate": 4.979864469961004e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -716.0,
      "loss": 0.2762,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -5.75,
      "step": 1890
    },
    {
      "epoch": 0.1371544069876561,
      "grad_norm": 9.692735634395657,
      "learning_rate": 4.979058642555307e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -452.0,
      "logps/rejected": -676.0,
      "loss": 0.2931,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.3125,
      "step": 1900
    },
    {
      "epoch": 0.13787627228759114,
      "grad_norm": 9.565594633496087,
      "learning_rate": 4.978237073059399e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -676.0,
      "loss": 0.2898,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 2.171875,
      "rewards/rejected": -5.25,
      "step": 1910
    },
    {
      "epoch": 0.13859813758752618,
      "grad_norm": 11.007909649107406,
      "learning_rate": 4.977399766690269e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -498.0,
      "logps/rejected": -796.0,
      "loss": 0.2902,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.3125,
      "step": 1920
    },
    {
      "epoch": 0.1393200028874612,
      "grad_norm": 9.107456714393756,
      "learning_rate": 4.976546728764836e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -498.0,
      "logps/rejected": -724.0,
      "loss": 0.284,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 2.4375,
      "rewards/rejected": -5.75,
      "step": 1930
    },
    {
      "epoch": 0.14004186818739622,
      "grad_norm": 8.927298343498093,
      "learning_rate": 4.975677964699912e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -502.0,
      "logps/rejected": -776.0,
      "loss": 0.2524,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.34375,
      "step": 1940
    },
    {
      "epoch": 0.14076373348733126,
      "grad_norm": 10.144209347121677,
      "learning_rate": 4.974793480012174e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -716.0,
      "loss": 0.2621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.5625,
      "step": 1950
    },
    {
      "epoch": 0.1414855987872663,
      "grad_norm": 7.533895015502329,
      "learning_rate": 4.973893280318125e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -764.0,
      "loss": 0.2844,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.125,
      "step": 1960
    },
    {
      "epoch": 0.14220746408720134,
      "grad_norm": 13.017212573238576,
      "learning_rate": 4.972977371334056e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -424.0,
      "logps/rejected": -672.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.96875,
      "step": 1970
    },
    {
      "epoch": 0.14292932938713637,
      "grad_norm": 8.958119059657708,
      "learning_rate": 4.972045758876014e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -488.0,
      "logps/rejected": -724.0,
      "loss": 0.2477,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.359375,
      "rewards/rejected": -5.625,
      "step": 1980
    },
    {
      "epoch": 0.14365119468707138,
      "grad_norm": 10.493203877125167,
      "learning_rate": 4.971098448859766e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -488.0,
      "logps/rejected": -692.0,
      "loss": 0.2933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.234375,
      "rewards/rejected": -5.5,
      "step": 1990
    },
    {
      "epoch": 0.14437305998700642,
      "grad_norm": 9.973862051012356,
      "learning_rate": 4.970135447300752e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.3125,
      "logps/chosen": -516.0,
      "logps/rejected": -788.0,
      "loss": 0.2595,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.0625,
      "step": 2000
    },
    {
      "epoch": 0.14509492528694146,
      "grad_norm": 13.28264552294119,
      "learning_rate": 4.969156760314064e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -524.0,
      "logps/rejected": -796.0,
      "loss": 0.2904,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.34375,
      "step": 2010
    },
    {
      "epoch": 0.1458167905868765,
      "grad_norm": 8.872169286182675,
      "learning_rate": 4.968162394114387e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -488.0,
      "logps/rejected": -744.0,
      "loss": 0.2523,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.96875,
      "step": 2020
    },
    {
      "epoch": 0.14653865588681153,
      "grad_norm": 12.757195149734072,
      "learning_rate": 4.967152355015974e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -478.0,
      "logps/rejected": -724.0,
      "loss": 0.2783,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.9375,
      "step": 2030
    },
    {
      "epoch": 0.14726052118674654,
      "grad_norm": 12.796953074989913,
      "learning_rate": 4.966126649432603e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -496.0,
      "logps/rejected": -780.0,
      "loss": 0.2723,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -6.125,
      "step": 2040
    },
    {
      "epoch": 0.14798238648668158,
      "grad_norm": 19.108616982264344,
      "learning_rate": 4.96508528387753e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.828125,
      "logps/chosen": -540.0,
      "logps/rejected": -812.0,
      "loss": 0.2763,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.96875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -6.6875,
      "step": 2050
    },
    {
      "epoch": 0.14870425178661661,
      "grad_norm": 9.380783201026548,
      "learning_rate": 4.964028264963456e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -516.0,
      "logps/rejected": -800.0,
      "loss": 0.2822,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -6.1875,
      "step": 2060
    },
    {
      "epoch": 0.14942611708655165,
      "grad_norm": 10.81595001993398,
      "learning_rate": 4.962955599402481e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -784.0,
      "loss": 0.266,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.25,
      "rewards/rejected": -6.25,
      "step": 2070
    },
    {
      "epoch": 0.1501479823864867,
      "grad_norm": 11.456291625001526,
      "learning_rate": 4.961867294006059e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.796875,
      "logps/chosen": -608.0,
      "logps/rejected": -836.0,
      "loss": 0.3064,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 2.40625,
      "rewards/rejected": -6.875,
      "step": 2080
    },
    {
      "epoch": 0.15086984768642173,
      "grad_norm": 7.960182686516041,
      "learning_rate": 4.96076335568496e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -516.0,
      "logps/rejected": -776.0,
      "loss": 0.2467,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.21875,
      "step": 2090
    },
    {
      "epoch": 0.15159171298635674,
      "grad_norm": 10.25400623338689,
      "learning_rate": 4.959643791449222e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -564.0,
      "logps/rejected": -788.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.375,
      "step": 2100
    },
    {
      "epoch": 0.15231357828629177,
      "grad_norm": 10.09340211565806,
      "learning_rate": 4.958508608408109e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -544.0,
      "logps/rejected": -736.0,
      "loss": 0.2829,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.921875,
      "rewards/margins": 2.03125,
      "rewards/rejected": -5.9375,
      "step": 2110
    },
    {
      "epoch": 0.1530354435862268,
      "grad_norm": 8.537784621679538,
      "learning_rate": 4.957357813770064e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -596.0,
      "logps/rejected": -860.0,
      "loss": 0.2479,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.28125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -7.03125,
      "step": 2120
    },
    {
      "epoch": 0.15375730888616185,
      "grad_norm": 8.985222338453365,
      "learning_rate": 4.956191414842665e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.71875,
      "logps/chosen": -572.0,
      "logps/rejected": -860.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.03125,
      "step": 2130
    },
    {
      "epoch": 0.15447917418609688,
      "grad_norm": 10.993021927824719,
      "learning_rate": 4.955009419032575e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -840.0,
      "loss": 0.2681,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.8125,
      "rewards/rejected": -6.8125,
      "step": 2140
    },
    {
      "epoch": 0.1552010394860319,
      "grad_norm": 11.792550816411442,
      "learning_rate": 4.953811833845503e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -528.0,
      "logps/rejected": -756.0,
      "loss": 0.265,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 2.328125,
      "rewards/rejected": -6.0625,
      "step": 2150
    },
    {
      "epoch": 0.15592290478596693,
      "grad_norm": 13.163433586155344,
      "learning_rate": 4.952598666886145e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -856.0,
      "loss": 0.2516,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.3125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.0,
      "step": 2160
    },
    {
      "epoch": 0.15664477008590197,
      "grad_norm": 15.36607001210862,
      "learning_rate": 4.951369925858147e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -792.0,
      "loss": 0.2598,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.5,
      "step": 2170
    },
    {
      "epoch": 0.157366635385837,
      "grad_norm": 8.701540096554936,
      "learning_rate": 4.950125618564046e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.765625,
      "logps/chosen": -528.0,
      "logps/rejected": -784.0,
      "loss": 0.251,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.625,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.1875,
      "step": 2180
    },
    {
      "epoch": 0.15808850068577204,
      "grad_norm": 10.616304811087307,
      "learning_rate": 4.948865752905228e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.703125,
      "logps/chosen": -576.0,
      "logps/rejected": -828.0,
      "loss": 0.2533,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.65625,
      "rewards/rejected": -6.75,
      "step": 2190
    },
    {
      "epoch": 0.15881036598570708,
      "grad_norm": 10.818629515177625,
      "learning_rate": 4.947590336881874e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -584.0,
      "logps/rejected": -816.0,
      "loss": 0.2593,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.625,
      "rewards/rejected": -6.6875,
      "step": 2200
    },
    {
      "epoch": 0.1595322312856421,
      "grad_norm": 10.595421594796772,
      "learning_rate": 4.946299378592912e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -532.0,
      "logps/rejected": -784.0,
      "loss": 0.2795,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 2.546875,
      "rewards/rejected": -6.125,
      "step": 2210
    },
    {
      "epoch": 0.16025409658557713,
      "grad_norm": 13.24386115549115,
      "learning_rate": 4.944992886235961e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -828.0,
      "loss": 0.2521,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.96875,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.71875,
      "step": 2220
    },
    {
      "epoch": 0.16097596188551216,
      "grad_norm": 8.619302388111215,
      "learning_rate": 4.943670868107284e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -832.0,
      "loss": 0.27,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.71875,
      "step": 2230
    },
    {
      "epoch": 0.1616978271854472,
      "grad_norm": 8.272453046055942,
      "learning_rate": 4.942333332601731e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -564.0,
      "logps/rejected": -864.0,
      "loss": 0.2661,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.984375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -7.15625,
      "step": 2240
    },
    {
      "epoch": 0.16241969248538224,
      "grad_norm": 6.171166558818759,
      "learning_rate": 4.940980288212691e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.375,
      "logps/chosen": -576.0,
      "logps/rejected": -800.0,
      "loss": 0.2574,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.03125,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.46875,
      "step": 2250
    },
    {
      "epoch": 0.16314155778531725,
      "grad_norm": 9.42178572046336,
      "learning_rate": 4.939611743532031e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -828.0,
      "loss": 0.2385,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.9375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.78125,
      "step": 2260
    },
    {
      "epoch": 0.16386342308525229,
      "grad_norm": 8.438515261081413,
      "learning_rate": 4.93822770725005e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -544.0,
      "logps/rejected": -776.0,
      "loss": 0.2737,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.734375,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.34375,
      "step": 2270
    },
    {
      "epoch": 0.16458528838518732,
      "grad_norm": 11.043392051990788,
      "learning_rate": 4.936828188155413e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -520.0,
      "logps/rejected": -776.0,
      "loss": 0.2496,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 2.640625,
      "rewards/rejected": -6.28125,
      "step": 2280
    },
    {
      "epoch": 0.16530715368512236,
      "grad_norm": 10.831257780639715,
      "learning_rate": 4.935413195135106e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -572.0,
      "logps/rejected": -872.0,
      "loss": 0.2479,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.15625,
      "step": 2290
    },
    {
      "epoch": 0.1660290189850574,
      "grad_norm": 9.747157511340994,
      "learning_rate": 4.933982737174374e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -852.0,
      "loss": 0.2701,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.875,
      "step": 2300
    },
    {
      "epoch": 0.1667508842849924,
      "grad_norm": 9.870871070277088,
      "learning_rate": 4.932536823356664e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -620.0,
      "logps/rejected": -880.0,
      "loss": 0.2543,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.578125,
      "rewards/rejected": -7.28125,
      "step": 2310
    },
    {
      "epoch": 0.16747274958492744,
      "grad_norm": 8.8575895636107,
      "learning_rate": 4.931075462863567e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -604.0,
      "logps/rejected": -912.0,
      "loss": 0.2594,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.4375,
      "step": 2320
    },
    {
      "epoch": 0.16819461488486248,
      "grad_norm": 7.816395810688846,
      "learning_rate": 4.929598664974762e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -652.0,
      "logps/rejected": -940.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.90625,
      "step": 2330
    },
    {
      "epoch": 0.16891648018479752,
      "grad_norm": 8.344064880799563,
      "learning_rate": 4.928106439067958e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -816.0,
      "loss": 0.2669,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.53125,
      "rewards/rejected": -6.625,
      "step": 2340
    },
    {
      "epoch": 0.16963834548473256,
      "grad_norm": 10.994252200422187,
      "learning_rate": 4.926598794618829e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -552.0,
      "logps/rejected": -784.0,
      "loss": 0.278,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.921875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.46875,
      "step": 2350
    },
    {
      "epoch": 0.1703602107846676,
      "grad_norm": 15.14815293816792,
      "learning_rate": 4.92507574120096e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -604.0,
      "logps/rejected": -852.0,
      "loss": 0.2777,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.609375,
      "rewards/rejected": -7.0,
      "step": 2360
    },
    {
      "epoch": 0.1710820760846026,
      "grad_norm": 7.821370479620557,
      "learning_rate": 4.923537288485779e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -812.0,
      "loss": 0.2638,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -6.6875,
      "step": 2370
    },
    {
      "epoch": 0.17180394138453764,
      "grad_norm": 9.211826498394792,
      "learning_rate": 4.921983446242506e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -644.0,
      "logps/rejected": -944.0,
      "loss": 0.2493,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -7.875,
      "step": 2380
    },
    {
      "epoch": 0.17252580668447268,
      "grad_norm": 8.298256232187494,
      "learning_rate": 4.92041422433808e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -548.0,
      "logps/rejected": -924.0,
      "loss": 0.2348,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.921875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.65625,
      "step": 2390
    },
    {
      "epoch": 0.17324767198440771,
      "grad_norm": 10.983625039262604,
      "learning_rate": 4.918829632737103e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -560.0,
      "logps/rejected": -896.0,
      "loss": 0.2633,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.125,
      "rewards/margins": 3.375,
      "rewards/rejected": -7.5,
      "step": 2400
    },
    {
      "epoch": 0.17396953728434275,
      "grad_norm": 12.61039915689691,
      "learning_rate": 4.917229681501774e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -580.0,
      "logps/rejected": -824.0,
      "loss": 0.264,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.6875,
      "step": 2410
    },
    {
      "epoch": 0.17469140258427776,
      "grad_norm": 11.643682996836732,
      "learning_rate": 4.91561438079183e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.34375,
      "logps/chosen": -628.0,
      "logps/rejected": -900.0,
      "loss": 0.2668,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.3125,
      "step": 2420
    },
    {
      "epoch": 0.1754132678842128,
      "grad_norm": 13.088952482660957,
      "learning_rate": 4.913983740864473e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -632.0,
      "logps/rejected": -920.0,
      "loss": 0.2359,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -7.4375,
      "step": 2430
    },
    {
      "epoch": 0.17613513318414784,
      "grad_norm": 10.188119267424455,
      "learning_rate": 4.91233777207431e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -592.0,
      "logps/rejected": -916.0,
      "loss": 0.2748,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -7.6875,
      "step": 2440
    },
    {
      "epoch": 0.17685699848408287,
      "grad_norm": 8.609826083562075,
      "learning_rate": 4.910676484873292e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.390625,
      "logps/chosen": -612.0,
      "logps/rejected": -864.0,
      "loss": 0.2658,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.4375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -7.09375,
      "step": 2450
    },
    {
      "epoch": 0.1775788637840179,
      "grad_norm": 10.48253643988837,
      "learning_rate": 4.908999889810633e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -572.0,
      "logps/rejected": -876.0,
      "loss": 0.2331,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.15625,
      "step": 2460
    },
    {
      "epoch": 0.17830072908395295,
      "grad_norm": 17.344352556263694,
      "learning_rate": 4.907307997532761e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -632.0,
      "logps/rejected": -984.0,
      "loss": 0.2354,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -8.125,
      "step": 2470
    },
    {
      "epoch": 0.17902259438388796,
      "grad_norm": 9.982854645114383,
      "learning_rate": 4.905600818783237e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -664.0,
      "logps/rejected": -980.0,
      "loss": 0.3273,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.25,
      "step": 2480
    },
    {
      "epoch": 0.179744459683823,
      "grad_norm": 10.68094022282627,
      "learning_rate": 4.903878364402691e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -836.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.953125,
      "rewards/margins": 2.953125,
      "rewards/rejected": -6.90625,
      "step": 2490
    },
    {
      "epoch": 0.18046632498375803,
      "grad_norm": 8.296383042780498,
      "learning_rate": 4.902140645328759e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -848.0,
      "loss": 0.2599,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -7.125,
      "step": 2500
    },
    {
      "epoch": 0.18118819028369307,
      "grad_norm": 9.103787824302069,
      "learning_rate": 4.900387672596003e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -528.0,
      "logps/rejected": -816.0,
      "loss": 0.2318,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.6875,
      "step": 2510
    },
    {
      "epoch": 0.1819100555836281,
      "grad_norm": 16.67988360600736,
      "learning_rate": 4.898619457335848e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -568.0,
      "logps/rejected": -844.0,
      "loss": 0.248,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.921875,
      "rewards/rejected": -6.9375,
      "step": 2520
    },
    {
      "epoch": 0.18263192088356311,
      "grad_norm": 10.469848545031832,
      "learning_rate": 4.896836010776509e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -644.0,
      "logps/rejected": -1012.0,
      "loss": 0.2595,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.78125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.375,
      "step": 2530
    },
    {
      "epoch": 0.18335378618349815,
      "grad_norm": 9.661637065408652,
      "learning_rate": 4.895037344242921e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.296875,
      "logps/chosen": -580.0,
      "logps/rejected": -856.0,
      "loss": 0.2585,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.125,
      "rewards/margins": 2.953125,
      "rewards/rejected": -7.09375,
      "step": 2540
    },
    {
      "epoch": 0.1840756514834332,
      "grad_norm": 8.853616581850865,
      "learning_rate": 4.893223469156664e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -584.0,
      "logps/rejected": -856.0,
      "loss": 0.249,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -6.9375,
      "step": 2550
    },
    {
      "epoch": 0.18479751678336823,
      "grad_norm": 14.23122096213618,
      "learning_rate": 4.891394397035896e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -588.0,
      "logps/rejected": -848.0,
      "loss": 0.2458,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.25,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.0,
      "step": 2560
    },
    {
      "epoch": 0.18551938208330326,
      "grad_norm": 7.165047283079565,
      "learning_rate": 4.889550139495275e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -872.0,
      "loss": 0.2193,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.90625,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.15625,
      "step": 2570
    },
    {
      "epoch": 0.1862412473832383,
      "grad_norm": 9.892022723104382,
      "learning_rate": 4.887690708245887e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -860.0,
      "loss": 0.2276,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.765625,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.875,
      "step": 2580
    },
    {
      "epoch": 0.1869631126831733,
      "grad_norm": 9.543867068973059,
      "learning_rate": 4.885816115095171e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -928.0,
      "loss": 0.2391,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -7.5625,
      "step": 2590
    },
    {
      "epoch": 0.18768497798310835,
      "grad_norm": 10.889738728754864,
      "learning_rate": 4.883926371946843e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -608.0,
      "logps/rejected": -872.0,
      "loss": 0.2702,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -7.03125,
      "step": 2600
    },
    {
      "epoch": 0.18840684328304338,
      "grad_norm": 6.728294158129367,
      "learning_rate": 4.882021490800826e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -580.0,
      "logps/rejected": -816.0,
      "loss": 0.2438,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.53125,
      "step": 2610
    },
    {
      "epoch": 0.18912870858297842,
      "grad_norm": 9.374612134195704,
      "learning_rate": 4.880101483753167e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.4375,
      "logps/chosen": -560.0,
      "logps/rejected": -896.0,
      "loss": 0.2198,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.34375,
      "step": 2620
    },
    {
      "epoch": 0.18985057388291346,
      "grad_norm": 8.403906697614635,
      "learning_rate": 4.878166362995963e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -588.0,
      "logps/rejected": -900.0,
      "loss": 0.2728,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.21875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -7.25,
      "step": 2630
    },
    {
      "epoch": 0.19057243918284847,
      "grad_norm": 9.739452944604345,
      "learning_rate": 4.876216140817285e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.296875,
      "logps/chosen": -568.0,
      "logps/rejected": -856.0,
      "loss": 0.2399,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.0,
      "step": 2640
    },
    {
      "epoch": 0.1912943044827835,
      "grad_norm": 9.412524327526398,
      "learning_rate": 4.874250829601094e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -672.0,
      "logps/rejected": -928.0,
      "loss": 0.2287,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.8125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -7.59375,
      "step": 2650
    },
    {
      "epoch": 0.19201616978271854,
      "grad_norm": 9.67143825795057,
      "learning_rate": 4.872270441827174e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -668.0,
      "logps/rejected": -960.0,
      "loss": 0.2358,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.0,
      "rewards/rejected": -8.0,
      "step": 2660
    },
    {
      "epoch": 0.19273803508265358,
      "grad_norm": 10.225198499138108,
      "learning_rate": 4.870274990071038e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -568.0,
      "logps/rejected": -868.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.3125,
      "step": 2670
    },
    {
      "epoch": 0.19345990038258862,
      "grad_norm": 8.93422692887298,
      "learning_rate": 4.868264487003862e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -588.0,
      "logps/rejected": -908.0,
      "loss": 0.2518,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.125,
      "rewards/rejected": -7.40625,
      "step": 2680
    },
    {
      "epoch": 0.19418176568252365,
      "grad_norm": 8.108160852039612,
      "learning_rate": 4.866238945392393e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -588.0,
      "logps/rejected": -840.0,
      "loss": 0.2465,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -6.96875,
      "step": 2690
    },
    {
      "epoch": 0.19490363098245866,
      "grad_norm": 8.305890952481619,
      "learning_rate": 4.864198378098877e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -510.0,
      "logps/rejected": -804.0,
      "loss": 0.237,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.53125,
      "step": 2700
    },
    {
      "epoch": 0.1956254962823937,
      "grad_norm": 12.954558843929119,
      "learning_rate": 4.862142798080973e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -848.0,
      "loss": 0.2245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.125,
      "step": 2710
    },
    {
      "epoch": 0.19634736158232874,
      "grad_norm": 6.775612276163364,
      "learning_rate": 4.860072218391669e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -872.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.90625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.21875,
      "step": 2720
    },
    {
      "epoch": 0.19706922688226378,
      "grad_norm": 8.770593576643066,
      "learning_rate": 4.857986652179203e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.4375,
      "logps/chosen": -596.0,
      "logps/rejected": -872.0,
      "loss": 0.2534,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.9375,
      "step": 2730
    },
    {
      "epoch": 0.1977910921821988,
      "grad_norm": 8.729235176349604,
      "learning_rate": 4.855886112686977e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -596.0,
      "logps/rejected": -920.0,
      "loss": 0.2018,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.3125,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.5625,
      "step": 2740
    },
    {
      "epoch": 0.19851295748213382,
      "grad_norm": 10.466383153721448,
      "learning_rate": 4.853770613253476e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -884.0,
      "loss": 0.2365,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.15625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -7.53125,
      "step": 2750
    },
    {
      "epoch": 0.19923482278206886,
      "grad_norm": 10.738593768361735,
      "learning_rate": 4.851640167312178e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -568.0,
      "logps/rejected": -868.0,
      "loss": 0.224,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.0,
      "step": 2760
    },
    {
      "epoch": 0.1999566880820039,
      "grad_norm": 9.416300695572696,
      "learning_rate": 4.849494788391473e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.4375,
      "logps/chosen": -664.0,
      "logps/rejected": -1032.0,
      "loss": 0.2284,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.8125,
      "step": 2770
    },
    {
      "epoch": 0.20067855338193893,
      "grad_norm": 9.528669823636646,
      "learning_rate": 4.847334490114576e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -628.0,
      "logps/rejected": -916.0,
      "loss": 0.2293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.8125,
      "step": 2780
    },
    {
      "epoch": 0.20140041868187397,
      "grad_norm": 9.797059355953671,
      "learning_rate": 4.84515928619944e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.59375,
      "logps/chosen": -632.0,
      "logps/rejected": -988.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.1875,
      "step": 2790
    },
    {
      "epoch": 0.20212228398180898,
      "grad_norm": 9.406533092835442,
      "learning_rate": 4.84296919045867e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.4375,
      "logps/chosen": -624.0,
      "logps/rejected": -968.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.78125,
      "step": 2800
    },
    {
      "epoch": 0.20284414928174402,
      "grad_norm": 9.254549987899654,
      "learning_rate": 4.840764216799433e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.375,
      "logps/chosen": -608.0,
      "logps/rejected": -904.0,
      "loss": 0.2337,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.46875,
      "step": 2810
    },
    {
      "epoch": 0.20356601458167906,
      "grad_norm": 9.14538857003236,
      "learning_rate": 4.838544379223373e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -592.0,
      "logps/rejected": -904.0,
      "loss": 0.2619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.5,
      "step": 2820
    },
    {
      "epoch": 0.2042878798816141,
      "grad_norm": 9.535277544276601,
      "learning_rate": 4.836309691826518e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -556.0,
      "logps/rejected": -840.0,
      "loss": 0.238,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.03125,
      "rewards/margins": 2.875,
      "rewards/rejected": -6.90625,
      "step": 2830
    },
    {
      "epoch": 0.20500974518154913,
      "grad_norm": 8.327834581371992,
      "learning_rate": 4.834060168799195e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.46875,
      "logps/chosen": -592.0,
      "logps/rejected": -856.0,
      "loss": 0.2305,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.875,
      "rewards/rejected": -7.03125,
      "step": 2840
    },
    {
      "epoch": 0.20573161048148417,
      "grad_norm": 7.7347670431429405,
      "learning_rate": 4.831795824425938e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -620.0,
      "logps/rejected": -916.0,
      "loss": 0.2542,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.625,
      "step": 2850
    },
    {
      "epoch": 0.20645347578141918,
      "grad_norm": 8.15521635648461,
      "learning_rate": 4.829516673085394e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2113,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.4375,
      "step": 2860
    },
    {
      "epoch": 0.20717534108135421,
      "grad_norm": 8.022215432273788,
      "learning_rate": 4.827222729250236e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -596.0,
      "logps/rejected": -916.0,
      "loss": 0.2461,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.21875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -7.65625,
      "step": 2870
    },
    {
      "epoch": 0.20789720638128925,
      "grad_norm": 7.199943747449723,
      "learning_rate": 4.824914007487072e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -588.0,
      "logps/rejected": -932.0,
      "loss": 0.2155,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -7.75,
      "step": 2880
    },
    {
      "epoch": 0.2086190716812243,
      "grad_norm": 12.583865202253415,
      "learning_rate": 4.822590522456347e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -560.0,
      "logps/rejected": -844.0,
      "loss": 0.2309,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.0,
      "step": 2890
    },
    {
      "epoch": 0.20934093698115933,
      "grad_norm": 10.061884816438084,
      "learning_rate": 4.820252288912254e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.671875,
      "logps/chosen": -600.0,
      "logps/rejected": -876.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.3125,
      "rewards/margins": 2.875,
      "rewards/rejected": -7.1875,
      "step": 2900
    },
    {
      "epoch": 0.21006280228109434,
      "grad_norm": 8.24310045419468,
      "learning_rate": 4.817899321702642e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -620.0,
      "logps/rejected": -952.0,
      "loss": 0.2341,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -7.84375,
      "step": 2910
    },
    {
      "epoch": 0.21078466758102937,
      "grad_norm": 9.993736684687924,
      "learning_rate": 4.815531635768916e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.390625,
      "logps/chosen": -628.0,
      "logps/rejected": -952.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.78125,
      "step": 2920
    },
    {
      "epoch": 0.2115065328809644,
      "grad_norm": 8.215550634524817,
      "learning_rate": 4.813149246145946e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -648.0,
      "logps/rejected": -968.0,
      "loss": 0.2413,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.0625,
      "step": 2930
    },
    {
      "epoch": 0.21222839818089945,
      "grad_norm": 6.880787558392017,
      "learning_rate": 4.810752167961971e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -624.0,
      "logps/rejected": -904.0,
      "loss": 0.2364,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -7.40625,
      "step": 2940
    },
    {
      "epoch": 0.21295026348083448,
      "grad_norm": 8.022287212940531,
      "learning_rate": 4.808340416438505e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -632.0,
      "logps/rejected": -880.0,
      "loss": 0.2212,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.75,
      "rewards/margins": 2.734375,
      "rewards/rejected": -7.46875,
      "step": 2950
    },
    {
      "epoch": 0.21367212878076952,
      "grad_norm": 10.935811215207599,
      "learning_rate": 4.805914006890234e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -692.0,
      "logps/rejected": -972.0,
      "loss": 0.2454,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.1875,
      "rewards/margins": 2.875,
      "rewards/rejected": -8.0625,
      "step": 2960
    },
    {
      "epoch": 0.21439399408070453,
      "grad_norm": 7.928137164090748,
      "learning_rate": 4.803472954724928e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -676.0,
      "logps/rejected": -944.0,
      "loss": 0.2296,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -7.96875,
      "step": 2970
    },
    {
      "epoch": 0.21511585938063957,
      "grad_norm": 10.309732813070605,
      "learning_rate": 4.801017275443331e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -676.0,
      "logps/rejected": -980.0,
      "loss": 0.2371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.375,
      "step": 2980
    },
    {
      "epoch": 0.2158377246805746,
      "grad_norm": 9.479637952138695,
      "learning_rate": 4.798546984639076e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -712.0,
      "logps/rejected": -1040.0,
      "loss": 0.216,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -8.875,
      "step": 2990
    },
    {
      "epoch": 0.21655958998050964,
      "grad_norm": 11.653196066334537,
      "learning_rate": 4.796062097998578e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.375,
      "logps/chosen": -700.0,
      "logps/rejected": -988.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.375,
      "step": 3000
    },
    {
      "epoch": 0.21728145528044468,
      "grad_norm": 12.258514957096317,
      "learning_rate": 4.793562631300932e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -712.0,
      "logps/rejected": -996.0,
      "loss": 0.2292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.5,
      "rewards/margins": 2.875,
      "rewards/rejected": -8.375,
      "step": 3010
    },
    {
      "epoch": 0.2180033205803797,
      "grad_norm": 7.907531921224019,
      "learning_rate": 4.791048600417824e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -660.0,
      "logps/rejected": -984.0,
      "loss": 0.2244,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -8.375,
      "step": 3020
    },
    {
      "epoch": 0.21872518588031473,
      "grad_norm": 8.125306615010935,
      "learning_rate": 4.788520021313416e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.8125,
      "logps/chosen": -636.0,
      "logps/rejected": -952.0,
      "loss": 0.2078,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.875,
      "step": 3030
    },
    {
      "epoch": 0.21944705118024976,
      "grad_norm": 9.119966563903791,
      "learning_rate": 4.785976910044255e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.703125,
      "logps/chosen": -576.0,
      "logps/rejected": -924.0,
      "loss": 0.2032,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.1875,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.8125,
      "step": 3040
    },
    {
      "epoch": 0.2201689164801848,
      "grad_norm": 12.1919687251649,
      "learning_rate": 4.783419282759168e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -624.0,
      "logps/rejected": -928.0,
      "loss": 0.2312,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.84375,
      "step": 3050
    },
    {
      "epoch": 0.22089078178011984,
      "grad_norm": 10.966337886877598,
      "learning_rate": 4.780847155699157e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.484375,
      "logps/chosen": -656.0,
      "logps/rejected": -960.0,
      "loss": 0.2282,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -8.0625,
      "step": 3060
    },
    {
      "epoch": 0.22161264708005488,
      "grad_norm": 10.89546424385192,
      "learning_rate": 4.778260545197301e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -628.0,
      "logps/rejected": -916.0,
      "loss": 0.2326,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.78125,
      "rewards/margins": 2.9375,
      "rewards/rejected": -7.71875,
      "step": 3070
    },
    {
      "epoch": 0.22233451237998988,
      "grad_norm": 8.635404642662332,
      "learning_rate": 4.775659467678645e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.71875,
      "logps/chosen": -688.0,
      "logps/rejected": -1004.0,
      "loss": 0.213,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.390625,
      "rewards/rejected": -8.625,
      "step": 3080
    },
    {
      "epoch": 0.22305637767992492,
      "grad_norm": 13.172555524642066,
      "learning_rate": 4.773043939660105e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -656.0,
      "logps/rejected": -956.0,
      "loss": 0.2527,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.0625,
      "step": 3090
    },
    {
      "epoch": 0.22377824297985996,
      "grad_norm": 10.053263971609315,
      "learning_rate": 4.770413977750353e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.515625,
      "logps/chosen": -680.0,
      "logps/rejected": -984.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.3125,
      "step": 3100
    },
    {
      "epoch": 0.224500108279795,
      "grad_norm": 7.080435865876819,
      "learning_rate": 4.7677695986497225e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.515625,
      "logps/chosen": -676.0,
      "logps/rejected": -956.0,
      "loss": 0.2285,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 2.90625,
      "rewards/rejected": -7.90625,
      "step": 3110
    },
    {
      "epoch": 0.22522197357973003,
      "grad_norm": 8.489071949818994,
      "learning_rate": 4.7651108191500896e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -624.0,
      "logps/rejected": -948.0,
      "loss": 0.2217,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.3125,
      "rewards/rejected": -7.75,
      "step": 3120
    },
    {
      "epoch": 0.22594383887966504,
      "grad_norm": 7.48950614324499,
      "learning_rate": 4.762437656134778e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -648.0,
      "logps/rejected": -956.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -8.0625,
      "step": 3130
    },
    {
      "epoch": 0.22666570417960008,
      "grad_norm": 11.147541103318718,
      "learning_rate": 4.7597501265784466e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -660.0,
      "logps/rejected": -924.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.0,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.75,
      "step": 3140
    },
    {
      "epoch": 0.22738756947953512,
      "grad_norm": 9.2136696723812,
      "learning_rate": 4.757048247546982e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -612.0,
      "logps/rejected": -940.0,
      "loss": 0.2588,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.34375,
      "rewards/rejected": -7.96875,
      "step": 3150
    },
    {
      "epoch": 0.22810943477947015,
      "grad_norm": 12.83236893491969,
      "learning_rate": 4.7543320361973884e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.625,
      "step": 3160
    },
    {
      "epoch": 0.2288313000794052,
      "grad_norm": 9.608775546642143,
      "learning_rate": 4.751601509777683e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -620.0,
      "logps/rejected": -968.0,
      "loss": 0.219,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.125,
      "step": 3170
    },
    {
      "epoch": 0.2295531653793402,
      "grad_norm": 8.893320591024583,
      "learning_rate": 4.748856685626784e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.6875,
      "logps/chosen": -620.0,
      "logps/rejected": -932.0,
      "loss": 0.2299,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.6875,
      "step": 3180
    },
    {
      "epoch": 0.23027503067927524,
      "grad_norm": 6.813416083172801,
      "learning_rate": 4.7460975811743986e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.2321,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -8.25,
      "step": 3190
    },
    {
      "epoch": 0.23099689597921028,
      "grad_norm": 11.910753128704322,
      "learning_rate": 4.743324213940917e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -580.0,
      "logps/rejected": -864.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.953125,
      "rewards/rejected": -7.15625,
      "step": 3200
    },
    {
      "epoch": 0.2317187612791453,
      "grad_norm": 10.880683342821822,
      "learning_rate": 4.740536601537295e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.71875,
      "logps/chosen": -632.0,
      "logps/rejected": -924.0,
      "loss": 0.226,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.125,
      "rewards/rejected": -7.6875,
      "step": 3210
    },
    {
      "epoch": 0.23244062657908035,
      "grad_norm": 9.387075119803855,
      "learning_rate": 4.73773476166495e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -620.0,
      "logps/rejected": -916.0,
      "loss": 0.2052,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.59375,
      "step": 3220
    },
    {
      "epoch": 0.2331624918790154,
      "grad_norm": 9.319384117903645,
      "learning_rate": 4.73491871211564e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -880.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.0,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.15625,
      "step": 3230
    },
    {
      "epoch": 0.2338843571789504,
      "grad_norm": 7.92062494143796,
      "learning_rate": 4.7320884707713573e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -572.0,
      "logps/rejected": -872.0,
      "loss": 0.226,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.09375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.15625,
      "step": 3240
    },
    {
      "epoch": 0.23460622247888543,
      "grad_norm": 7.5905378566734445,
      "learning_rate": 4.7292440556042116e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.421875,
      "logps/chosen": -636.0,
      "logps/rejected": -956.0,
      "loss": 0.2381,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -8.125,
      "step": 3250
    },
    {
      "epoch": 0.23532808777882047,
      "grad_norm": 7.104939281324458,
      "learning_rate": 4.726385484676318e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -600.0,
      "logps/rejected": -876.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.4375,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.28125,
      "step": 3260
    },
    {
      "epoch": 0.2360499530787555,
      "grad_norm": 7.112864388089024,
      "learning_rate": 4.723512776139679e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -592.0,
      "logps/rejected": -872.0,
      "loss": 0.218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.65625,
      "rewards/rejected": -7.21875,
      "step": 3270
    },
    {
      "epoch": 0.23677181837869055,
      "grad_norm": 9.110259152947808,
      "learning_rate": 4.7206259482360734e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -624.0,
      "logps/rejected": -916.0,
      "loss": 0.2342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.65625,
      "step": 3280
    },
    {
      "epoch": 0.23749368367862556,
      "grad_norm": 8.883129926158574,
      "learning_rate": 4.7177250192969364e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -648.0,
      "logps/rejected": -976.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.25,
      "step": 3290
    },
    {
      "epoch": 0.2382155489785606,
      "grad_norm": 10.573842474018138,
      "learning_rate": 4.714810007743247e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -744.0,
      "logps/rejected": -1080.0,
      "loss": 0.2226,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.609375,
      "rewards/rejected": -9.25,
      "step": 3300
    },
    {
      "epoch": 0.23893741427849563,
      "grad_norm": 9.209497924010089,
      "learning_rate": 4.7118809320854077e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.46875,
      "logps/chosen": -672.0,
      "logps/rejected": -1000.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.6875,
      "step": 3310
    },
    {
      "epoch": 0.23965927957843067,
      "grad_norm": 9.372117356581454,
      "learning_rate": 4.7089378109231294e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -688.0,
      "logps/rejected": -984.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -8.3125,
      "step": 3320
    },
    {
      "epoch": 0.2403811448783657,
      "grad_norm": 8.352914263375952,
      "learning_rate": 4.7059806629453116e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -648.0,
      "logps/rejected": -920.0,
      "loss": 0.2278,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.65625,
      "step": 3330
    },
    {
      "epoch": 0.24110301017830074,
      "grad_norm": 10.512336386108508,
      "learning_rate": 4.7030095069299257e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.546875,
      "logps/chosen": -656.0,
      "logps/rejected": -976.0,
      "loss": 0.2011,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -8.125,
      "step": 3340
    },
    {
      "epoch": 0.24182487547823575,
      "grad_norm": 9.166170654186674,
      "learning_rate": 4.700024361743893e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -620.0,
      "logps/rejected": -960.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.5625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.125,
      "step": 3350
    },
    {
      "epoch": 0.2425467407781708,
      "grad_norm": 10.052841673647455,
      "learning_rate": 4.697025246342967e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -608.0,
      "logps/rejected": -916.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.40625,
      "rewards/rejected": -7.90625,
      "step": 3360
    },
    {
      "epoch": 0.24326860607810583,
      "grad_norm": 12.87931292960993,
      "learning_rate": 4.6940121797716127e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -588.0,
      "logps/rejected": -912.0,
      "loss": 0.227,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -7.625,
      "step": 3370
    },
    {
      "epoch": 0.24399047137804086,
      "grad_norm": 11.246155735848257,
      "learning_rate": 4.6909851811628853e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -644.0,
      "logps/rejected": -948.0,
      "loss": 0.1985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.96875,
      "step": 3380
    },
    {
      "epoch": 0.2447123366779759,
      "grad_norm": 8.247097831323758,
      "learning_rate": 4.68794426973831e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -584.0,
      "logps/rejected": -892.0,
      "loss": 0.2203,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.375,
      "step": 3390
    },
    {
      "epoch": 0.2454342019779109,
      "grad_norm": 7.982031142460416,
      "learning_rate": 4.684889464807755e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -588.0,
      "logps/rejected": -912.0,
      "loss": 0.2141,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.3125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.46875,
      "step": 3400
    },
    {
      "epoch": 0.24615606727784595,
      "grad_norm": 8.859057538137234,
      "learning_rate": 4.681820785769317e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -624.0,
      "logps/rejected": -932.0,
      "loss": 0.2002,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -7.875,
      "step": 3410
    },
    {
      "epoch": 0.24687793257778098,
      "grad_norm": 10.507595703830146,
      "learning_rate": 4.678738252109192e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -604.0,
      "logps/rejected": -916.0,
      "loss": 0.2272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.4375,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.71875,
      "step": 3420
    },
    {
      "epoch": 0.24759979787771602,
      "grad_norm": 9.701082860121796,
      "learning_rate": 4.675641883401553e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -624.0,
      "logps/rejected": -936.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.78125,
      "step": 3430
    },
    {
      "epoch": 0.24832166317765106,
      "grad_norm": 9.249938002509944,
      "learning_rate": 4.672531699308425e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -628.0,
      "logps/rejected": -964.0,
      "loss": 0.2167,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.1875,
      "step": 3440
    },
    {
      "epoch": 0.2490435284775861,
      "grad_norm": 10.341717167410792,
      "learning_rate": 4.669407719579562e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -604.0,
      "logps/rejected": -920.0,
      "loss": 0.2035,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.71875,
      "step": 3450
    },
    {
      "epoch": 0.2497653937775211,
      "grad_norm": 8.681196124713722,
      "learning_rate": 4.666269964052322e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -596.0,
      "logps/rejected": -952.0,
      "loss": 0.2126,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.0625,
      "step": 3460
    },
    {
      "epoch": 0.25048725907745617,
      "grad_norm": 9.212928240257371,
      "learning_rate": 4.6631184526515384e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -624.0,
      "logps/rejected": -924.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.6875,
      "step": 3470
    },
    {
      "epoch": 0.2512091243773912,
      "grad_norm": 9.229554206053074,
      "learning_rate": 4.6599532053893936e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -660.0,
      "logps/rejected": -944.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.90625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.875,
      "step": 3480
    },
    {
      "epoch": 0.2519309896773262,
      "grad_norm": 7.2543396832627876,
      "learning_rate": 4.6567742423652955e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -972.0,
      "loss": 0.2158,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.1875,
      "step": 3490
    },
    {
      "epoch": 0.25265285497726125,
      "grad_norm": 8.156746365270518,
      "learning_rate": 4.6535815837657456e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.359375,
      "logps/chosen": -688.0,
      "logps/rejected": -1024.0,
      "loss": 0.2254,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.625,
      "step": 3500
    },
    {
      "epoch": 0.25337472027719626,
      "grad_norm": 9.476042547482471,
      "learning_rate": 4.6503752498642133e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.421875,
      "logps/chosen": -696.0,
      "logps/rejected": -972.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 2.953125,
      "rewards/rejected": -8.3125,
      "step": 3510
    },
    {
      "epoch": 0.25409658557713133,
      "grad_norm": 11.068802868497535,
      "learning_rate": 4.6471552610210073e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -648.0,
      "logps/rejected": -968.0,
      "loss": 0.2236,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.125,
      "step": 3520
    },
    {
      "epoch": 0.25481845087706634,
      "grad_norm": 9.184379291366328,
      "learning_rate": 4.6439216376831447e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -628.0,
      "logps/rejected": -972.0,
      "loss": 0.2076,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.0625,
      "step": 3530
    },
    {
      "epoch": 0.25554031617700135,
      "grad_norm": 12.322007532110128,
      "learning_rate": 4.6406744003842213e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -612.0,
      "logps/rejected": -920.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.75,
      "step": 3540
    },
    {
      "epoch": 0.2562621814769364,
      "grad_norm": 7.159317606243659,
      "learning_rate": 4.6374135697442833e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -596.0,
      "logps/rejected": -940.0,
      "loss": 0.2098,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.40625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.875,
      "step": 3550
    },
    {
      "epoch": 0.2569840467768714,
      "grad_norm": 29.4711769624419,
      "learning_rate": 4.634139166469695e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -620.0,
      "logps/rejected": -944.0,
      "loss": 0.2267,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.59375,
      "step": 3560
    },
    {
      "epoch": 0.2577059120768065,
      "grad_norm": 7.80532559868148,
      "learning_rate": 4.6308512113530063e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -852.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.984375,
      "rewards/rejected": -7.0,
      "step": 3570
    },
    {
      "epoch": 0.2584277773767415,
      "grad_norm": 7.185706232473536,
      "learning_rate": 4.627549725272822e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -604.0,
      "logps/rejected": -916.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -7.59375,
      "step": 3580
    },
    {
      "epoch": 0.2591496426766765,
      "grad_norm": 7.836489814314065,
      "learning_rate": 4.62423472919367e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.421875,
      "logps/chosen": -604.0,
      "logps/rejected": -896.0,
      "loss": 0.2143,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.40625,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.40625,
      "step": 3590
    },
    {
      "epoch": 0.25987150797661157,
      "grad_norm": 10.312658330041376,
      "learning_rate": 4.6209062441658654e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.359375,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.5625,
      "step": 3600
    },
    {
      "epoch": 0.2605933732765466,
      "grad_norm": 7.660185299883346,
      "learning_rate": 4.6175642913253783e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -604.0,
      "logps/rejected": -912.0,
      "loss": 0.1951,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.65625,
      "step": 3610
    },
    {
      "epoch": 0.26131523857648165,
      "grad_norm": 9.630986662907384,
      "learning_rate": 4.6142088918937016e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -608.0,
      "logps/rejected": -952.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.4375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -8.0625,
      "step": 3620
    },
    {
      "epoch": 0.26203710387641665,
      "grad_norm": 7.9637824352647195,
      "learning_rate": 4.6108400671777135e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.40625,
      "logps/chosen": -620.0,
      "logps/rejected": -928.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.75,
      "step": 3630
    },
    {
      "epoch": 0.2627589691763517,
      "grad_norm": 10.302801071286787,
      "learning_rate": 4.607457838569544e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.296875,
      "logps/chosen": -600.0,
      "logps/rejected": -964.0,
      "loss": 0.208,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -7.84375,
      "step": 3640
    },
    {
      "epoch": 0.26348083447628673,
      "grad_norm": 9.200762753858287,
      "learning_rate": 4.6040622275464355e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -960.0,
      "loss": 0.2058,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.0625,
      "step": 3650
    },
    {
      "epoch": 0.26420269977622174,
      "grad_norm": 8.72614280680941,
      "learning_rate": 4.6006532556706124e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -532.0,
      "logps/rejected": -856.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.15625,
      "step": 3660
    },
    {
      "epoch": 0.2649245650761568,
      "grad_norm": 9.95080301383368,
      "learning_rate": 4.5972309445891386e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -884.0,
      "loss": 0.2212,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.859375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -7.46875,
      "step": 3670
    },
    {
      "epoch": 0.2656464303760918,
      "grad_norm": 7.898181548134635,
      "learning_rate": 4.593795316033783e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -588.0,
      "logps/rejected": -872.0,
      "loss": 0.2504,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.28125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.28125,
      "step": 3680
    },
    {
      "epoch": 0.2663682956760269,
      "grad_norm": 9.27071552978421,
      "learning_rate": 4.5903463918208815e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.296875,
      "logps/chosen": -560.0,
      "logps/rejected": -904.0,
      "loss": 0.2182,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -7.5,
      "step": 3690
    },
    {
      "epoch": 0.2670901609759619,
      "grad_norm": 7.471297940396982,
      "learning_rate": 4.586884193851197e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -864.0,
      "loss": 0.2165,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.125,
      "rewards/margins": 2.921875,
      "rewards/rejected": -7.03125,
      "step": 3700
    },
    {
      "epoch": 0.2678120262758969,
      "grad_norm": 9.198139304204727,
      "learning_rate": 4.5834087441097806e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -960.0,
      "loss": 0.1971,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -8.0625,
      "step": 3710
    },
    {
      "epoch": 0.26853389157583196,
      "grad_norm": 7.074538199167009,
      "learning_rate": 4.5799200646658345e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -900.0,
      "loss": 0.1914,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.53125,
      "step": 3720
    },
    {
      "epoch": 0.26925575687576697,
      "grad_norm": 7.9353245918675,
      "learning_rate": 4.576418177672568e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.6875,
      "logps/chosen": -680.0,
      "logps/rejected": -1020.0,
      "loss": 0.2194,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.5625,
      "step": 3730
    },
    {
      "epoch": 0.26997762217570204,
      "grad_norm": 7.362232757163594,
      "learning_rate": 4.5729031053670596e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -888.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.0625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -7.46875,
      "step": 3740
    },
    {
      "epoch": 0.27069948747563705,
      "grad_norm": 9.803377167019393,
      "learning_rate": 4.569374870070114e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -620.0,
      "logps/rejected": -960.0,
      "loss": 0.2077,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.125,
      "step": 3750
    },
    {
      "epoch": 0.27142135277557206,
      "grad_norm": 6.923819353668018,
      "learning_rate": 4.565833494186122e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.734375,
      "logps/chosen": -644.0,
      "logps/rejected": -968.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -8.1875,
      "step": 3760
    },
    {
      "epoch": 0.2721432180755071,
      "grad_norm": 7.409462373704994,
      "learning_rate": 4.562279000202919e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -624.0,
      "logps/rejected": -952.0,
      "loss": 0.1891,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.125,
      "step": 3770
    },
    {
      "epoch": 0.27286508337544213,
      "grad_norm": 8.55425171800717,
      "learning_rate": 4.5587114106916366e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.609375,
      "logps/chosen": -664.0,
      "logps/rejected": -1040.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.6875,
      "step": 3780
    },
    {
      "epoch": 0.2735869486753772,
      "grad_norm": 8.283760196639793,
      "learning_rate": 4.5551307483065664e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -716.0,
      "logps/rejected": -1064.0,
      "loss": 0.2264,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.125,
      "step": 3790
    },
    {
      "epoch": 0.2743088139753122,
      "grad_norm": 9.334259623082545,
      "learning_rate": 4.5515370357850136e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -664.0,
      "logps/rejected": -1056.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.890625,
      "rewards/rejected": -8.875,
      "step": 3800
    },
    {
      "epoch": 0.2750306792752472,
      "grad_norm": 8.790752805238721,
      "learning_rate": 4.547930295947151e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -640.0,
      "logps/rejected": -960.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.71875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.25,
      "step": 3810
    },
    {
      "epoch": 0.2757525445751823,
      "grad_norm": 11.07615206391536,
      "learning_rate": 4.5443105516958737e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.640625,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.25,
      "step": 3820
    },
    {
      "epoch": 0.2764744098751173,
      "grad_norm": 12.156443108793454,
      "learning_rate": 4.5406778260166586e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -648.0,
      "logps/rejected": -984.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.5,
      "step": 3830
    },
    {
      "epoch": 0.27719627517505235,
      "grad_norm": 9.054451472292419,
      "learning_rate": 4.5370321419774117e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -616.0,
      "logps/rejected": -936.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.78125,
      "step": 3840
    },
    {
      "epoch": 0.27791814047498736,
      "grad_norm": 6.698493216999741,
      "learning_rate": 4.5333735227283274e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -676.0,
      "logps/rejected": -992.0,
      "loss": 0.2083,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.4375,
      "step": 3850
    },
    {
      "epoch": 0.2786400057749224,
      "grad_norm": 9.735985205726063,
      "learning_rate": 4.5297019915017375e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -1032.0,
      "loss": 0.1875,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.875,
      "step": 3860
    },
    {
      "epoch": 0.27936187107485744,
      "grad_norm": 8.810687694126111,
      "learning_rate": 4.5260175716119655e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -660.0,
      "logps/rejected": -968.0,
      "loss": 0.212,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.71875,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.0625,
      "step": 3870
    },
    {
      "epoch": 0.28008373637479245,
      "grad_norm": 10.6784883112821,
      "learning_rate": 4.522320286455179e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -632.0,
      "logps/rejected": -944.0,
      "loss": 0.2101,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.1875,
      "step": 3880
    },
    {
      "epoch": 0.2808056016747275,
      "grad_norm": 9.94830452756906,
      "learning_rate": 4.5186101595092403e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -664.0,
      "logps/rejected": -980.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.3125,
      "step": 3890
    },
    {
      "epoch": 0.2815274669746625,
      "grad_norm": 9.418299320110647,
      "learning_rate": 4.5148872143335566e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -656.0,
      "logps/rejected": -1016.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.625,
      "step": 3900
    },
    {
      "epoch": 0.2822493322745976,
      "grad_norm": 11.904953591713511,
      "learning_rate": 4.5111514745689307e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.4375,
      "logps/chosen": -640.0,
      "logps/rejected": -1024.0,
      "loss": 0.2084,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -8.875,
      "step": 3910
    },
    {
      "epoch": 0.2829711975745326,
      "grad_norm": 11.15522188195295,
      "learning_rate": 4.507402963937414e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -704.0,
      "logps/rejected": -1064.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.0,
      "step": 3920
    },
    {
      "epoch": 0.2836930628744676,
      "grad_norm": 11.36508327799745,
      "learning_rate": 4.5036417062421506e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -748.0,
      "logps/rejected": -1088.0,
      "loss": 0.2226,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.8125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.375,
      "step": 3930
    },
    {
      "epoch": 0.28441492817440267,
      "grad_norm": 9.897929517208977,
      "learning_rate": 4.4998677253672297e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -732.0,
      "logps/rejected": -1032.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -8.6875,
      "step": 3940
    },
    {
      "epoch": 0.2851367934743377,
      "grad_norm": 9.778813209573705,
      "learning_rate": 4.496081045277533e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.359375,
      "logps/chosen": -736.0,
      "logps/rejected": -1080.0,
      "loss": 0.2031,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.875,
      "step": 3950
    },
    {
      "epoch": 0.28585865877427274,
      "grad_norm": 7.8313055942532275,
      "learning_rate": 4.492281690018583e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -696.0,
      "logps/rejected": -1048.0,
      "loss": 0.1818,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.75,
      "rewards/rejected": -9.125,
      "step": 3960
    },
    {
      "epoch": 0.28658052407420775,
      "grad_norm": 8.317207474938273,
      "learning_rate": 4.4884696837163905e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -748.0,
      "logps/rejected": -1056.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.9375,
      "step": 3970
    },
    {
      "epoch": 0.28730238937414276,
      "grad_norm": 10.26976433894447,
      "learning_rate": 4.484645050577299e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.375,
      "logps/chosen": -724.0,
      "logps/rejected": -1056.0,
      "loss": 0.2079,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.9375,
      "step": 3980
    },
    {
      "epoch": 0.28802425467407783,
      "grad_norm": 11.840261029137109,
      "learning_rate": 4.480807814887833e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -660.0,
      "logps/rejected": -960.0,
      "loss": 0.214,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.25,
      "step": 3990
    },
    {
      "epoch": 0.28874611997401284,
      "grad_norm": 8.811651662253073,
      "learning_rate": 4.476958001014544e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.4375,
      "logps/chosen": -728.0,
      "logps/rejected": -1064.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.125,
      "step": 4000
    },
    {
      "epoch": 0.2894679852739479,
      "grad_norm": 8.267204182411973,
      "learning_rate": 4.4730956334038565e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -736.0,
      "logps/rejected": -1088.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.4375,
      "step": 4010
    },
    {
      "epoch": 0.2901898505738829,
      "grad_norm": 7.77369954894956,
      "learning_rate": 4.46922073658191e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -708.0,
      "logps/rejected": -1056.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.1875,
      "step": 4020
    },
    {
      "epoch": 0.2909117158738179,
      "grad_norm": 8.886847365802758,
      "learning_rate": 4.465333335154406e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.40625,
      "logps/chosen": -684.0,
      "logps/rejected": -1020.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.6875,
      "step": 4030
    },
    {
      "epoch": 0.291633581173753,
      "grad_norm": 11.540304268095925,
      "learning_rate": 4.461433453806449e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.34375,
      "logps/chosen": -712.0,
      "logps/rejected": -1032.0,
      "loss": 0.2096,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.6875,
      "step": 4040
    },
    {
      "epoch": 0.292355446473688,
      "grad_norm": 11.614291076457404,
      "learning_rate": 4.457521117302392e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.40625,
      "logps/chosen": -636.0,
      "logps/rejected": -956.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.125,
      "step": 4050
    },
    {
      "epoch": 0.29307731177362306,
      "grad_norm": 11.959955939062517,
      "learning_rate": 4.45359635048568e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -8.375,
      "step": 4060
    },
    {
      "epoch": 0.29379917707355807,
      "grad_norm": 10.625522954210599,
      "learning_rate": 4.4496591782786883e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.2562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.25,
      "rewards/margins": 3.890625,
      "rewards/rejected": -10.125,
      "step": 4070
    },
    {
      "epoch": 0.2945210423734931,
      "grad_norm": 8.706370989411305,
      "learning_rate": 4.44570962568257e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.9375,
      "step": 4080
    },
    {
      "epoch": 0.29524290767342815,
      "grad_norm": 10.084781546011934,
      "learning_rate": 4.4417477177770905e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -704.0,
      "logps/rejected": -1056.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.609375,
      "rewards/rejected": -9.125,
      "step": 4090
    },
    {
      "epoch": 0.29596477297336315,
      "grad_norm": 10.211730254994997,
      "learning_rate": 4.4377734797204747e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1056.0,
      "loss": 0.1966,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.640625,
      "rewards/rejected": -9.125,
      "step": 4100
    },
    {
      "epoch": 0.2966866382732982,
      "grad_norm": 8.821514902632222,
      "learning_rate": 4.433786936749241e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -744.0,
      "logps/rejected": -1080.0,
      "loss": 0.2048,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.3125,
      "step": 4110
    },
    {
      "epoch": 0.29740850357323323,
      "grad_norm": 9.210606252468882,
      "learning_rate": 4.429788114178049e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -744.0,
      "logps/rejected": -1072.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -9.25,
      "step": 4120
    },
    {
      "epoch": 0.2981303688731683,
      "grad_norm": 8.00555161785549,
      "learning_rate": 4.425777037399529e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -744.0,
      "logps/rejected": -1064.0,
      "loss": 0.1886,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.40625,
      "rewards/rejected": -9.1875,
      "step": 4130
    },
    {
      "epoch": 0.2988522341731033,
      "grad_norm": 11.207346805964308,
      "learning_rate": 4.42175373188413e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -780.0,
      "logps/rejected": -1080.0,
      "loss": 0.2083,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.1875,
      "rewards/margins": 3.25,
      "rewards/rejected": -9.4375,
      "step": 4140
    },
    {
      "epoch": 0.2995740994730383,
      "grad_norm": 8.172701676742552,
      "learning_rate": 4.4177182231799513e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.34375,
      "logps/chosen": -716.0,
      "logps/rejected": -1080.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.125,
      "step": 4150
    },
    {
      "epoch": 0.3002959647729734,
      "grad_norm": 5.801373290887345,
      "learning_rate": 4.413670536912584e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -728.0,
      "logps/rejected": -1048.0,
      "loss": 0.2031,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.875,
      "step": 4160
    },
    {
      "epoch": 0.3010178300729084,
      "grad_norm": 6.307391224443462,
      "learning_rate": 4.4096106987849457e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.515625,
      "logps/chosen": -732.0,
      "logps/rejected": -1120.0,
      "loss": 0.2027,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.625,
      "step": 4170
    },
    {
      "epoch": 0.30173969537284345,
      "grad_norm": 11.867957943680011,
      "learning_rate": 4.405538734577121e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -744.0,
      "logps/rejected": -1136.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.75,
      "step": 4180
    },
    {
      "epoch": 0.30246156067277846,
      "grad_norm": 9.987667285927838,
      "learning_rate": 4.401454670146193e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -712.0,
      "logps/rejected": -1080.0,
      "loss": 0.2163,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.25,
      "step": 4190
    },
    {
      "epoch": 0.30318342597271347,
      "grad_norm": 8.977809465407699,
      "learning_rate": 4.3973585314260836e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -668.0,
      "logps/rejected": -1048.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.8125,
      "step": 4200
    },
    {
      "epoch": 0.30390529127264854,
      "grad_norm": 10.825945402642533,
      "learning_rate": 4.393250344427385e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -736.0,
      "logps/rejected": -1056.0,
      "loss": 0.2047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -9.125,
      "step": 4210
    },
    {
      "epoch": 0.30462715657258355,
      "grad_norm": 8.126172608631684,
      "learning_rate": 4.389130135237196e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -692.0,
      "logps/rejected": -1088.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.4375,
      "step": 4220
    },
    {
      "epoch": 0.3053490218725186,
      "grad_norm": 7.626491220106016,
      "learning_rate": 4.38499793001896e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.2096,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.4375,
      "step": 4230
    },
    {
      "epoch": 0.3060708871724536,
      "grad_norm": 9.904614258788264,
      "learning_rate": 4.3808537550122913e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.296875,
      "logps/chosen": -656.0,
      "logps/rejected": -948.0,
      "loss": 0.211,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -8.0625,
      "step": 4240
    },
    {
      "epoch": 0.30679275247238863,
      "grad_norm": 9.163789602486757,
      "learning_rate": 4.376697636532816e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -668.0,
      "logps/rejected": -1040.0,
      "loss": 0.1866,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.875,
      "step": 4250
    },
    {
      "epoch": 0.3075146177723237,
      "grad_norm": 9.658094123529755,
      "learning_rate": 4.3725296009719985e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5625,
      "logps/chosen": -672.0,
      "logps/rejected": -1064.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.125,
      "step": 4260
    },
    {
      "epoch": 0.3082364830722587,
      "grad_norm": 6.663743741837934,
      "learning_rate": 4.3683496747969804e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -668.0,
      "logps/rejected": -1040.0,
      "loss": 0.1913,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -8.9375,
      "step": 4270
    },
    {
      "epoch": 0.30895834837219377,
      "grad_norm": 9.431783701004399,
      "learning_rate": 4.364157884550406e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1120.0,
      "loss": 0.2088,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.625,
      "step": 4280
    },
    {
      "epoch": 0.3096802136721288,
      "grad_norm": 10.281531783875531,
      "learning_rate": 4.359954256850259e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -692.0,
      "logps/rejected": -1064.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.1875,
      "step": 4290
    },
    {
      "epoch": 0.3104020789720638,
      "grad_norm": 9.63227220127926,
      "learning_rate": 4.35573881838969e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1032.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.9375,
      "step": 4300
    },
    {
      "epoch": 0.31112394427199885,
      "grad_norm": 10.289092693608229,
      "learning_rate": 4.3515115959368476e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -716.0,
      "logps/rejected": -1016.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.6875,
      "step": 4310
    },
    {
      "epoch": 0.31184580957193386,
      "grad_norm": 10.974607276803276,
      "learning_rate": 4.3472726163347116e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -728.0,
      "logps/rejected": -1048.0,
      "loss": 0.2179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.125,
      "step": 4320
    },
    {
      "epoch": 0.31256767487186893,
      "grad_norm": 6.412992045987879,
      "learning_rate": 4.3430219065009177e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -700.0,
      "logps/rejected": -1040.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -8.8125,
      "step": 4330
    },
    {
      "epoch": 0.31328954017180394,
      "grad_norm": 10.403220300532743,
      "learning_rate": 4.3387594934275896e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1112.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.5,
      "step": 4340
    },
    {
      "epoch": 0.31401140547173895,
      "grad_norm": 6.73518127809711,
      "learning_rate": 4.334485404181167e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1104.0,
      "loss": 0.186,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.5,
      "step": 4350
    },
    {
      "epoch": 0.314733270771674,
      "grad_norm": 10.054893296825982,
      "learning_rate": 4.3301996659022334e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.28125,
      "logps/chosen": -664.0,
      "logps/rejected": -1020.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.578125,
      "rewards/rejected": -8.5625,
      "step": 4360
    },
    {
      "epoch": 0.315455136071609,
      "grad_norm": 7.828814625819192,
      "learning_rate": 4.3259023058053444e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.328125,
      "logps/chosen": -724.0,
      "logps/rejected": -1040.0,
      "loss": 0.2147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.875,
      "step": 4370
    },
    {
      "epoch": 0.3161770013715441,
      "grad_norm": 7.009502435156667,
      "learning_rate": 4.3215933511788544e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.28125,
      "logps/chosen": -716.0,
      "logps/rejected": -1020.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.875,
      "step": 4380
    },
    {
      "epoch": 0.3168988666714791,
      "grad_norm": 9.3284353021479,
      "learning_rate": 4.317272829384743e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -704.0,
      "logps/rejected": -1016.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.625,
      "step": 4390
    },
    {
      "epoch": 0.31762073197141416,
      "grad_norm": 8.811880524775491,
      "learning_rate": 4.3129407678584414e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.390625,
      "logps/chosen": -672.0,
      "logps/rejected": -1016.0,
      "loss": 0.2084,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -8.75,
      "step": 4400
    },
    {
      "epoch": 0.31834259727134917,
      "grad_norm": 8.125525464636453,
      "learning_rate": 4.3085971941086585e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.390625,
      "logps/chosen": -728.0,
      "logps/rejected": -1032.0,
      "loss": 0.1915,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.8125,
      "step": 4410
    },
    {
      "epoch": 0.3190644625712842,
      "grad_norm": 8.751734979718838,
      "learning_rate": 4.3042421357172076e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.3125,
      "logps/chosen": -708.0,
      "logps/rejected": -1072.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.9375,
      "step": 4420
    },
    {
      "epoch": 0.31978632787121924,
      "grad_norm": 7.608014094617588,
      "learning_rate": 4.2998756203388285e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -684.0,
      "logps/rejected": -1032.0,
      "loss": 0.2152,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.8125,
      "rewards/rejected": -8.8125,
      "step": 4430
    },
    {
      "epoch": 0.32050819317115425,
      "grad_norm": 7.639743064757877,
      "learning_rate": 4.2954976757010133e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -672.0,
      "logps/rejected": -1032.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -8.875,
      "step": 4440
    },
    {
      "epoch": 0.3212300584710893,
      "grad_norm": 11.460888477745158,
      "learning_rate": 4.2911083296038285e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -688.0,
      "logps/rejected": -1056.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.0625,
      "step": 4450
    },
    {
      "epoch": 0.32195192377102433,
      "grad_norm": 9.224352773153688,
      "learning_rate": 4.2867076099197446e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -668.0,
      "logps/rejected": -980.0,
      "loss": 0.1895,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.5,
      "rewards/rejected": -8.4375,
      "step": 4460
    },
    {
      "epoch": 0.32267378907095934,
      "grad_norm": 9.244876887068662,
      "learning_rate": 4.2822955445934505e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1080.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.3125,
      "step": 4470
    },
    {
      "epoch": 0.3233956543708944,
      "grad_norm": 8.906087322512382,
      "learning_rate": 4.277872161641681e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.453125,
      "logps/chosen": -760.0,
      "logps/rejected": -1120.0,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.5625,
      "step": 4480
    },
    {
      "epoch": 0.3241175196708294,
      "grad_norm": 9.689414725700667,
      "learning_rate": 4.273437489153041e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -716.0,
      "logps/rejected": -1048.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -9.0,
      "step": 4490
    },
    {
      "epoch": 0.3248393849707645,
      "grad_norm": 8.996613366246144,
      "learning_rate": 4.2689915552878207e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -688.0,
      "logps/rejected": -1008.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -8.5,
      "step": 4500
    },
    {
      "epoch": 0.3255612502706995,
      "grad_norm": 9.607472232534843,
      "learning_rate": 4.2645343882778215e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -720.0,
      "logps/rejected": -1088.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -9.4375,
      "step": 4510
    },
    {
      "epoch": 0.3262831155706345,
      "grad_norm": 8.846494209878479,
      "learning_rate": 4.260066016426177e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -680.0,
      "logps/rejected": -1032.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.8125,
      "step": 4520
    },
    {
      "epoch": 0.32700498087056956,
      "grad_norm": 11.245824423136213,
      "learning_rate": 4.25558646810717e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1020.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.6875,
      "step": 4530
    },
    {
      "epoch": 0.32772684617050457,
      "grad_norm": 8.60581993865266,
      "learning_rate": 4.251095771766055e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -672.0,
      "logps/rejected": -1048.0,
      "loss": 0.177,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.75,
      "step": 4540
    },
    {
      "epoch": 0.32844871147043964,
      "grad_norm": 7.6110846227784625,
      "learning_rate": 4.246593955918877e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -716.0,
      "logps/rejected": -1048.0,
      "loss": 0.2188,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -9.0625,
      "step": 4550
    },
    {
      "epoch": 0.32917057677037465,
      "grad_norm": 7.571006600020749,
      "learning_rate": 4.2420810491522896e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -672.0,
      "logps/rejected": -988.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.4375,
      "step": 4560
    },
    {
      "epoch": 0.32989244207030965,
      "grad_norm": 9.619717180276336,
      "learning_rate": 4.237557080123373e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.4375,
      "logps/chosen": -672.0,
      "logps/rejected": -956.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -8.25,
      "step": 4570
    },
    {
      "epoch": 0.3306143073702447,
      "grad_norm": 9.813397994219041,
      "learning_rate": 4.2330220775594567e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -660.0,
      "logps/rejected": -972.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -8.25,
      "step": 4580
    },
    {
      "epoch": 0.33133617267017973,
      "grad_norm": 8.765749359430536,
      "learning_rate": 4.228476070257929e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -680.0,
      "logps/rejected": -992.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.4375,
      "step": 4590
    },
    {
      "epoch": 0.3320580379701148,
      "grad_norm": 10.82246666876482,
      "learning_rate": 4.223919087086062e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -640.0,
      "logps/rejected": -1012.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -8.75,
      "step": 4600
    },
    {
      "epoch": 0.3327799032700498,
      "grad_norm": 7.941514770204173,
      "learning_rate": 4.2193511569808225e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -616.0,
      "logps/rejected": -988.0,
      "loss": 0.1991,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.5625,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.3125,
      "step": 4610
    },
    {
      "epoch": 0.3335017685699848,
      "grad_norm": 7.912745662185516,
      "learning_rate": 4.214772308948693e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.453125,
      "logps/chosen": -600.0,
      "logps/rejected": -1024.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.40625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -8.8125,
      "step": 4620
    },
    {
      "epoch": 0.3342236338699199,
      "grad_norm": 10.9521651772932,
      "learning_rate": 4.2101825720654827e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.484375,
      "logps/chosen": -644.0,
      "logps/rejected": -936.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -7.90625,
      "step": 4630
    },
    {
      "epoch": 0.3349454991698549,
      "grad_norm": 6.99023599526195,
      "learning_rate": 4.205581975476146e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.421875,
      "logps/chosen": -652.0,
      "logps/rejected": -988.0,
      "loss": 0.2041,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.4375,
      "step": 4640
    },
    {
      "epoch": 0.33566736446978995,
      "grad_norm": 8.603344530140768,
      "learning_rate": 4.200970548394598e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -724.0,
      "logps/rejected": -1040.0,
      "loss": 0.1965,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.875,
      "step": 4650
    },
    {
      "epoch": 0.33638922976972496,
      "grad_norm": 9.717823679934542,
      "learning_rate": 4.196348320103527e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -656.0,
      "logps/rejected": -1024.0,
      "loss": 0.213,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.625,
      "step": 4660
    },
    {
      "epoch": 0.33711109506966,
      "grad_norm": 7.935814772911749,
      "learning_rate": 4.191715319954209e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -644.0,
      "logps/rejected": -1008.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.625,
      "rewards/rejected": -8.5625,
      "step": 4670
    },
    {
      "epoch": 0.33783296036959504,
      "grad_norm": 8.769171773282734,
      "learning_rate": 4.187071577366321e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -684.0,
      "logps/rejected": -1008.0,
      "loss": 0.176,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.75,
      "step": 4680
    },
    {
      "epoch": 0.33855482566953005,
      "grad_norm": 7.096843320580391,
      "learning_rate": 4.182417121827755e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -676.0,
      "logps/rejected": -1040.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.875,
      "step": 4690
    },
    {
      "epoch": 0.3392766909694651,
      "grad_norm": 10.058671513695627,
      "learning_rate": 4.177751982894433e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -916.0,
      "loss": 0.1903,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -7.90625,
      "step": 4700
    },
    {
      "epoch": 0.3399985562694001,
      "grad_norm": 10.565296750226013,
      "learning_rate": 4.1730761901901135e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -684.0,
      "logps/rejected": -1064.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.0,
      "step": 4710
    },
    {
      "epoch": 0.3407204215693352,
      "grad_norm": 9.850412180420557,
      "learning_rate": 4.168389773406209e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -696.0,
      "logps/rejected": -1032.0,
      "loss": 0.2055,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.875,
      "step": 4720
    },
    {
      "epoch": 0.3414422868692702,
      "grad_norm": 11.906676776121467,
      "learning_rate": 4.1636927623015927e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.578125,
      "logps/chosen": -656.0,
      "logps/rejected": -1016.0,
      "loss": 0.1838,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.6875,
      "step": 4730
    },
    {
      "epoch": 0.3421641521692052,
      "grad_norm": 10.205674489245993,
      "learning_rate": 4.158985186702415e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.4375,
      "logps/chosen": -628.0,
      "logps/rejected": -1056.0,
      "loss": 0.2092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.75,
      "rewards/margins": 4.125,
      "rewards/rejected": -8.875,
      "step": 4740
    },
    {
      "epoch": 0.34288601746914027,
      "grad_norm": 9.260984330253814,
      "learning_rate": 4.1542670765019104e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -672.0,
      "logps/rejected": -1040.0,
      "loss": 0.1958,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.9375,
      "step": 4750
    },
    {
      "epoch": 0.3436078827690753,
      "grad_norm": 8.949027397431555,
      "learning_rate": 4.149538461660206e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -672.0,
      "logps/rejected": -1004.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -8.5625,
      "step": 4760
    },
    {
      "epoch": 0.34432974806901034,
      "grad_norm": 6.720014771416349,
      "learning_rate": 4.144799372204136e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -644.0,
      "logps/rejected": -956.0,
      "loss": 0.1826,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.125,
      "step": 4770
    },
    {
      "epoch": 0.34505161336894535,
      "grad_norm": 8.959024936705582,
      "learning_rate": 4.140049838227049e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -664.0,
      "logps/rejected": -1012.0,
      "loss": 0.1987,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.5625,
      "step": 4780
    },
    {
      "epoch": 0.34577347866888036,
      "grad_norm": 10.372748362230645,
      "learning_rate": 4.135289889888615e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -728.0,
      "logps/rejected": -1088.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.3125,
      "step": 4790
    },
    {
      "epoch": 0.34649534396881543,
      "grad_norm": 8.270570715124181,
      "learning_rate": 4.130519557414636e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -660.0,
      "logps/rejected": -1040.0,
      "loss": 0.2018,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -8.875,
      "step": 4800
    },
    {
      "epoch": 0.34721720926875044,
      "grad_norm": 9.893879868420212,
      "learning_rate": 4.1257388710968533e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -648.0,
      "logps/rejected": -1008.0,
      "loss": 0.1925,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -8.625,
      "step": 4810
    },
    {
      "epoch": 0.3479390745686855,
      "grad_norm": 12.331591426723168,
      "learning_rate": 4.120947861292757e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -716.0,
      "logps/rejected": -1072.0,
      "loss": 0.1966,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.1875,
      "step": 4820
    },
    {
      "epoch": 0.3486609398686205,
      "grad_norm": 10.172376095851071,
      "learning_rate": 4.11614655842539e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.65625,
      "logps/chosen": -724.0,
      "logps/rejected": -1056.0,
      "loss": 0.2024,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.9375,
      "step": 4830
    },
    {
      "epoch": 0.3493828051685555,
      "grad_norm": 11.310294528639917,
      "learning_rate": 4.111334992983156e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -640.0,
      "logps/rejected": -992.0,
      "loss": 0.1973,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.375,
      "step": 4840
    },
    {
      "epoch": 0.3501046704684906,
      "grad_norm": 9.138376308040302,
      "learning_rate": 4.1065131955196294e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -656.0,
      "logps/rejected": -1020.0,
      "loss": 0.1957,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -8.6875,
      "step": 4850
    },
    {
      "epoch": 0.3508265357684256,
      "grad_norm": 10.948111955434124,
      "learning_rate": 4.1016811966533536e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -708.0,
      "logps/rejected": -1104.0,
      "loss": 0.1823,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.5625,
      "step": 4860
    },
    {
      "epoch": 0.35154840106836066,
      "grad_norm": 6.8483776654650965,
      "learning_rate": 4.096839027067656e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -644.0,
      "logps/rejected": -1016.0,
      "loss": 0.2069,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.625,
      "step": 4870
    },
    {
      "epoch": 0.35227026636829567,
      "grad_norm": 7.262308675444548,
      "learning_rate": 4.0919867175104435e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -656.0,
      "logps/rejected": -972.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -8.25,
      "step": 4880
    },
    {
      "epoch": 0.35299213166823074,
      "grad_norm": 17.175486759838638,
      "learning_rate": 4.0871242987940155e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -688.0,
      "logps/rejected": -1048.0,
      "loss": 0.1981,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.0625,
      "step": 4890
    },
    {
      "epoch": 0.35371399696816574,
      "grad_norm": 6.418913714694265,
      "learning_rate": 4.082251801794865e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.71875,
      "logps/chosen": -684.0,
      "logps/rejected": -1048.0,
      "loss": 0.1888,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.75,
      "step": 4900
    },
    {
      "epoch": 0.35443586226810075,
      "grad_norm": 6.8125539564399515,
      "learning_rate": 4.0773692574534795e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -684.0,
      "logps/rejected": -1000.0,
      "loss": 0.1871,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.5625,
      "step": 4910
    },
    {
      "epoch": 0.3551577275680358,
      "grad_norm": 7.814190613362822,
      "learning_rate": 4.072476696774151e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.546875,
      "logps/chosen": -720.0,
      "logps/rejected": -1112.0,
      "loss": 0.1852,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.4375,
      "step": 4920
    },
    {
      "epoch": 0.35587959286797083,
      "grad_norm": 10.826610307359978,
      "learning_rate": 4.0675741508247715e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -680.0,
      "logps/rejected": -1144.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.0625,
      "step": 4930
    },
    {
      "epoch": 0.3566014581679059,
      "grad_norm": 8.596687269801581,
      "learning_rate": 4.062661650736643e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.765625,
      "logps/chosen": -728.0,
      "logps/rejected": -1112.0,
      "loss": 0.1869,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.625,
      "step": 4940
    },
    {
      "epoch": 0.3573233234678409,
      "grad_norm": 6.416195679478135,
      "learning_rate": 4.0577392277042766e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1056.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.0625,
      "step": 4950
    },
    {
      "epoch": 0.3580451887677759,
      "grad_norm": 8.918928643083534,
      "learning_rate": 4.0528069129851914e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -712.0,
      "logps/rejected": -1088.0,
      "loss": 0.1846,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.25,
      "step": 4960
    },
    {
      "epoch": 0.358767054067711,
      "grad_norm": 8.47545192360217,
      "learning_rate": 4.0478647378997223e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -724.0,
      "logps/rejected": -1048.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.0625,
      "step": 4970
    },
    {
      "epoch": 0.359488919367646,
      "grad_norm": 8.09646150016637,
      "learning_rate": 4.0429127338308154e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -700.0,
      "logps/rejected": -1024.0,
      "loss": 0.206,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.8125,
      "step": 4980
    },
    {
      "epoch": 0.36021078466758105,
      "grad_norm": 7.912242395511905,
      "learning_rate": 4.037950932223832e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -676.0,
      "logps/rejected": -1072.0,
      "loss": 0.186,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.0,
      "step": 4990
    },
    {
      "epoch": 0.36093264996751606,
      "grad_norm": 8.510565195808283,
      "learning_rate": 4.032979364586349e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.703125,
      "logps/chosen": -688.0,
      "logps/rejected": -1064.0,
      "loss": 0.2007,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.0,
      "step": 5000
    },
    {
      "epoch": 0.36093264996751606,
      "eval_logits/chosen": -2.90625,
      "eval_logits/rejected": -2.65625,
      "eval_logps/chosen": -700.0,
      "eval_logps/rejected": -1040.0,
      "eval_loss": 0.22352416813373566,
      "eval_rewards/accuracies": 0.9081980586051941,
      "eval_rewards/chosen": -5.3125,
      "eval_rewards/margins": 3.546875,
      "eval_rewards/rejected": -8.875,
      "eval_runtime": 3594.5422,
      "eval_samples_per_second": 27.403,
      "eval_steps_per_second": 0.428,
      "step": 5000
    },
    {
      "epoch": 0.36165451526745107,
      "grad_norm": 9.508216962196549,
      "learning_rate": 4.027998062487955e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.703125,
      "logps/chosen": -664.0,
      "logps/rejected": -1012.0,
      "loss": 0.1912,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.625,
      "rewards/rejected": -8.8125,
      "step": 5010
    },
    {
      "epoch": 0.36237638056738614,
      "grad_norm": 11.433083041803624,
      "learning_rate": 4.023007057560057e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.84375,
      "logps/chosen": -728.0,
      "logps/rejected": -1080.0,
      "loss": 0.1908,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.125,
      "step": 5020
    },
    {
      "epoch": 0.36309824586732115,
      "grad_norm": 9.561775751013597,
      "learning_rate": 4.018006381495671e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.8125,
      "logps/chosen": -764.0,
      "logps/rejected": -1152.0,
      "loss": 0.1764,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.9375,
      "step": 5030
    },
    {
      "epoch": 0.3638201111672562,
      "grad_norm": 8.343880295640096,
      "learning_rate": 4.012996066049229e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.640625,
      "logps/chosen": -740.0,
      "logps/rejected": -1112.0,
      "loss": 0.1988,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.4375,
      "step": 5040
    },
    {
      "epoch": 0.3645419764671912,
      "grad_norm": 7.788342844437915,
      "learning_rate": 4.0079761430363676e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.421875,
      "logps/chosen": -644.0,
      "logps/rejected": -1020.0,
      "loss": 0.1751,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -8.5,
      "step": 5050
    },
    {
      "epoch": 0.36526384176712623,
      "grad_norm": 9.014324627097576,
      "learning_rate": 4.002946644333739e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.40625,
      "logps/chosen": -748.0,
      "logps/rejected": -1104.0,
      "loss": 0.1926,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.578125,
      "rewards/rejected": -9.4375,
      "step": 5060
    },
    {
      "epoch": 0.3659857070670613,
      "grad_norm": 10.445201108593542,
      "learning_rate": 3.997907601878796e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -772.0,
      "logps/rejected": -1128.0,
      "loss": 0.1911,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -9.375,
      "step": 5070
    },
    {
      "epoch": 0.3667075723669963,
      "grad_norm": 8.250001844997403,
      "learning_rate": 3.992859047669596e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -764.0,
      "logps/rejected": -1088.0,
      "loss": 0.1844,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.90625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -9.1875,
      "step": 5080
    },
    {
      "epoch": 0.36742943766693137,
      "grad_norm": 11.753669122881204,
      "learning_rate": 3.987801013764596e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.34375,
      "logps/chosen": -784.0,
      "logps/rejected": -1152.0,
      "loss": 0.1879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -6.28125,
      "rewards/margins": 3.625,
      "rewards/rejected": -9.875,
      "step": 5090
    },
    {
      "epoch": 0.3681513029668664,
      "grad_norm": 12.658837897492882,
      "learning_rate": 3.9827335322824496e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -748.0,
      "logps/rejected": -1088.0,
      "loss": 0.207,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.5,
      "step": 5100
    },
    {
      "epoch": 0.3688731682668014,
      "grad_norm": 7.844631820767091,
      "learning_rate": 3.977656635401805e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1120.0,
      "loss": 0.1954,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.5625,
      "step": 5110
    },
    {
      "epoch": 0.36959503356673645,
      "grad_norm": 7.842154920577124,
      "learning_rate": 3.972570355361093e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.40625,
      "logps/chosen": -760.0,
      "logps/rejected": -1088.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -9.375,
      "step": 5120
    },
    {
      "epoch": 0.37031689886667146,
      "grad_norm": 12.429197801825422,
      "learning_rate": 3.967474724458334e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -704.0,
      "logps/rejected": -1056.0,
      "loss": 0.1919,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.0,
      "step": 5130
    },
    {
      "epoch": 0.3710387641666065,
      "grad_norm": 9.022738866247183,
      "learning_rate": 3.962369775050922e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -688.0,
      "logps/rejected": -1056.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.9375,
      "step": 5140
    },
    {
      "epoch": 0.37176062946654154,
      "grad_norm": 7.735894816170176,
      "learning_rate": 3.957255539555426e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -704.0,
      "logps/rejected": -1104.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.4375,
      "step": 5150
    },
    {
      "epoch": 0.3724824947664766,
      "grad_norm": 7.904648881093523,
      "learning_rate": 3.9521320504473804e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -656.0,
      "logps/rejected": -1048.0,
      "loss": 0.1898,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.0,
      "step": 5160
    },
    {
      "epoch": 0.3732043600664116,
      "grad_norm": 8.068500078950347,
      "learning_rate": 3.9469993402610823e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -668.0,
      "logps/rejected": -1000.0,
      "loss": 0.1822,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.4375,
      "rewards/rejected": -8.4375,
      "step": 5170
    },
    {
      "epoch": 0.3739262253663466,
      "grad_norm": 9.258484145773922,
      "learning_rate": 3.941857441589382e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -664.0,
      "logps/rejected": -984.0,
      "loss": 0.1924,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.4375,
      "step": 5180
    },
    {
      "epoch": 0.3746480906662817,
      "grad_norm": 9.720892187785136,
      "learning_rate": 3.936706387083476e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -696.0,
      "logps/rejected": -1064.0,
      "loss": 0.1939,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.125,
      "step": 5190
    },
    {
      "epoch": 0.3753699559662167,
      "grad_norm": 5.5288559635381365,
      "learning_rate": 3.9315462094527006e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -676.0,
      "logps/rejected": -1064.0,
      "loss": 0.1796,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.1875,
      "step": 5200
    },
    {
      "epoch": 0.37609182126615176,
      "grad_norm": 7.613626077988378,
      "learning_rate": 3.9263769414643273e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -728.0,
      "logps/rejected": -1064.0,
      "loss": 0.1846,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.578125,
      "rewards/rejected": -9.0625,
      "step": 5210
    },
    {
      "epoch": 0.37681368656608677,
      "grad_norm": 6.807486326402407,
      "learning_rate": 3.9211986159433477e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -664.0,
      "logps/rejected": -1020.0,
      "loss": 0.1722,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.875,
      "step": 5220
    },
    {
      "epoch": 0.3775355518660218,
      "grad_norm": 9.160049813590764,
      "learning_rate": 3.91601126577227e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.34375,
      "logps/chosen": -624.0,
      "logps/rejected": -1040.0,
      "loss": 0.1827,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.6875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -8.75,
      "step": 5230
    },
    {
      "epoch": 0.37825741716595684,
      "grad_norm": 10.144956306040507,
      "learning_rate": 3.910814923890911e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -636.0,
      "logps/rejected": -1020.0,
      "loss": 0.1734,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.625,
      "step": 5240
    },
    {
      "epoch": 0.37897928246589185,
      "grad_norm": 10.789604509188436,
      "learning_rate": 3.9056096232961834e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.75,
      "logps/chosen": -684.0,
      "logps/rejected": -1056.0,
      "loss": 0.1953,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.0625,
      "step": 5250
    },
    {
      "epoch": 0.3797011477658269,
      "grad_norm": 8.03159337115462,
      "learning_rate": 3.9003953970418877e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -640.0,
      "logps/rejected": -1012.0,
      "loss": 0.1836,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.5625,
      "step": 5260
    },
    {
      "epoch": 0.38042301306576193,
      "grad_norm": 9.599822536896923,
      "learning_rate": 3.8951722782385046e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.65625,
      "logps/chosen": -644.0,
      "logps/rejected": -1048.0,
      "loss": 0.1911,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.90625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.0625,
      "step": 5270
    },
    {
      "epoch": 0.38114487836569694,
      "grad_norm": 10.071856627849183,
      "learning_rate": 3.8899403000529803e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -668.0,
      "logps/rejected": -1016.0,
      "loss": 0.1888,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.8125,
      "step": 5280
    },
    {
      "epoch": 0.381866743665632,
      "grad_norm": 10.524225607275767,
      "learning_rate": 3.8846994957085194e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -680.0,
      "logps/rejected": -1056.0,
      "loss": 0.1952,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -8.875,
      "step": 5290
    },
    {
      "epoch": 0.382588608965567,
      "grad_norm": 8.616712400688618,
      "learning_rate": 3.879449898484374e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -644.0,
      "logps/rejected": -1012.0,
      "loss": 0.1873,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.484375,
      "rewards/rejected": -8.375,
      "step": 5300
    },
    {
      "epoch": 0.3833104742655021,
      "grad_norm": 8.45520493106579,
      "learning_rate": 3.874191541715629e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.578125,
      "logps/chosen": -684.0,
      "logps/rejected": -1056.0,
      "loss": 0.1777,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.78125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -8.75,
      "step": 5310
    },
    {
      "epoch": 0.3840323395654371,
      "grad_norm": 6.786490880185805,
      "learning_rate": 3.868924458792994e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.640625,
      "logps/chosen": -720.0,
      "logps/rejected": -1000.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.625,
      "step": 5320
    },
    {
      "epoch": 0.3847542048653721,
      "grad_norm": 9.306230356711307,
      "learning_rate": 3.86364868316259e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -680.0,
      "logps/rejected": -1056.0,
      "loss": 0.1961,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.75,
      "step": 5330
    },
    {
      "epoch": 0.38547607016530716,
      "grad_norm": 6.874539829983695,
      "learning_rate": 3.8583642483257374e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -708.0,
      "logps/rejected": -1088.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.1875,
      "step": 5340
    },
    {
      "epoch": 0.38619793546524217,
      "grad_norm": 9.535981769729352,
      "learning_rate": 3.85307118783874e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.765625,
      "logps/chosen": -724.0,
      "logps/rejected": -1104.0,
      "loss": 0.1932,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.5625,
      "step": 5350
    },
    {
      "epoch": 0.38691980076517724,
      "grad_norm": 7.916792530196583,
      "learning_rate": 3.8477695353126785e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -736.0,
      "logps/rejected": -1080.0,
      "loss": 0.1841,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -9.25,
      "step": 5360
    },
    {
      "epoch": 0.38764166606511224,
      "grad_norm": 11.021130141780702,
      "learning_rate": 3.8424593244131897e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -700.0,
      "logps/rejected": -1080.0,
      "loss": 0.1947,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.1875,
      "step": 5370
    },
    {
      "epoch": 0.3883635313650473,
      "grad_norm": 8.203256258991978,
      "learning_rate": 3.83714058886026e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -740.0,
      "logps/rejected": -1080.0,
      "loss": 0.1599,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.375,
      "step": 5380
    },
    {
      "epoch": 0.3890853966649823,
      "grad_norm": 7.923518332280399,
      "learning_rate": 3.8318133624280046e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -744.0,
      "logps/rejected": -1112.0,
      "loss": 0.1796,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.9375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.8125,
      "step": 5390
    },
    {
      "epoch": 0.38980726196491733,
      "grad_norm": 13.720539493582635,
      "learning_rate": 3.826477678944457e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -680.0,
      "logps/rejected": -1048.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.1875,
      "step": 5400
    },
    {
      "epoch": 0.3905291272648524,
      "grad_norm": 7.1393388118343495,
      "learning_rate": 3.821133572291354e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -688.0,
      "logps/rejected": -996.0,
      "loss": 0.1814,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.5625,
      "step": 5410
    },
    {
      "epoch": 0.3912509925647874,
      "grad_norm": 9.329246298724282,
      "learning_rate": 3.8157810764039187e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -720.0,
      "logps/rejected": -1080.0,
      "loss": 0.1938,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.3125,
      "step": 5420
    },
    {
      "epoch": 0.39197285786472247,
      "grad_norm": 6.811570201519407,
      "learning_rate": 3.810420225270647e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -732.0,
      "logps/rejected": -1080.0,
      "loss": 0.207,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.25,
      "step": 5430
    },
    {
      "epoch": 0.3926947231646575,
      "grad_norm": 6.663922484272897,
      "learning_rate": 3.80505105293309e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.515625,
      "logps/chosen": -704.0,
      "logps/rejected": -1056.0,
      "loss": 0.1847,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.625,
      "rewards/rejected": -9.0625,
      "step": 5440
    },
    {
      "epoch": 0.3934165884645925,
      "grad_norm": 9.592577634918783,
      "learning_rate": 3.799673593485638e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.359375,
      "logps/chosen": -748.0,
      "logps/rejected": -1120.0,
      "loss": 0.1773,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.5,
      "step": 5450
    },
    {
      "epoch": 0.39413845376452755,
      "grad_norm": 9.134235793523999,
      "learning_rate": 3.794287881075307e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -744.0,
      "logps/rejected": -1136.0,
      "loss": 0.1741,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.75,
      "step": 5460
    },
    {
      "epoch": 0.39486031906446256,
      "grad_norm": 8.925533209727694,
      "learning_rate": 3.788893949901517e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -764.0,
      "logps/rejected": -1152.0,
      "loss": 0.183,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.6875,
      "step": 5470
    },
    {
      "epoch": 0.3955821843643976,
      "grad_norm": 7.220703628171884,
      "learning_rate": 3.783491834215879e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -708.0,
      "logps/rejected": -1072.0,
      "loss": 0.1819,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.125,
      "step": 5480
    },
    {
      "epoch": 0.39630404966433264,
      "grad_norm": 10.56460893328502,
      "learning_rate": 3.778081568321976e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -716.0,
      "logps/rejected": -1112.0,
      "loss": 0.194,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.625,
      "step": 5490
    },
    {
      "epoch": 0.39702591496426765,
      "grad_norm": 10.362564107960075,
      "learning_rate": 3.772663186575143e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -1048.0,
      "loss": 0.1888,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -8.9375,
      "step": 5500
    },
    {
      "epoch": 0.3977477802642027,
      "grad_norm": 9.293115038985665,
      "learning_rate": 3.7672367233822537e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.34375,
      "logps/chosen": -712.0,
      "logps/rejected": -1096.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5,
      "step": 5510
    },
    {
      "epoch": 0.3984696455641377,
      "grad_norm": 8.820973054087007,
      "learning_rate": 3.761802213201498e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.59375,
      "logps/chosen": -696.0,
      "logps/rejected": -1168.0,
      "loss": 0.164,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -9.9375,
      "step": 5520
    },
    {
      "epoch": 0.3991915108640728,
      "grad_norm": 8.927273915289893,
      "learning_rate": 3.756359690542163e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.59375,
      "logps/chosen": -720.0,
      "logps/rejected": -1096.0,
      "loss": 0.1821,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.5625,
      "step": 5530
    },
    {
      "epoch": 0.3999133761640078,
      "grad_norm": 9.274501424450483,
      "learning_rate": 3.750909189964417e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -660.0,
      "logps/rejected": -1024.0,
      "loss": 0.1828,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.6875,
      "step": 5540
    },
    {
      "epoch": 0.4006352414639428,
      "grad_norm": 14.27324930632179,
      "learning_rate": 3.7454507460790893e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.53125,
      "logps/chosen": -656.0,
      "logps/rejected": -1020.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.625,
      "step": 5550
    },
    {
      "epoch": 0.40135710676387787,
      "grad_norm": 5.934956865301077,
      "learning_rate": 3.739984393547446e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.578125,
      "logps/chosen": -712.0,
      "logps/rejected": -1088.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.375,
      "step": 5560
    },
    {
      "epoch": 0.4020789720638129,
      "grad_norm": 8.744047306639088,
      "learning_rate": 3.7345101670809755e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.640625,
      "logps/chosen": -720.0,
      "logps/rejected": -1080.0,
      "loss": 0.1916,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.375,
      "step": 5570
    },
    {
      "epoch": 0.40280083736374794,
      "grad_norm": 10.141402293967214,
      "learning_rate": 3.7290281014411655e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -656.0,
      "logps/rejected": -1032.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.8125,
      "step": 5580
    },
    {
      "epoch": 0.40352270266368295,
      "grad_norm": 8.951801438600679,
      "learning_rate": 3.723538231439284e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -728.0,
      "logps/rejected": -1072.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -9.1875,
      "step": 5590
    },
    {
      "epoch": 0.40424456796361796,
      "grad_norm": 9.18772270207208,
      "learning_rate": 3.7180405919361547e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -692.0,
      "logps/rejected": -1096.0,
      "loss": 0.1996,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.5,
      "step": 5600
    },
    {
      "epoch": 0.404966433263553,
      "grad_norm": 11.802072092894162,
      "learning_rate": 3.7125352178419396e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -724.0,
      "logps/rejected": -1088.0,
      "loss": 0.1828,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -9.4375,
      "step": 5610
    },
    {
      "epoch": 0.40568829856348804,
      "grad_norm": 7.441092805572635,
      "learning_rate": 3.707022144115914e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -744.0,
      "logps/rejected": -1104.0,
      "loss": 0.1979,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.625,
      "step": 5620
    },
    {
      "epoch": 0.4064101638634231,
      "grad_norm": 8.542112860478404,
      "learning_rate": 3.701501405766248e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -740.0,
      "logps/rejected": -1104.0,
      "loss": 0.1833,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.90625,
      "rewards/margins": 3.8125,
      "rewards/rejected": -9.75,
      "step": 5630
    },
    {
      "epoch": 0.4071320291633581,
      "grad_norm": 8.690977859423688,
      "learning_rate": 3.69597303784978e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.765625,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.6875,
      "step": 5640
    },
    {
      "epoch": 0.4078538944632932,
      "grad_norm": 7.133375992414793,
      "learning_rate": 3.690437075471797e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -768.0,
      "logps/rejected": -1160.0,
      "loss": 0.1834,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.0,
      "rewards/rejected": -10.0,
      "step": 5650
    },
    {
      "epoch": 0.4085757597632282,
      "grad_norm": 9.060343279065481,
      "learning_rate": 3.684893553785815e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.53125,
      "logps/chosen": -752.0,
      "logps/rejected": -1152.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.0625,
      "step": 5660
    },
    {
      "epoch": 0.4092976250631632,
      "grad_norm": 9.150169701262556,
      "learning_rate": 3.6793425079933446e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.515625,
      "logps/chosen": -704.0,
      "logps/rejected": -1056.0,
      "loss": 0.1984,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -8.875,
      "step": 5670
    },
    {
      "epoch": 0.41001949036309826,
      "grad_norm": 11.459154584979037,
      "learning_rate": 3.67378397334368e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.40625,
      "logps/chosen": -696.0,
      "logps/rejected": -1032.0,
      "loss": 0.191,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.75,
      "step": 5680
    },
    {
      "epoch": 0.41074135566303327,
      "grad_norm": 10.628103177908564,
      "learning_rate": 3.668217985133668e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.59375,
      "logps/chosen": -704.0,
      "logps/rejected": -1064.0,
      "loss": 0.1765,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.625,
      "rewards/rejected": -9.125,
      "step": 5690
    },
    {
      "epoch": 0.41146322096296833,
      "grad_norm": 10.663063859010725,
      "learning_rate": 3.662644578707486e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -680.0,
      "logps/rejected": -1056.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.0,
      "step": 5700
    },
    {
      "epoch": 0.41218508626290334,
      "grad_norm": 9.68068326501746,
      "learning_rate": 3.657063789456418e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.609375,
      "logps/chosen": -704.0,
      "logps/rejected": -1096.0,
      "loss": 0.1893,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.4375,
      "step": 5710
    },
    {
      "epoch": 0.41290695156283835,
      "grad_norm": 9.038765481874343,
      "learning_rate": 3.6514756528186283e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -672.0,
      "logps/rejected": -1104.0,
      "loss": 0.1737,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.125,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.5,
      "step": 5720
    },
    {
      "epoch": 0.4136288168627734,
      "grad_norm": 7.8137408122782315,
      "learning_rate": 3.645880204278936e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.6875,
      "logps/chosen": -700.0,
      "logps/rejected": -1040.0,
      "loss": 0.1936,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.8125,
      "step": 5730
    },
    {
      "epoch": 0.41435068216270843,
      "grad_norm": 10.519994806474793,
      "learning_rate": 3.640277479368594e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -720.0,
      "logps/rejected": -1048.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.390625,
      "rewards/rejected": -9.0,
      "step": 5740
    },
    {
      "epoch": 0.4150725474626435,
      "grad_norm": 6.618841094002535,
      "learning_rate": 3.6346675136650595e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.515625,
      "logps/chosen": -680.0,
      "logps/rejected": -1032.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.75,
      "step": 5750
    },
    {
      "epoch": 0.4157944127625785,
      "grad_norm": 9.449161368315643,
      "learning_rate": 3.629050342791766e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -692.0,
      "logps/rejected": -1040.0,
      "loss": 0.1883,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.0,
      "step": 5760
    },
    {
      "epoch": 0.4165162780625135,
      "grad_norm": 8.645062230868888,
      "learning_rate": 3.6234260024179033e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1080.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.25,
      "step": 5770
    },
    {
      "epoch": 0.4172381433624486,
      "grad_norm": 8.61829308724476,
      "learning_rate": 3.6177945282581866e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.78125,
      "logps/chosen": -664.0,
      "logps/rejected": -1048.0,
      "loss": 0.1835,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -9.0625,
      "step": 5780
    },
    {
      "epoch": 0.4179600086623836,
      "grad_norm": 8.790765287429858,
      "learning_rate": 3.6121559560726313e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.578125,
      "logps/chosen": -676.0,
      "logps/rejected": -1064.0,
      "loss": 0.1752,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.875,
      "step": 5790
    },
    {
      "epoch": 0.41868187396231865,
      "grad_norm": 10.485975334565037,
      "learning_rate": 3.6065103216663237e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -684.0,
      "logps/rejected": -1056.0,
      "loss": 0.187,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.9375,
      "step": 5800
    },
    {
      "epoch": 0.41940373926225366,
      "grad_norm": 7.596154187472179,
      "learning_rate": 3.600857660889199e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.625,
      "logps/chosen": -632.0,
      "logps/rejected": -1012.0,
      "loss": 0.1578,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.5,
      "step": 5810
    },
    {
      "epoch": 0.42012560456218867,
      "grad_norm": 7.510040671882969,
      "learning_rate": 3.5951980096358055e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.765625,
      "logps/chosen": -632.0,
      "logps/rejected": -996.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.78125,
      "rewards/margins": 3.671875,
      "rewards/rejected": -8.4375,
      "step": 5820
    },
    {
      "epoch": 0.42084746986212374,
      "grad_norm": 6.151781029516534,
      "learning_rate": 3.589531403845085e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.75,
      "logps/chosen": -648.0,
      "logps/rejected": -1032.0,
      "loss": 0.1812,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -8.8125,
      "step": 5830
    },
    {
      "epoch": 0.42156933516205874,
      "grad_norm": 10.349825384153817,
      "learning_rate": 3.5838578795001393e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.59375,
      "logps/chosen": -724.0,
      "logps/rejected": -1152.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.625,
      "step": 5840
    },
    {
      "epoch": 0.4222912004619938,
      "grad_norm": 8.687369080835087,
      "learning_rate": 3.578177472628002e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.671875,
      "logps/chosen": -776.0,
      "logps/rejected": -1160.0,
      "loss": 0.1821,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.9375,
      "step": 5850
    },
    {
      "epoch": 0.4230130657619288,
      "grad_norm": 9.245880790108894,
      "learning_rate": 3.572490219299414e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.515625,
      "logps/chosen": -740.0,
      "logps/rejected": -1104.0,
      "loss": 0.1933,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.625,
      "step": 5860
    },
    {
      "epoch": 0.42373493106186383,
      "grad_norm": 10.513199703384252,
      "learning_rate": 3.5667961556285876e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -748.0,
      "logps/rejected": -1112.0,
      "loss": 0.172,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8125,
      "rewards/margins": 3.75,
      "rewards/rejected": -9.5625,
      "step": 5870
    },
    {
      "epoch": 0.4244567963617989,
      "grad_norm": 8.067276310939405,
      "learning_rate": 3.561095317772984e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.734375,
      "logps/chosen": -772.0,
      "logps/rejected": -1168.0,
      "loss": 0.1881,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.0,
      "step": 5880
    },
    {
      "epoch": 0.4251786616617339,
      "grad_norm": 9.335763045264015,
      "learning_rate": 3.5553877419330797e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.625,
      "logps/chosen": -748.0,
      "logps/rejected": -1152.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.8125,
      "step": 5890
    },
    {
      "epoch": 0.42590052696166897,
      "grad_norm": 8.536819036424028,
      "learning_rate": 3.5496734643521364e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.53125,
      "logps/chosen": -716.0,
      "logps/rejected": -1128.0,
      "loss": 0.198,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.625,
      "step": 5900
    },
    {
      "epoch": 0.426622392261604,
      "grad_norm": 10.712349432459751,
      "learning_rate": 3.543952521315975e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -664.0,
      "logps/rejected": -1048.0,
      "loss": 0.1793,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.9375,
      "step": 5910
    },
    {
      "epoch": 0.42734425756153904,
      "grad_norm": 9.762289940914592,
      "learning_rate": 3.538224949152738e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.546875,
      "logps/chosen": -688.0,
      "logps/rejected": -1088.0,
      "loss": 0.179,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.4375,
      "step": 5920
    },
    {
      "epoch": 0.42806612286147405,
      "grad_norm": 8.279157154276295,
      "learning_rate": 3.5324907842326676e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.765625,
      "logps/chosen": -692.0,
      "logps/rejected": -1024.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.65625,
      "rewards/rejected": -8.9375,
      "step": 5930
    },
    {
      "epoch": 0.42878798816140906,
      "grad_norm": 7.703443487862446,
      "learning_rate": 3.526750062967866e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.671875,
      "logps/chosen": -660.0,
      "logps/rejected": -1012.0,
      "loss": 0.1927,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.625,
      "step": 5940
    },
    {
      "epoch": 0.4295098534613441,
      "grad_norm": 8.915401918749712,
      "learning_rate": 3.5210028218120714e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.421875,
      "logps/chosen": -692.0,
      "logps/rejected": -1080.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.125,
      "step": 5950
    },
    {
      "epoch": 0.43023171876127914,
      "grad_norm": 7.356754892003317,
      "learning_rate": 3.515249097260422e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.1744,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.3125,
      "step": 5960
    },
    {
      "epoch": 0.4309535840612142,
      "grad_norm": 8.27577072471933,
      "learning_rate": 3.509488925849227e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.515625,
      "logps/chosen": -704.0,
      "logps/rejected": -1112.0,
      "loss": 0.1813,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.9375,
      "rewards/rejected": -9.4375,
      "step": 5970
    },
    {
      "epoch": 0.4316754493611492,
      "grad_norm": 8.91909433201475,
      "learning_rate": 3.503722344155731e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -720.0,
      "logps/rejected": -1064.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.578125,
      "rewards/rejected": -9.0,
      "step": 5980
    },
    {
      "epoch": 0.4323973146610842,
      "grad_norm": 10.528827951999629,
      "learning_rate": 3.4979493887978864e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1120.0,
      "loss": 0.1912,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.8125,
      "step": 5990
    },
    {
      "epoch": 0.4331191799610193,
      "grad_norm": 6.577916440055691,
      "learning_rate": 3.4921700964341173e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.6875,
      "logps/chosen": -764.0,
      "logps/rejected": -1120.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.6875,
      "step": 6000
    },
    {
      "epoch": 0.4338410452609543,
      "grad_norm": 9.810703167678307,
      "learning_rate": 3.486384503763089e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -744.0,
      "logps/rejected": -1080.0,
      "loss": 0.1617,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.5625,
      "step": 6010
    },
    {
      "epoch": 0.43456291056088936,
      "grad_norm": 7.569455182394066,
      "learning_rate": 3.480592647523472e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1088.0,
      "loss": 0.1617,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.375,
      "step": 6020
    },
    {
      "epoch": 0.43528477586082437,
      "grad_norm": 6.998884892266847,
      "learning_rate": 3.4747945644937133e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.4375,
      "logps/chosen": -700.0,
      "logps/rejected": -1136.0,
      "loss": 0.1807,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.9375,
      "step": 6030
    },
    {
      "epoch": 0.4360066411607594,
      "grad_norm": 7.151254090045432,
      "learning_rate": 3.468990291491799e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.515625,
      "logps/chosen": -680.0,
      "logps/rejected": -1048.0,
      "loss": 0.1727,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.0,
      "step": 6040
    },
    {
      "epoch": 0.43672850646069444,
      "grad_norm": 6.561607683446632,
      "learning_rate": 3.46317986537502e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.625,
      "logps/chosen": -720.0,
      "logps/rejected": -1176.0,
      "loss": 0.1876,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.0625,
      "step": 6050
    },
    {
      "epoch": 0.43745037176062945,
      "grad_norm": 6.7139191061366805,
      "learning_rate": 3.4573633230397414e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.75,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1948,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.6875,
      "step": 6060
    },
    {
      "epoch": 0.4381722370605645,
      "grad_norm": 8.411313986797488,
      "learning_rate": 3.451540701421167e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.671875,
      "logps/chosen": -656.0,
      "logps/rejected": -1032.0,
      "loss": 0.1803,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.921875,
      "rewards/rejected": -8.75,
      "step": 6070
    },
    {
      "epoch": 0.4388941023604995,
      "grad_norm": 8.23452700502391,
      "learning_rate": 3.445712037493104e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.703125,
      "logps/chosen": -700.0,
      "logps/rejected": -1080.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.125,
      "step": 6080
    },
    {
      "epoch": 0.43961596766043454,
      "grad_norm": 7.143620144559741,
      "learning_rate": 3.439877368267725e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -700.0,
      "logps/rejected": -1096.0,
      "loss": 0.1989,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.4375,
      "step": 6090
    },
    {
      "epoch": 0.4403378329603696,
      "grad_norm": 5.802482964711945,
      "learning_rate": 3.434036730795342e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -676.0,
      "logps/rejected": -1072.0,
      "loss": 0.1589,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.3125,
      "step": 6100
    },
    {
      "epoch": 0.4410596982603046,
      "grad_norm": 8.719522994114646,
      "learning_rate": 3.428190162164162e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.78125,
      "logps/chosen": -684.0,
      "logps/rejected": -1120.0,
      "loss": 0.1428,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -9.8125,
      "step": 6110
    },
    {
      "epoch": 0.4417815635602397,
      "grad_norm": 8.401165390196324,
      "learning_rate": 3.422337699500056e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -756.0,
      "logps/rejected": -1144.0,
      "loss": 0.1904,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.9375,
      "step": 6120
    },
    {
      "epoch": 0.4425034288601747,
      "grad_norm": 7.273765835778647,
      "learning_rate": 3.4164793799663207e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -708.0,
      "logps/rejected": -1128.0,
      "loss": 0.1881,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.75,
      "step": 6130
    },
    {
      "epoch": 0.44322529416010975,
      "grad_norm": 11.508967977205781,
      "learning_rate": 3.410615240763446e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -676.0,
      "logps/rejected": -1056.0,
      "loss": 0.2057,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.0,
      "step": 6140
    },
    {
      "epoch": 0.44394715946004476,
      "grad_norm": 9.492547090583608,
      "learning_rate": 3.4047453191288775e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1104.0,
      "loss": 0.1783,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.4375,
      "step": 6150
    },
    {
      "epoch": 0.44466902475997977,
      "grad_norm": 6.826894009448305,
      "learning_rate": 3.3988696523367777e-07,
      "logits/chosen": -3.171875,
      "logits/rejected": -2.671875,
      "logps/chosen": -688.0,
      "logps/rejected": -1104.0,
      "loss": 0.1822,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.375,
      "step": 6160
    },
    {
      "epoch": 0.44539089005991483,
      "grad_norm": 7.908259059531054,
      "learning_rate": 3.3929882776977905e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.75,
      "logps/chosen": -724.0,
      "logps/rejected": -1080.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.3125,
      "step": 6170
    },
    {
      "epoch": 0.44611275535984984,
      "grad_norm": 9.323313899318885,
      "learning_rate": 3.387101232558807e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -684.0,
      "logps/rejected": -1080.0,
      "loss": 0.1896,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.1875,
      "step": 6180
    },
    {
      "epoch": 0.4468346206597849,
      "grad_norm": 10.574159066231,
      "learning_rate": 3.381208554302723e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.609375,
      "logps/chosen": -688.0,
      "logps/rejected": -1072.0,
      "loss": 0.1926,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.1875,
      "step": 6190
    },
    {
      "epoch": 0.4475564859597199,
      "grad_norm": 9.45659923763495,
      "learning_rate": 3.3753102803482094e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -684.0,
      "logps/rejected": -1040.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.875,
      "step": 6200
    },
    {
      "epoch": 0.44827835125965493,
      "grad_norm": 7.8086390499161045,
      "learning_rate": 3.3694064481494655e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.578125,
      "logps/chosen": -684.0,
      "logps/rejected": -1056.0,
      "loss": 0.19,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.0,
      "step": 6210
    },
    {
      "epoch": 0.44900021655959,
      "grad_norm": 6.5084303039612985,
      "learning_rate": 3.3634970951959867e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -708.0,
      "logps/rejected": -1072.0,
      "loss": 0.1641,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.1875,
      "step": 6220
    },
    {
      "epoch": 0.449722081859525,
      "grad_norm": 8.137911391940417,
      "learning_rate": 3.357582259012327e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.71875,
      "logps/chosen": -700.0,
      "logps/rejected": -1048.0,
      "loss": 0.1842,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.578125,
      "rewards/rejected": -9.0,
      "step": 6230
    },
    {
      "epoch": 0.45044394715946007,
      "grad_norm": 8.376896629675258,
      "learning_rate": 3.351661977157859e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.5625,
      "logps/chosen": -664.0,
      "logps/rejected": -1144.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -9.75,
      "step": 6240
    },
    {
      "epoch": 0.4511658124593951,
      "grad_norm": 10.56542182965613,
      "learning_rate": 3.3457362872265324e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -704.0,
      "logps/rejected": -1080.0,
      "loss": 0.1604,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.375,
      "step": 6250
    },
    {
      "epoch": 0.4518876777593301,
      "grad_norm": 7.833165560372883,
      "learning_rate": 3.339805226846642e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -680.0,
      "logps/rejected": -1088.0,
      "loss": 0.1914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.4375,
      "step": 6260
    },
    {
      "epoch": 0.45260954305926515,
      "grad_norm": 7.8349709497727025,
      "learning_rate": 3.3338688336805823e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1056.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.125,
      "step": 6270
    },
    {
      "epoch": 0.45333140835920016,
      "grad_norm": 9.092744073048326,
      "learning_rate": 3.327927145424613e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -760.0,
      "logps/rejected": -1136.0,
      "loss": 0.2058,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.875,
      "step": 6280
    },
    {
      "epoch": 0.4540532736591352,
      "grad_norm": 10.949550588632437,
      "learning_rate": 3.321980199808616e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.4375,
      "logps/chosen": -712.0,
      "logps/rejected": -1104.0,
      "loss": 0.2021,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.625,
      "step": 6290
    },
    {
      "epoch": 0.45477513895907024,
      "grad_norm": 10.844258468917948,
      "learning_rate": 3.316028034595861e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -668.0,
      "logps/rejected": -1024.0,
      "loss": 0.166,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.75,
      "step": 6300
    },
    {
      "epoch": 0.45549700425900524,
      "grad_norm": 13.438072021230731,
      "learning_rate": 3.3100706875827576e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -9.0,
      "step": 6310
    },
    {
      "epoch": 0.4562188695589403,
      "grad_norm": 10.43903315431466,
      "learning_rate": 3.304108196598624e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -668.0,
      "logps/rejected": -1024.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.875,
      "step": 6320
    },
    {
      "epoch": 0.4569407348588753,
      "grad_norm": 9.193341851899063,
      "learning_rate": 3.29814059950544e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1699,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.5,
      "step": 6330
    },
    {
      "epoch": 0.4576626001588104,
      "grad_norm": 9.553793370343385,
      "learning_rate": 3.2921679341976104e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.1803,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.3125,
      "step": 6340
    },
    {
      "epoch": 0.4583844654587454,
      "grad_norm": 9.073658565124525,
      "learning_rate": 3.286190238601725e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.625,
      "logps/chosen": -656.0,
      "logps/rejected": -1056.0,
      "loss": 0.1755,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.96875,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.0625,
      "step": 6350
    },
    {
      "epoch": 0.4591063307586804,
      "grad_norm": 10.747256513722208,
      "learning_rate": 3.280207550676312e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -648.0,
      "logps/rejected": -1024.0,
      "loss": 0.1773,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.859375,
      "rewards/rejected": -8.625,
      "step": 6360
    },
    {
      "epoch": 0.45982819605861547,
      "grad_norm": 10.072254030204776,
      "learning_rate": 3.274219908411605e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.453125,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1759,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.5,
      "step": 6370
    },
    {
      "epoch": 0.4605500613585505,
      "grad_norm": 9.55913362196861,
      "learning_rate": 3.2682273498292957e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.34375,
      "logps/chosen": -692.0,
      "logps/rejected": -1088.0,
      "loss": 0.19,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.15625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.25,
      "step": 6380
    },
    {
      "epoch": 0.46127192665848554,
      "grad_norm": 7.6694528016919055,
      "learning_rate": 3.262229912982295e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -684.0,
      "logps/rejected": -1080.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.1875,
      "step": 6390
    },
    {
      "epoch": 0.46199379195842055,
      "grad_norm": 10.685410106690007,
      "learning_rate": 3.2562276359544917e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -712.0,
      "logps/rejected": -1080.0,
      "loss": 0.175,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.25,
      "step": 6400
    },
    {
      "epoch": 0.4627156572583556,
      "grad_norm": 6.117106649506752,
      "learning_rate": 3.2502205568605087e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -696.0,
      "logps/rejected": -1008.0,
      "loss": 0.167,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.40625,
      "rewards/rejected": -8.75,
      "step": 6410
    },
    {
      "epoch": 0.4634375225582906,
      "grad_norm": 10.7792422427223,
      "learning_rate": 3.244208713845461e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -672.0,
      "logps/rejected": -1096.0,
      "loss": 0.1907,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -9.4375,
      "step": 6420
    },
    {
      "epoch": 0.46415938785822564,
      "grad_norm": 7.813563275474187,
      "learning_rate": 3.2381921450847187e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -640.0,
      "logps/rejected": -1020.0,
      "loss": 0.19,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.875,
      "rewards/rejected": -8.75,
      "step": 6430
    },
    {
      "epoch": 0.4648812531581607,
      "grad_norm": 10.042020947007074,
      "learning_rate": 3.2321708887836573e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -660.0,
      "logps/rejected": -988.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.375,
      "step": 6440
    },
    {
      "epoch": 0.4656031184580957,
      "grad_norm": 10.113981129252544,
      "learning_rate": 3.226144983177419e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -648.0,
      "logps/rejected": -968.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -8.25,
      "step": 6450
    },
    {
      "epoch": 0.4663249837580308,
      "grad_norm": 10.153352236033115,
      "learning_rate": 3.220114466530668e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -692.0,
      "logps/rejected": -1072.0,
      "loss": 0.1811,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.25,
      "step": 6460
    },
    {
      "epoch": 0.4670468490579658,
      "grad_norm": 10.905089224037248,
      "learning_rate": 3.214079377137352e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -696.0,
      "logps/rejected": -1080.0,
      "loss": 0.1801,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.09375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.3125,
      "step": 6470
    },
    {
      "epoch": 0.4677687143579008,
      "grad_norm": 8.04177832973184,
      "learning_rate": 3.2080397533204526e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -672.0,
      "logps/rejected": -1024.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.65625,
      "rewards/rejected": -8.8125,
      "step": 6480
    },
    {
      "epoch": 0.46849057965783586,
      "grad_norm": 13.034607545233241,
      "learning_rate": 3.201995633431747e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.390625,
      "logps/chosen": -692.0,
      "logps/rejected": -1048.0,
      "loss": 0.183,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.125,
      "step": 6490
    },
    {
      "epoch": 0.46921244495777087,
      "grad_norm": 6.884563442702245,
      "learning_rate": 3.1959470558515613e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.3125,
      "logps/chosen": -744.0,
      "logps/rejected": -1064.0,
      "loss": 0.1807,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.453125,
      "rewards/rejected": -9.1875,
      "step": 6500
    },
    {
      "epoch": 0.46993431025770593,
      "grad_norm": 9.142548647265967,
      "learning_rate": 3.189894058988528e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -652.0,
      "logps/rejected": -1048.0,
      "loss": 0.1747,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.0,
      "step": 6510
    },
    {
      "epoch": 0.47065617555764094,
      "grad_norm": 10.80278074451447,
      "learning_rate": 3.183836681279344e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -676.0,
      "logps/rejected": -1032.0,
      "loss": 0.1948,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.875,
      "step": 6520
    },
    {
      "epoch": 0.47137804085757595,
      "grad_norm": 10.130379027496794,
      "learning_rate": 3.177774961188524e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -680.0,
      "logps/rejected": -1112.0,
      "loss": 0.1888,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.375,
      "step": 6530
    },
    {
      "epoch": 0.472099906157511,
      "grad_norm": 7.542971934363328,
      "learning_rate": 3.171708937208154e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.328125,
      "logps/chosen": -644.0,
      "logps/rejected": -1048.0,
      "loss": 0.2047,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -8.8125,
      "step": 6540
    },
    {
      "epoch": 0.472821771457446,
      "grad_norm": 10.844193020610753,
      "learning_rate": 3.1656386478576553e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -692.0,
      "logps/rejected": -1104.0,
      "loss": 0.1511,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.5,
      "step": 6550
    },
    {
      "epoch": 0.4735436367573811,
      "grad_norm": 8.465995887258453,
      "learning_rate": 3.159564131683529e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.609375,
      "logps/chosen": -700.0,
      "logps/rejected": -1112.0,
      "loss": 0.1607,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.5,
      "step": 6560
    },
    {
      "epoch": 0.4742655020573161,
      "grad_norm": 10.357924748793268,
      "learning_rate": 3.15348542725912e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.515625,
      "logps/chosen": -680.0,
      "logps/rejected": -1072.0,
      "loss": 0.1754,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.25,
      "step": 6570
    },
    {
      "epoch": 0.4749873673572511,
      "grad_norm": 8.386237735602531,
      "learning_rate": 3.1474025731843675e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -704.0,
      "logps/rejected": -1088.0,
      "loss": 0.1686,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.3125,
      "step": 6580
    },
    {
      "epoch": 0.4757092326571862,
      "grad_norm": 7.538386472518327,
      "learning_rate": 3.141315608085561e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -724.0,
      "logps/rejected": -1112.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5625,
      "step": 6590
    },
    {
      "epoch": 0.4764310979571212,
      "grad_norm": 11.830534544208183,
      "learning_rate": 3.1352245706150966e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -736.0,
      "logps/rejected": -1112.0,
      "loss": 0.1679,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.625,
      "step": 6600
    },
    {
      "epoch": 0.47715296325705625,
      "grad_norm": 8.216481532924343,
      "learning_rate": 3.129129499451229e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -728.0,
      "logps/rejected": -1104.0,
      "loss": 0.1877,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.375,
      "step": 6610
    },
    {
      "epoch": 0.47787482855699126,
      "grad_norm": 8.346007528728483,
      "learning_rate": 3.123030433297823e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -672.0,
      "logps/rejected": -1048.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.0,
      "step": 6620
    },
    {
      "epoch": 0.4785966938569263,
      "grad_norm": 11.851428689841443,
      "learning_rate": 3.11692741088412e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.390625,
      "logps/chosen": -712.0,
      "logps/rejected": -1136.0,
      "loss": 0.2184,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.8125,
      "step": 6630
    },
    {
      "epoch": 0.47931855915686133,
      "grad_norm": 11.616947258566963,
      "learning_rate": 3.110820470964479e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -684.0,
      "logps/rejected": -1080.0,
      "loss": 0.1676,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.1875,
      "step": 6640
    },
    {
      "epoch": 0.48004042445679634,
      "grad_norm": 8.689762439600946,
      "learning_rate": 3.1047096523181334e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.484375,
      "logps/chosen": -784.0,
      "logps/rejected": -1144.0,
      "loss": 0.1712,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.09375,
      "rewards/margins": 3.875,
      "rewards/rejected": -10.0,
      "step": 6650
    },
    {
      "epoch": 0.4807622897567314,
      "grad_norm": 11.54632495710895,
      "learning_rate": 3.098594993748949e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1152.0,
      "loss": 0.1905,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.0625,
      "step": 6660
    },
    {
      "epoch": 0.4814841550566664,
      "grad_norm": 8.85697591328895,
      "learning_rate": 3.092476534085177e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.328125,
      "logps/chosen": -716.0,
      "logps/rejected": -1096.0,
      "loss": 0.1831,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5,
      "step": 6670
    },
    {
      "epoch": 0.4822060203566015,
      "grad_norm": 6.837080063986226,
      "learning_rate": 3.0863543121792023e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -732.0,
      "logps/rejected": -1112.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.5,
      "step": 6680
    },
    {
      "epoch": 0.4829278856565365,
      "grad_norm": 8.376719148936688,
      "learning_rate": 3.080228366907302e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.8125,
      "logps/chosen": -752.0,
      "logps/rejected": -1152.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.0,
      "step": 6690
    },
    {
      "epoch": 0.4836497509564715,
      "grad_norm": 11.991657008786461,
      "learning_rate": 3.074098737169395e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.78125,
      "logps/chosen": -756.0,
      "logps/rejected": -1176.0,
      "loss": 0.183,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.375,
      "step": 6700
    },
    {
      "epoch": 0.48437161625640657,
      "grad_norm": 6.801458005281131,
      "learning_rate": 3.0679654618887997e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -708.0,
      "logps/rejected": -1080.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.3125,
      "step": 6710
    },
    {
      "epoch": 0.4850934815563416,
      "grad_norm": 9.454920867053952,
      "learning_rate": 3.0618285800119807e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.515625,
      "logps/chosen": -736.0,
      "logps/rejected": -1120.0,
      "loss": 0.1882,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.8125,
      "rewards/rejected": -9.5625,
      "step": 6720
    },
    {
      "epoch": 0.48581534685627664,
      "grad_norm": 8.405586504954512,
      "learning_rate": 3.0556881305083064e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.515625,
      "logps/chosen": -712.0,
      "logps/rejected": -1096.0,
      "loss": 0.1845,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5,
      "step": 6730
    },
    {
      "epoch": 0.48653721215621165,
      "grad_norm": 9.286913760440065,
      "learning_rate": 3.049544152369798e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -764.0,
      "logps/rejected": -1104.0,
      "loss": 0.1808,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -9.5,
      "step": 6740
    },
    {
      "epoch": 0.48725907745614666,
      "grad_norm": 9.84775376184747,
      "learning_rate": 3.043396684610887e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.5,
      "step": 6750
    },
    {
      "epoch": 0.4879809427560817,
      "grad_norm": 9.876467151373804,
      "learning_rate": 3.0372457662681597e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -680.0,
      "logps/rejected": -1096.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.4375,
      "step": 6760
    },
    {
      "epoch": 0.48870280805601674,
      "grad_norm": 8.982483288644246,
      "learning_rate": 3.031091436400117e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -684.0,
      "logps/rejected": -1096.0,
      "loss": 0.1926,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.5625,
      "step": 6770
    },
    {
      "epoch": 0.4894246733559518,
      "grad_norm": 8.183235398587287,
      "learning_rate": 3.024933734086922e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -652.0,
      "logps/rejected": -1072.0,
      "loss": 0.1747,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.09375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.25,
      "step": 6780
    },
    {
      "epoch": 0.4901465386558868,
      "grad_norm": 7.608116127645515,
      "learning_rate": 3.018772698430152e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -692.0,
      "logps/rejected": -1056.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.9375,
      "step": 6790
    },
    {
      "epoch": 0.4908684039558218,
      "grad_norm": 11.231827143842242,
      "learning_rate": 3.0126083685525526e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -680.0,
      "logps/rejected": -1056.0,
      "loss": 0.1669,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.1875,
      "step": 6800
    },
    {
      "epoch": 0.4915902692557569,
      "grad_norm": 16.428828202202983,
      "learning_rate": 3.006440783597787e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -716.0,
      "logps/rejected": -1104.0,
      "loss": 0.1696,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.5625,
      "step": 6810
    },
    {
      "epoch": 0.4923121345556919,
      "grad_norm": 11.61196417270313,
      "learning_rate": 3.000269982730189e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -724.0,
      "logps/rejected": -1112.0,
      "loss": 0.1766,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.78125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -9.625,
      "step": 6820
    },
    {
      "epoch": 0.49303399985562696,
      "grad_norm": 8.17788758978238,
      "learning_rate": 2.9940960051345135e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -724.0,
      "logps/rejected": -1104.0,
      "loss": 0.1803,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.5,
      "step": 6830
    },
    {
      "epoch": 0.49375586515556197,
      "grad_norm": 9.07977507120006,
      "learning_rate": 2.987918890015685e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.484375,
      "logps/chosen": -716.0,
      "logps/rejected": -1088.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.375,
      "step": 6840
    },
    {
      "epoch": 0.494477730455497,
      "grad_norm": 13.691630630266364,
      "learning_rate": 2.981738676598555e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -724.0,
      "logps/rejected": -1136.0,
      "loss": 0.164,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.6875,
      "step": 6850
    },
    {
      "epoch": 0.49519959575543204,
      "grad_norm": 10.993076123812866,
      "learning_rate": 2.9755554041276465e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -688.0,
      "logps/rejected": -1072.0,
      "loss": 0.1889,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.25,
      "step": 6860
    },
    {
      "epoch": 0.49592146105536705,
      "grad_norm": 11.646557564452921,
      "learning_rate": 2.9693691118669117e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -724.0,
      "logps/rejected": -1080.0,
      "loss": 0.1677,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.65625,
      "rewards/rejected": -9.3125,
      "step": 6870
    },
    {
      "epoch": 0.4966433263553021,
      "grad_norm": 8.874925129899568,
      "learning_rate": 2.9631798390994726e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -708.0,
      "logps/rejected": -1080.0,
      "loss": 0.1841,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.3125,
      "step": 6880
    },
    {
      "epoch": 0.4973651916552371,
      "grad_norm": 8.267348409663557,
      "learning_rate": 2.9569876251273826e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.75,
      "logps/chosen": -720.0,
      "logps/rejected": -1112.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.625,
      "step": 6890
    },
    {
      "epoch": 0.4980870569551722,
      "grad_norm": 11.307205545557014,
      "learning_rate": 2.9507925092713685e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.71875,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1592,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.6875,
      "step": 6900
    },
    {
      "epoch": 0.4988089222551072,
      "grad_norm": 9.259383366590928,
      "learning_rate": 2.9445945308705866e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.1956,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.1875,
      "step": 6910
    },
    {
      "epoch": 0.4995307875550422,
      "grad_norm": 7.91332378148377,
      "learning_rate": 2.938393729282369e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.359375,
      "logps/chosen": -700.0,
      "logps/rejected": -1064.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.65625,
      "rewards/rejected": -9.0625,
      "step": 6920
    },
    {
      "epoch": 0.5002526528549772,
      "grad_norm": 9.581050058932107,
      "learning_rate": 2.932190143881976e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -696.0,
      "logps/rejected": -1112.0,
      "loss": 0.165,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.5,
      "step": 6930
    },
    {
      "epoch": 0.5009745181549123,
      "grad_norm": 11.548026800858397,
      "learning_rate": 2.925983814062342e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1144.0,
      "loss": 0.1582,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0,
      "step": 6940
    },
    {
      "epoch": 0.5016963834548473,
      "grad_norm": 9.32142861680438,
      "learning_rate": 2.9197747792338344e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.6875,
      "logps/chosen": -724.0,
      "logps/rejected": -1112.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.6875,
      "step": 6950
    },
    {
      "epoch": 0.5024182487547824,
      "grad_norm": 8.751197433738259,
      "learning_rate": 2.9135630788239936e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -684.0,
      "logps/rejected": -1040.0,
      "loss": 0.1916,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.671875,
      "rewards/rejected": -8.9375,
      "step": 6960
    },
    {
      "epoch": 0.5031401140547174,
      "grad_norm": 9.093507158463877,
      "learning_rate": 2.907348752277286e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.421875,
      "logps/chosen": -716.0,
      "logps/rejected": -1088.0,
      "loss": 0.1908,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.125,
      "step": 6970
    },
    {
      "epoch": 0.5038619793546524,
      "grad_norm": 10.072538127620183,
      "learning_rate": 2.901131839054856e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.453125,
      "logps/chosen": -728.0,
      "logps/rejected": -1112.0,
      "loss": 0.1771,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.9375,
      "rewards/rejected": -9.5625,
      "step": 6980
    },
    {
      "epoch": 0.5045838446545875,
      "grad_norm": 7.9225853596575515,
      "learning_rate": 2.894912378634272e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -708.0,
      "logps/rejected": -1096.0,
      "loss": 0.168,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.3125,
      "step": 6990
    },
    {
      "epoch": 0.5053057099545225,
      "grad_norm": 8.508668273373068,
      "learning_rate": 2.888690410509278e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -660.0,
      "logps/rejected": -1056.0,
      "loss": 0.1732,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.875,
      "rewards/rejected": -8.9375,
      "step": 7000
    },
    {
      "epoch": 0.5060275752544575,
      "grad_norm": 9.683917887834271,
      "learning_rate": 2.8824659741895424e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -716.0,
      "logps/rejected": -1040.0,
      "loss": 0.1784,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.8125,
      "step": 7010
    },
    {
      "epoch": 0.5067494405543925,
      "grad_norm": 11.888118286993855,
      "learning_rate": 2.8762391092004054e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.53125,
      "logps/chosen": -700.0,
      "logps/rejected": -1112.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.6875,
      "step": 7020
    },
    {
      "epoch": 0.5074713058543275,
      "grad_norm": 10.520962949102882,
      "learning_rate": 2.8700098550826317e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -684.0,
      "logps/rejected": -1040.0,
      "loss": 0.1766,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.9375,
      "step": 7030
    },
    {
      "epoch": 0.5081931711542627,
      "grad_norm": 6.050995114285996,
      "learning_rate": 2.8637782513921537e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.421875,
      "logps/chosen": -688.0,
      "logps/rejected": -1088.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.3125,
      "step": 7040
    },
    {
      "epoch": 0.5089150364541977,
      "grad_norm": 11.386068773907736,
      "learning_rate": 2.857544337699826e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -1032.0,
      "loss": 0.1785,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.765625,
      "rewards/rejected": -8.875,
      "step": 7050
    },
    {
      "epoch": 0.5096369017541327,
      "grad_norm": 9.512039437649568,
      "learning_rate": 2.85130815359117e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -720.0,
      "logps/rejected": -1096.0,
      "loss": 0.17,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.4375,
      "step": 7060
    },
    {
      "epoch": 0.5103587670540677,
      "grad_norm": 11.114854171347922,
      "learning_rate": 2.845069738666128e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.671875,
      "logps/chosen": -716.0,
      "logps/rejected": -1080.0,
      "loss": 0.1952,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.3125,
      "step": 7070
    },
    {
      "epoch": 0.5110806323540027,
      "grad_norm": 6.776138063327284,
      "learning_rate": 2.8388291325388024e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.5625,
      "logps/chosen": -680.0,
      "logps/rejected": -1032.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.609375,
      "rewards/rejected": -8.8125,
      "step": 7080
    },
    {
      "epoch": 0.5118024976539378,
      "grad_norm": 9.638830345053352,
      "learning_rate": 2.832586374837215e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.609375,
      "logps/chosen": -700.0,
      "logps/rejected": -1032.0,
      "loss": 0.1917,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.5,
      "rewards/rejected": -8.875,
      "step": 7090
    },
    {
      "epoch": 0.5125243629538728,
      "grad_norm": 10.274548246717263,
      "learning_rate": 2.826341505203047e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.4375,
      "logps/chosen": -680.0,
      "logps/rejected": -1032.0,
      "loss": 0.1662,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.875,
      "step": 7100
    },
    {
      "epoch": 0.5132462282538078,
      "grad_norm": 10.65030499452255,
      "learning_rate": 2.8200945632913917e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -676.0,
      "logps/rejected": -1040.0,
      "loss": 0.1692,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.875,
      "rewards/rejected": -8.875,
      "step": 7110
    },
    {
      "epoch": 0.5139680935537428,
      "grad_norm": 8.65595924084498,
      "learning_rate": 2.8138455887705017e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -724.0,
      "logps/rejected": -1168.0,
      "loss": 0.1748,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.875,
      "step": 7120
    },
    {
      "epoch": 0.5146899588536779,
      "grad_norm": 7.416014752541617,
      "learning_rate": 2.807594621321536e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -664.0,
      "logps/rejected": -1040.0,
      "loss": 0.1723,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.0625,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.0625,
      "step": 7130
    },
    {
      "epoch": 0.515411824153613,
      "grad_norm": 9.986965415691905,
      "learning_rate": 2.801341700638307e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.578125,
      "logps/chosen": -688.0,
      "logps/rejected": -1104.0,
      "loss": 0.1896,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.3125,
      "step": 7140
    },
    {
      "epoch": 0.516133689453548,
      "grad_norm": 8.63015173531394,
      "learning_rate": 2.7950868664270355e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -680.0,
      "logps/rejected": -1080.0,
      "loss": 0.1624,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.3125,
      "step": 7150
    },
    {
      "epoch": 0.516855554753483,
      "grad_norm": 8.92932402159572,
      "learning_rate": 2.788830158406088e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1160.0,
      "loss": 0.1784,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.1875,
      "step": 7160
    },
    {
      "epoch": 0.517577420053418,
      "grad_norm": 9.18548337260447,
      "learning_rate": 2.782571616305731e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.53125,
      "logps/chosen": -712.0,
      "logps/rejected": -1120.0,
      "loss": 0.1761,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.625,
      "step": 7170
    },
    {
      "epoch": 0.518299285353353,
      "grad_norm": 7.559246854549426,
      "learning_rate": 2.7763112798678787e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -648.0,
      "logps/rejected": -1032.0,
      "loss": 0.168,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.859375,
      "rewards/rejected": -8.9375,
      "step": 7180
    },
    {
      "epoch": 0.5190211506532881,
      "grad_norm": 8.366664721309567,
      "learning_rate": 2.77004918884584e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.625,
      "logps/chosen": -716.0,
      "logps/rejected": -1104.0,
      "loss": 0.1928,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.4375,
      "step": 7190
    },
    {
      "epoch": 0.5197430159532231,
      "grad_norm": 11.737242043765724,
      "learning_rate": 2.763785383004063e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -760.0,
      "logps/rejected": -1120.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.6875,
      "step": 7200
    },
    {
      "epoch": 0.5204648812531582,
      "grad_norm": 7.926610565257705,
      "learning_rate": 2.7575199021178855e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -768.0,
      "logps/rejected": -1152.0,
      "loss": 0.1644,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.09375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -10.0,
      "step": 7210
    },
    {
      "epoch": 0.5211867465530932,
      "grad_norm": 11.714571880510713,
      "learning_rate": 2.751252785973284e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -744.0,
      "logps/rejected": -1136.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.9375,
      "step": 7220
    },
    {
      "epoch": 0.5219086118530282,
      "grad_norm": 8.471661619696453,
      "learning_rate": 2.744984074366617e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -760.0,
      "logps/rejected": -1128.0,
      "loss": 0.1723,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.875,
      "step": 7230
    },
    {
      "epoch": 0.5226304771529633,
      "grad_norm": 9.494843803969578,
      "learning_rate": 2.738713807104375e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5625,
      "logps/chosen": -740.0,
      "logps/rejected": -1120.0,
      "loss": 0.1728,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.4375,
      "step": 7240
    },
    {
      "epoch": 0.5233523424528983,
      "grad_norm": 7.787106376583822,
      "learning_rate": 2.732442024002926e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -704.0,
      "logps/rejected": -1128.0,
      "loss": 0.1461,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.8125,
      "step": 7250
    },
    {
      "epoch": 0.5240742077528333,
      "grad_norm": 6.681800096835566,
      "learning_rate": 2.726168764888264e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -752.0,
      "logps/rejected": -1160.0,
      "loss": 0.1681,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.9375,
      "step": 7260
    },
    {
      "epoch": 0.5247960730527683,
      "grad_norm": 9.923678846098278,
      "learning_rate": 2.7198940695957573e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.6875,
      "logps/chosen": -728.0,
      "logps/rejected": -1144.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.0,
      "step": 7270
    },
    {
      "epoch": 0.5255179383527034,
      "grad_norm": 9.08851776389925,
      "learning_rate": 2.713617977969892e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.734375,
      "logps/chosen": -768.0,
      "logps/rejected": -1136.0,
      "loss": 0.2118,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.875,
      "step": 7280
    },
    {
      "epoch": 0.5262398036526384,
      "grad_norm": 9.6683925726083,
      "learning_rate": 2.707340529864022e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1112.0,
      "loss": 0.1804,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -9.4375,
      "step": 7290
    },
    {
      "epoch": 0.5269616689525735,
      "grad_norm": 10.896898496114463,
      "learning_rate": 2.7010617651401133e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.625,
      "logps/chosen": -760.0,
      "logps/rejected": -1104.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 3.625,
      "rewards/rejected": -9.75,
      "step": 7300
    },
    {
      "epoch": 0.5276835342525085,
      "grad_norm": 10.726650561848246,
      "learning_rate": 2.694781723668495e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.671875,
      "logps/chosen": -752.0,
      "logps/rejected": -1152.0,
      "loss": 0.1673,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.03125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.8125,
      "step": 7310
    },
    {
      "epoch": 0.5284053995524435,
      "grad_norm": 10.009720787997963,
      "learning_rate": 2.6885004453276014e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.71875,
      "logps/chosen": -768.0,
      "logps/rejected": -1208.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.4375,
      "step": 7320
    },
    {
      "epoch": 0.5291272648523786,
      "grad_norm": 10.393937281174756,
      "learning_rate": 2.682217970003724e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.6875,
      "logps/chosen": -704.0,
      "logps/rejected": -1152.0,
      "loss": 0.1809,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.4375,
      "rewards/rejected": -9.875,
      "step": 7330
    },
    {
      "epoch": 0.5298491301523136,
      "grad_norm": 6.564234569234908,
      "learning_rate": 2.6759343375907497e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.765625,
      "logps/chosen": -672.0,
      "logps/rejected": -1096.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.5625,
      "step": 7340
    },
    {
      "epoch": 0.5305709954522486,
      "grad_norm": 8.875705075362786,
      "learning_rate": 2.669649587989918e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.765625,
      "logps/chosen": -724.0,
      "logps/rejected": -1136.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.8125,
      "step": 7350
    },
    {
      "epoch": 0.5312928607521836,
      "grad_norm": 8.149716947695198,
      "learning_rate": 2.6633637611095615e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.65625,
      "logps/chosen": -704.0,
      "logps/rejected": -1104.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.4375,
      "step": 7360
    },
    {
      "epoch": 0.5320147260521186,
      "grad_norm": 9.117623642609074,
      "learning_rate": 2.6570768968648515e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.734375,
      "logps/chosen": -708.0,
      "logps/rejected": -1104.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.5625,
      "step": 7370
    },
    {
      "epoch": 0.5327365913520538,
      "grad_norm": 7.7833308392028115,
      "learning_rate": 2.6507890351775485e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.703125,
      "logps/chosen": -672.0,
      "logps/rejected": -1080.0,
      "loss": 0.1529,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.125,
      "step": 7380
    },
    {
      "epoch": 0.5334584566519888,
      "grad_norm": 11.143661735085152,
      "learning_rate": 2.644500215975747e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.515625,
      "logps/chosen": -724.0,
      "logps/rejected": -1088.0,
      "loss": 0.177,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.375,
      "step": 7390
    },
    {
      "epoch": 0.5341803219519238,
      "grad_norm": 6.014943903391089,
      "learning_rate": 2.63821047919362e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1905,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.8125,
      "step": 7400
    },
    {
      "epoch": 0.5349021872518588,
      "grad_norm": 9.506926726417404,
      "learning_rate": 2.631919864771168e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -712.0,
      "logps/rejected": -1120.0,
      "loss": 0.153,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.625,
      "step": 7410
    },
    {
      "epoch": 0.5356240525517938,
      "grad_norm": 8.989178126574519,
      "learning_rate": 2.6256284126539657e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -748.0,
      "logps/rejected": -1152.0,
      "loss": 0.1589,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.0,
      "step": 7420
    },
    {
      "epoch": 0.5363459178517289,
      "grad_norm": 8.096664424589033,
      "learning_rate": 2.619336162792905e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -736.0,
      "logps/rejected": -1192.0,
      "loss": 0.1436,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.5,
      "step": 7430
    },
    {
      "epoch": 0.5370677831516639,
      "grad_norm": 16.841361704172662,
      "learning_rate": 2.6130431551439456e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.75,
      "logps/chosen": -732.0,
      "logps/rejected": -1176.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.1875,
      "step": 7440
    },
    {
      "epoch": 0.5377896484515989,
      "grad_norm": 10.149973787308852,
      "learning_rate": 2.606749429667859e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.796875,
      "logps/chosen": -788.0,
      "logps/rejected": -1216.0,
      "loss": 0.1571,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.5625,
      "step": 7450
    },
    {
      "epoch": 0.5385115137515339,
      "grad_norm": 11.046710856292618,
      "learning_rate": 2.6004550263299724e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 7460
    },
    {
      "epoch": 0.539233379051469,
      "grad_norm": 9.061807143540431,
      "learning_rate": 2.5941599850999194e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.4375,
      "logps/chosen": -792.0,
      "logps/rejected": -1160.0,
      "loss": 0.1776,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -6.34375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -10.125,
      "step": 7470
    },
    {
      "epoch": 0.5399552443514041,
      "grad_norm": 10.136512372235272,
      "learning_rate": 2.5878643459513857e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1714,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 7480
    },
    {
      "epoch": 0.5406771096513391,
      "grad_norm": 9.091121034585713,
      "learning_rate": 2.58156814886185e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -740.0,
      "logps/rejected": -1072.0,
      "loss": 0.1804,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.375,
      "step": 7490
    },
    {
      "epoch": 0.5413989749512741,
      "grad_norm": 11.023486939102211,
      "learning_rate": 2.575271433812338e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -720.0,
      "logps/rejected": -1112.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.5625,
      "step": 7500
    },
    {
      "epoch": 0.5421208402512091,
      "grad_norm": 12.378483126658342,
      "learning_rate": 2.5689742407871615e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.65625,
      "logps/chosen": -712.0,
      "logps/rejected": -1088.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.4375,
      "step": 7510
    },
    {
      "epoch": 0.5428427055511441,
      "grad_norm": 9.630319451714357,
      "learning_rate": 2.562676609773669e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -736.0,
      "logps/rejected": -1160.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.0625,
      "step": 7520
    },
    {
      "epoch": 0.5435645708510792,
      "grad_norm": 8.457889597977909,
      "learning_rate": 2.556378580761989e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -720.0,
      "logps/rejected": -1136.0,
      "loss": 0.1797,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.9375,
      "step": 7530
    },
    {
      "epoch": 0.5442864361510142,
      "grad_norm": 8.046649935179103,
      "learning_rate": 2.5500801937447794e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1104.0,
      "loss": 0.1379,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.625,
      "step": 7540
    },
    {
      "epoch": 0.5450083014509493,
      "grad_norm": 9.046127207562416,
      "learning_rate": 2.5437814887169684e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -688.0,
      "logps/rejected": -1120.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.5625,
      "step": 7550
    },
    {
      "epoch": 0.5457301667508843,
      "grad_norm": 9.329772174624635,
      "learning_rate": 2.5374825056755073e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -728.0,
      "logps/rejected": -1128.0,
      "loss": 0.1942,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.875,
      "step": 7560
    },
    {
      "epoch": 0.5464520320508193,
      "grad_norm": 9.9179656907273,
      "learning_rate": 2.531183284619109e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.625,
      "logps/chosen": -704.0,
      "logps/rejected": -1120.0,
      "loss": 0.1747,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.5625,
      "step": 7570
    },
    {
      "epoch": 0.5471738973507544,
      "grad_norm": 9.557689966293122,
      "learning_rate": 2.524883865548e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.1632,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.3125,
      "step": 7580
    },
    {
      "epoch": 0.5478957626506894,
      "grad_norm": 9.086340027521503,
      "learning_rate": 2.5185842884636633e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -720.0,
      "logps/rejected": -1080.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.1875,
      "step": 7590
    },
    {
      "epoch": 0.5486176279506244,
      "grad_norm": 8.606477368401597,
      "learning_rate": 2.5122845933685873e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.609375,
      "logps/chosen": -672.0,
      "logps/rejected": -1096.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.625,
      "step": 7600
    },
    {
      "epoch": 0.5493394932505594,
      "grad_norm": 9.684120993406607,
      "learning_rate": 2.505984820266007e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -716.0,
      "logps/rejected": -1128.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.75,
      "step": 7610
    },
    {
      "epoch": 0.5500613585504944,
      "grad_norm": 11.673738089201613,
      "learning_rate": 2.4996850091596535e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -704.0,
      "logps/rejected": -1136.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.875,
      "step": 7620
    },
    {
      "epoch": 0.5507832238504295,
      "grad_norm": 9.43384042742059,
      "learning_rate": 2.493385200053502e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.6875,
      "logps/chosen": -752.0,
      "logps/rejected": -1136.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.875,
      "step": 7630
    },
    {
      "epoch": 0.5515050891503646,
      "grad_norm": 7.477187732924252,
      "learning_rate": 2.48708543295151e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.671875,
      "logps/chosen": -712.0,
      "logps/rejected": -1168.0,
      "loss": 0.1651,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.0625,
      "step": 7640
    },
    {
      "epoch": 0.5522269544502996,
      "grad_norm": 7.419622643126169,
      "learning_rate": 2.480785747857372e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -748.0,
      "logps/rejected": -1112.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.625,
      "step": 7650
    },
    {
      "epoch": 0.5529488197502346,
      "grad_norm": 7.6418997353667955,
      "learning_rate": 2.474486184774262e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.25,
      "step": 7660
    },
    {
      "epoch": 0.5536706850501696,
      "grad_norm": 11.143956805660984,
      "learning_rate": 2.468186783704575e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.734375,
      "logps/chosen": -740.0,
      "logps/rejected": -1160.0,
      "loss": 0.1838,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.0625,
      "step": 7670
    },
    {
      "epoch": 0.5543925503501047,
      "grad_norm": 12.296442116097905,
      "learning_rate": 2.461887584649684e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.65625,
      "logps/chosen": -736.0,
      "logps/rejected": -1128.0,
      "loss": 0.1699,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.75,
      "step": 7680
    },
    {
      "epoch": 0.5551144156500397,
      "grad_norm": 13.050278426508344,
      "learning_rate": 2.4555886276096713e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -724.0,
      "logps/rejected": -1112.0,
      "loss": 0.1799,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.625,
      "step": 7690
    },
    {
      "epoch": 0.5558362809499747,
      "grad_norm": 7.331592070155222,
      "learning_rate": 2.4492899525830896e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.65625,
      "logps/chosen": -684.0,
      "logps/rejected": -1080.0,
      "loss": 0.1688,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.375,
      "step": 7700
    },
    {
      "epoch": 0.5565581462499097,
      "grad_norm": 10.969811602721066,
      "learning_rate": 2.4429915995666967e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.59375,
      "logps/chosen": -692.0,
      "logps/rejected": -1080.0,
      "loss": 0.1692,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.0625,
      "step": 7710
    },
    {
      "epoch": 0.5572800115498447,
      "grad_norm": 6.714447148185967,
      "learning_rate": 2.4366936085552055e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -684.0,
      "logps/rejected": -1072.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.3125,
      "step": 7720
    },
    {
      "epoch": 0.5580018768497799,
      "grad_norm": 6.283567486920552,
      "learning_rate": 2.430396019541032e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.671875,
      "logps/chosen": -716.0,
      "logps/rejected": -1184.0,
      "loss": 0.1521,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.0625,
      "step": 7730
    },
    {
      "epoch": 0.5587237421497149,
      "grad_norm": 9.641423565186832,
      "learning_rate": 2.42409887251404e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.65625,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.5625,
      "step": 7740
    },
    {
      "epoch": 0.5594456074496499,
      "grad_norm": 9.82613619768625,
      "learning_rate": 2.4178022074612834e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.78125,
      "logps/chosen": -720.0,
      "logps/rejected": -1104.0,
      "loss": 0.1583,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.4375,
      "step": 7750
    },
    {
      "epoch": 0.5601674727495849,
      "grad_norm": 8.116478311651692,
      "learning_rate": 2.4115060643667596e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -728.0,
      "logps/rejected": -1152.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.9375,
      "step": 7760
    },
    {
      "epoch": 0.56088933804952,
      "grad_norm": 9.994907736669813,
      "learning_rate": 2.4052104832111477e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.875,
      "step": 7770
    },
    {
      "epoch": 0.561611203349455,
      "grad_norm": 8.76711793301904,
      "learning_rate": 2.39891550397156e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.546875,
      "logps/chosen": -704.0,
      "logps/rejected": -1088.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.4375,
      "step": 7780
    },
    {
      "epoch": 0.56233306864939,
      "grad_norm": 8.295373017899776,
      "learning_rate": 2.392621166621289e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.515625,
      "logps/chosen": -712.0,
      "logps/rejected": -1144.0,
      "loss": 0.1816,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -9.9375,
      "step": 7790
    },
    {
      "epoch": 0.563054933949325,
      "grad_norm": 7.740744626757316,
      "learning_rate": 2.386327511129548e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.46875,
      "logps/chosen": -728.0,
      "logps/rejected": -1104.0,
      "loss": 0.1984,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.4375,
      "step": 7800
    },
    {
      "epoch": 0.56377679924926,
      "grad_norm": 10.604529688792997,
      "learning_rate": 2.380034577461221e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -676.0,
      "logps/rejected": -1112.0,
      "loss": 0.1593,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.15625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.5,
      "step": 7810
    },
    {
      "epoch": 0.5644986645491952,
      "grad_norm": 6.778093404198004,
      "learning_rate": 2.373742405576611e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -720.0,
      "logps/rejected": -1072.0,
      "loss": 0.1712,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.25,
      "step": 7820
    },
    {
      "epoch": 0.5652205298491302,
      "grad_norm": 10.12858521011711,
      "learning_rate": 2.3674510354311787e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.421875,
      "logps/chosen": -728.0,
      "logps/rejected": -1136.0,
      "loss": 0.1778,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.75,
      "step": 7830
    },
    {
      "epoch": 0.5659423951490652,
      "grad_norm": 8.064195843692932,
      "learning_rate": 2.3611605069752988e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.453125,
      "logps/chosen": -756.0,
      "logps/rejected": -1136.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.0,
      "step": 7840
    },
    {
      "epoch": 0.5666642604490002,
      "grad_norm": 9.775841021106938,
      "learning_rate": 2.3548708601539982e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -752.0,
      "logps/rejected": -1136.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -9.875,
      "step": 7850
    },
    {
      "epoch": 0.5673861257489352,
      "grad_norm": 10.098938805871972,
      "learning_rate": 2.3485821349067062e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -720.0,
      "logps/rejected": -1136.0,
      "loss": 0.1695,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.0,
      "step": 7860
    },
    {
      "epoch": 0.5681079910488703,
      "grad_norm": 8.30412143652236,
      "learning_rate": 2.3422943711670015e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1088.0,
      "loss": 0.1625,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.375,
      "step": 7870
    },
    {
      "epoch": 0.5688298563488053,
      "grad_norm": 6.817526291974242,
      "learning_rate": 2.3360076088623544e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.703125,
      "logps/chosen": -708.0,
      "logps/rejected": -1072.0,
      "loss": 0.1805,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.375,
      "step": 7880
    },
    {
      "epoch": 0.5695517216487404,
      "grad_norm": 10.06504278640758,
      "learning_rate": 2.3297218879138775e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -704.0,
      "logps/rejected": -1080.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.25,
      "step": 7890
    },
    {
      "epoch": 0.5702735869486754,
      "grad_norm": 7.820665146752294,
      "learning_rate": 2.3234372482360726e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -712.0,
      "logps/rejected": -1104.0,
      "loss": 0.1642,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.5625,
      "step": 7900
    },
    {
      "epoch": 0.5709954522486104,
      "grad_norm": 11.261516761461985,
      "learning_rate": 2.3171537297365718e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.765625,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1524,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -9.625,
      "step": 7910
    },
    {
      "epoch": 0.5717173175485455,
      "grad_norm": 12.326978372880708,
      "learning_rate": 2.3108713723158908e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.625,
      "logps/chosen": -664.0,
      "logps/rejected": -1136.0,
      "loss": 0.1601,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -9.8125,
      "step": 7920
    },
    {
      "epoch": 0.5724391828484805,
      "grad_norm": 12.561004648741765,
      "learning_rate": 2.304590215867169e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.75,
      "logps/chosen": -716.0,
      "logps/rejected": -1112.0,
      "loss": 0.1912,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.5625,
      "step": 7930
    },
    {
      "epoch": 0.5731610481484155,
      "grad_norm": 7.388599673863509,
      "learning_rate": 2.298310300275924e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1104.0,
      "loss": 0.1454,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.5,
      "step": 7940
    },
    {
      "epoch": 0.5738829134483505,
      "grad_norm": 6.502079922036917,
      "learning_rate": 2.2920316654197912e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -736.0,
      "logps/rejected": -1120.0,
      "loss": 0.156,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.75,
      "step": 7950
    },
    {
      "epoch": 0.5746047787482855,
      "grad_norm": 8.707306377676787,
      "learning_rate": 2.2857543511682721e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -732.0,
      "logps/rejected": -1088.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.375,
      "step": 7960
    },
    {
      "epoch": 0.5753266440482206,
      "grad_norm": 9.94916607775227,
      "learning_rate": 2.279478397382486e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.640625,
      "logps/chosen": -752.0,
      "logps/rejected": -1160.0,
      "loss": 0.186,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.0625,
      "step": 7970
    },
    {
      "epoch": 0.5760485093481557,
      "grad_norm": 6.43085308116789,
      "learning_rate": 2.273203843914911e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -708.0,
      "logps/rejected": -1080.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.375,
      "step": 7980
    },
    {
      "epoch": 0.5767703746480907,
      "grad_norm": 9.384961354414472,
      "learning_rate": 2.266930730609132e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.640625,
      "logps/chosen": -736.0,
      "logps/rejected": -1104.0,
      "loss": 0.1473,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.5625,
      "step": 7990
    },
    {
      "epoch": 0.5774922399480257,
      "grad_norm": 7.869464671685872,
      "learning_rate": 2.260659097299591e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.65625,
      "logps/chosen": -732.0,
      "logps/rejected": -1144.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.0,
      "step": 8000
    },
    {
      "epoch": 0.5782141052479607,
      "grad_norm": 6.036220985959268,
      "learning_rate": 2.2543889838113297e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.65625,
      "logps/chosen": -740.0,
      "logps/rejected": -1200.0,
      "loss": 0.1523,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.5,
      "step": 8010
    },
    {
      "epoch": 0.5789359705478958,
      "grad_norm": 9.091611380745027,
      "learning_rate": 2.2481204299597422e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.640625,
      "logps/chosen": -728.0,
      "logps/rejected": -1136.0,
      "loss": 0.1708,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.9375,
      "step": 8020
    },
    {
      "epoch": 0.5796578358478308,
      "grad_norm": 8.969271615418297,
      "learning_rate": 2.2418534755503169e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -720.0,
      "logps/rejected": -1144.0,
      "loss": 0.154,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.75,
      "step": 8030
    },
    {
      "epoch": 0.5803797011477658,
      "grad_norm": 8.869611211845172,
      "learning_rate": 2.235588160378384e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.625,
      "logps/chosen": -724.0,
      "logps/rejected": -1096.0,
      "loss": 0.1892,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.3125,
      "step": 8040
    },
    {
      "epoch": 0.5811015664477008,
      "grad_norm": 8.585563297437242,
      "learning_rate": 2.229324524228868e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.71875,
      "logps/chosen": -704.0,
      "logps/rejected": -1048.0,
      "loss": 0.1972,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.0,
      "step": 8050
    },
    {
      "epoch": 0.5818234317476358,
      "grad_norm": 6.361935798258764,
      "learning_rate": 2.2230626068760297e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -680.0,
      "logps/rejected": -1088.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.25,
      "step": 8060
    },
    {
      "epoch": 0.582545297047571,
      "grad_norm": 7.123503913889777,
      "learning_rate": 2.216802448083214e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.671875,
      "logps/chosen": -672.0,
      "logps/rejected": -1048.0,
      "loss": 0.1524,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.71875,
      "rewards/rejected": -9.0,
      "step": 8070
    },
    {
      "epoch": 0.583267162347506,
      "grad_norm": 5.3274420243546645,
      "learning_rate": 2.2105440876026026e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.671875,
      "logps/chosen": -712.0,
      "logps/rejected": -1144.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.5,
      "rewards/rejected": -9.75,
      "step": 8080
    },
    {
      "epoch": 0.583989027647441,
      "grad_norm": 8.126969483825905,
      "learning_rate": 2.2042875651749548e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.1605,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.8125,
      "step": 8090
    },
    {
      "epoch": 0.584710892947376,
      "grad_norm": 6.82538345344394,
      "learning_rate": 2.1980329205293585e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.625,
      "logps/chosen": -720.0,
      "logps/rejected": -1104.0,
      "loss": 0.1633,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.75,
      "step": 8100
    },
    {
      "epoch": 0.585432758247311,
      "grad_norm": 12.200961629937073,
      "learning_rate": 2.1917801933829807e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -716.0,
      "logps/rejected": -1120.0,
      "loss": 0.1516,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.6875,
      "step": 8110
    },
    {
      "epoch": 0.5861546235472461,
      "grad_norm": 7.906908172237448,
      "learning_rate": 2.1855294234408068e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.59375,
      "logps/chosen": -724.0,
      "logps/rejected": -1104.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.5625,
      "step": 8120
    },
    {
      "epoch": 0.5868764888471811,
      "grad_norm": 10.313898842262107,
      "learning_rate": 2.1792806503953986e-07,
      "logits/chosen": -3.125,
      "logits/rejected": -2.875,
      "logps/chosen": -704.0,
      "logps/rejected": -1160.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.125,
      "step": 8130
    },
    {
      "epoch": 0.5875983541471161,
      "grad_norm": 9.428346584688365,
      "learning_rate": 2.1730339139266368e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.75,
      "logps/chosen": -740.0,
      "logps/rejected": -1176.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.25,
      "step": 8140
    },
    {
      "epoch": 0.5883202194470512,
      "grad_norm": 9.348567817555846,
      "learning_rate": 2.1667892537014664e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.625,
      "logps/chosen": -736.0,
      "logps/rejected": -1128.0,
      "loss": 0.1673,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.6875,
      "step": 8150
    },
    {
      "epoch": 0.5890420847469862,
      "grad_norm": 9.578462386462538,
      "learning_rate": 2.1605467093736534e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.65625,
      "logps/chosen": -704.0,
      "logps/rejected": -1144.0,
      "loss": 0.1611,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.875,
      "step": 8160
    },
    {
      "epoch": 0.5897639500469213,
      "grad_norm": 7.084598817360508,
      "learning_rate": 2.1543063205835232e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.734375,
      "logps/chosen": -704.0,
      "logps/rejected": -1160.0,
      "loss": 0.1716,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.125,
      "step": 8170
    },
    {
      "epoch": 0.5904858153468563,
      "grad_norm": 10.096964364918204,
      "learning_rate": 2.1480681269577156e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -756.0,
      "logps/rejected": -1128.0,
      "loss": 0.1439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.6875,
      "step": 8180
    },
    {
      "epoch": 0.5912076806467913,
      "grad_norm": 13.036565026698682,
      "learning_rate": 2.141832168108932e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.765625,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1695,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.75,
      "step": 8190
    },
    {
      "epoch": 0.5919295459467263,
      "grad_norm": 10.503733261465367,
      "learning_rate": 2.1355984836356805e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.5625,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1803,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.8125,
      "step": 8200
    },
    {
      "epoch": 0.5926514112466613,
      "grad_norm": 8.837085032744609,
      "learning_rate": 2.1293671131220277e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.53125,
      "logps/chosen": -720.0,
      "logps/rejected": -1184.0,
      "loss": 0.1517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.125,
      "step": 8210
    },
    {
      "epoch": 0.5933732765465964,
      "grad_norm": 13.492509568178333,
      "learning_rate": 2.123138096137349e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -748.0,
      "logps/rejected": -1168.0,
      "loss": 0.1718,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.9375,
      "step": 8220
    },
    {
      "epoch": 0.5940951418465314,
      "grad_norm": 4.422649076334775,
      "learning_rate": 2.1169114722360708e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.46875,
      "logps/chosen": -716.0,
      "logps/rejected": -1208.0,
      "loss": 0.1322,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.5,
      "rewards/margins": 5.0,
      "rewards/rejected": -10.5,
      "step": 8230
    },
    {
      "epoch": 0.5948170071464665,
      "grad_norm": 8.207509821314797,
      "learning_rate": 2.1106872809574266e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -708.0,
      "logps/rejected": -1120.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.875,
      "step": 8240
    },
    {
      "epoch": 0.5955388724464015,
      "grad_norm": 11.51646939268645,
      "learning_rate": 2.1044655618251994e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -768.0,
      "logps/rejected": -1168.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.125,
      "step": 8250
    },
    {
      "epoch": 0.5962607377463366,
      "grad_norm": 8.81738149975298,
      "learning_rate": 2.098246354347477e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -704.0,
      "logps/rejected": -1088.0,
      "loss": 0.1499,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.5,
      "step": 8260
    },
    {
      "epoch": 0.5969826030462716,
      "grad_norm": 9.880728738377933,
      "learning_rate": 2.0920296980163975e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -672.0,
      "logps/rejected": -1112.0,
      "loss": 0.1828,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.09375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.375,
      "step": 8270
    },
    {
      "epoch": 0.5977044683462066,
      "grad_norm": 12.130445254705615,
      "learning_rate": 2.0858156323078986e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -696.0,
      "logps/rejected": -1104.0,
      "loss": 0.1695,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5,
      "step": 8280
    },
    {
      "epoch": 0.5984263336461416,
      "grad_norm": 9.18646238410699,
      "learning_rate": 2.0796041966814658e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.671875,
      "logps/chosen": -740.0,
      "logps/rejected": -1120.0,
      "loss": 0.1595,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.5,
      "step": 8290
    },
    {
      "epoch": 0.5991481989460766,
      "grad_norm": 7.4415035510148595,
      "learning_rate": 2.0733954305798884e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5,
      "logps/chosen": -688.0,
      "logps/rejected": -1088.0,
      "loss": 0.1668,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.375,
      "step": 8300
    },
    {
      "epoch": 0.5998700642460117,
      "grad_norm": 9.921182858419193,
      "learning_rate": 2.0671893734289984e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.609375,
      "logps/chosen": -680.0,
      "logps/rejected": -1080.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.4375,
      "step": 8310
    },
    {
      "epoch": 0.6005919295459468,
      "grad_norm": 6.738345944941375,
      "learning_rate": 2.0609860646374305e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -732.0,
      "logps/rejected": -1096.0,
      "loss": 0.1682,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.375,
      "step": 8320
    },
    {
      "epoch": 0.6013137948458818,
      "grad_norm": 9.581111671762294,
      "learning_rate": 2.054785543596363e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -660.0,
      "logps/rejected": -1104.0,
      "loss": 0.174,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.4375,
      "step": 8330
    },
    {
      "epoch": 0.6020356601458168,
      "grad_norm": 9.266451111611559,
      "learning_rate": 2.0485878496792758e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.671875,
      "logps/chosen": -684.0,
      "logps/rejected": -1088.0,
      "loss": 0.1594,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.25,
      "step": 8340
    },
    {
      "epoch": 0.6027575254457518,
      "grad_norm": 6.924514229924403,
      "learning_rate": 2.042393022241695e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -692.0,
      "logps/rejected": -1080.0,
      "loss": 0.1331,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.8125,
      "rewards/rejected": -9.1875,
      "step": 8350
    },
    {
      "epoch": 0.6034793907456869,
      "grad_norm": 11.878033012527993,
      "learning_rate": 2.0362011006209429e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.515625,
      "logps/chosen": -692.0,
      "logps/rejected": -1136.0,
      "loss": 0.168,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -9.8125,
      "step": 8360
    },
    {
      "epoch": 0.6042012560456219,
      "grad_norm": 6.456356558581018,
      "learning_rate": 2.0300121241358925e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.546875,
      "logps/chosen": -720.0,
      "logps/rejected": -1128.0,
      "loss": 0.1412,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.8125,
      "step": 8370
    },
    {
      "epoch": 0.6049231213455569,
      "grad_norm": 6.2766000485447595,
      "learning_rate": 2.023826132086714e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -732.0,
      "logps/rejected": -1120.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.625,
      "step": 8380
    },
    {
      "epoch": 0.6056449866454919,
      "grad_norm": 10.010107774455438,
      "learning_rate": 2.0176431637546243e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.78125,
      "logps/chosen": -712.0,
      "logps/rejected": -1112.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.75,
      "step": 8390
    },
    {
      "epoch": 0.6063668519454269,
      "grad_norm": 6.93155516769275,
      "learning_rate": 2.0114632584016444e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -756.0,
      "logps/rejected": -1160.0,
      "loss": 0.1442,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.0625,
      "step": 8400
    },
    {
      "epoch": 0.6070887172453621,
      "grad_norm": 9.44671796286649,
      "learning_rate": 2.0052864552703392e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -720.0,
      "logps/rejected": -1184.0,
      "loss": 0.1738,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.0,
      "step": 8410
    },
    {
      "epoch": 0.6078105825452971,
      "grad_norm": 10.256129872630872,
      "learning_rate": 1.9991127935835796e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -652.0,
      "logps/rejected": -1096.0,
      "loss": 0.1439,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -9.5,
      "step": 8420
    },
    {
      "epoch": 0.6085324478452321,
      "grad_norm": 10.112281066079031,
      "learning_rate": 1.9929423125442866e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -708.0,
      "logps/rejected": -1112.0,
      "loss": 0.1722,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5625,
      "step": 8430
    },
    {
      "epoch": 0.6092543131451671,
      "grad_norm": 5.989996318215951,
      "learning_rate": 1.9867750513351813e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -764.0,
      "logps/rejected": -1136.0,
      "loss": 0.1405,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.8125,
      "step": 8440
    },
    {
      "epoch": 0.6099761784451021,
      "grad_norm": 7.127345044430176,
      "learning_rate": 1.9806110491185434e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1168.0,
      "loss": 0.1401,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.9375,
      "step": 8450
    },
    {
      "epoch": 0.6106980437450372,
      "grad_norm": 8.882033575289205,
      "learning_rate": 1.9744503450359555e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.75,
      "step": 8460
    },
    {
      "epoch": 0.6114199090449722,
      "grad_norm": 11.636675089992194,
      "learning_rate": 1.968292978208055e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -736.0,
      "logps/rejected": -1144.0,
      "loss": 0.1555,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.0,
      "step": 8470
    },
    {
      "epoch": 0.6121417743449072,
      "grad_norm": 4.708987187458577,
      "learning_rate": 1.9621389877342926e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -692.0,
      "logps/rejected": -1152.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.0,
      "step": 8480
    },
    {
      "epoch": 0.6128636396448423,
      "grad_norm": 8.691036530455966,
      "learning_rate": 1.9559884126926742e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -748.0,
      "logps/rejected": -1120.0,
      "loss": 0.1662,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.75,
      "step": 8490
    },
    {
      "epoch": 0.6135855049447773,
      "grad_norm": 7.033771928824113,
      "learning_rate": 1.9498412921395196e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.515625,
      "logps/chosen": -704.0,
      "logps/rejected": -1120.0,
      "loss": 0.1484,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.625,
      "step": 8500
    },
    {
      "epoch": 0.6143073702447124,
      "grad_norm": 6.420805831736498,
      "learning_rate": 1.9436976651092142e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5,
      "logps/chosen": -644.0,
      "logps/rejected": -1112.0,
      "loss": 0.1517,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.90625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -9.5,
      "step": 8510
    },
    {
      "epoch": 0.6150292355446474,
      "grad_norm": 9.804155572476585,
      "learning_rate": 1.9375575706139557e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -684.0,
      "logps/rejected": -1080.0,
      "loss": 0.168,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.15625,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.375,
      "step": 8520
    },
    {
      "epoch": 0.6157511008445824,
      "grad_norm": 8.402725478167426,
      "learning_rate": 1.931421047643513e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.875,
      "logps/chosen": -704.0,
      "logps/rejected": -1096.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.4375,
      "step": 8530
    },
    {
      "epoch": 0.6164729661445174,
      "grad_norm": 7.928649293353565,
      "learning_rate": 1.9252881351649764e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -656.0,
      "logps/rejected": -1120.0,
      "loss": 0.1643,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.9375,
      "rewards/margins": 4.71875,
      "rewards/rejected": -9.6875,
      "step": 8540
    },
    {
      "epoch": 0.6171948314444524,
      "grad_norm": 10.584642277815288,
      "learning_rate": 1.919158872122505e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.609375,
      "logps/chosen": -656.0,
      "logps/rejected": -1088.0,
      "loss": 0.1709,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.0,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.375,
      "step": 8550
    },
    {
      "epoch": 0.6179166967443875,
      "grad_norm": 5.528300128432007,
      "learning_rate": 1.9130332974370888e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.6875,
      "step": 8560
    },
    {
      "epoch": 0.6186385620443225,
      "grad_norm": 8.381790511594774,
      "learning_rate": 1.9069114500062944e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.625,
      "logps/chosen": -700.0,
      "logps/rejected": -1112.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.5625,
      "step": 8570
    },
    {
      "epoch": 0.6193604273442576,
      "grad_norm": 7.672005385585843,
      "learning_rate": 1.9007933687040192e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -704.0,
      "logps/rejected": -1104.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.6875,
      "step": 8580
    },
    {
      "epoch": 0.6200822926441926,
      "grad_norm": 7.6497315518223505,
      "learning_rate": 1.8946790923802491e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.6875,
      "logps/chosen": -728.0,
      "logps/rejected": -1160.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.125,
      "step": 8590
    },
    {
      "epoch": 0.6208041579441276,
      "grad_norm": 8.83669969757212,
      "learning_rate": 1.8885686598608044e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1160.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.0625,
      "step": 8600
    },
    {
      "epoch": 0.6215260232440627,
      "grad_norm": 11.726451803340803,
      "learning_rate": 1.8824621099470986e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.65625,
      "logps/chosen": -744.0,
      "logps/rejected": -1152.0,
      "loss": 0.1516,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.9375,
      "step": 8610
    },
    {
      "epoch": 0.6222478885439977,
      "grad_norm": 6.926635938541544,
      "learning_rate": 1.8763594814158926e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1712,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.75,
      "step": 8620
    },
    {
      "epoch": 0.6229697538439327,
      "grad_norm": 29.047689387473163,
      "learning_rate": 1.8702608130190427e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -728.0,
      "logps/rejected": -1072.0,
      "loss": 0.1711,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.609375,
      "rewards/rejected": -9.3125,
      "step": 8630
    },
    {
      "epoch": 0.6236916191438677,
      "grad_norm": 12.046850711595459,
      "learning_rate": 1.8641661434832615e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1144.0,
      "loss": 0.1552,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.9375,
      "step": 8640
    },
    {
      "epoch": 0.6244134844438027,
      "grad_norm": 13.389050315360532,
      "learning_rate": 1.858075511509865e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.6875,
      "logps/chosen": -732.0,
      "logps/rejected": -1112.0,
      "loss": 0.1718,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.6875,
      "step": 8650
    },
    {
      "epoch": 0.6251353497437379,
      "grad_norm": 8.48405716596924,
      "learning_rate": 1.8519889557745355e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.578125,
      "logps/chosen": -732.0,
      "logps/rejected": -1136.0,
      "loss": 0.1627,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.8125,
      "step": 8660
    },
    {
      "epoch": 0.6258572150436729,
      "grad_norm": 7.718084904787592,
      "learning_rate": 1.8459065149270675e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -720.0,
      "logps/rejected": -1088.0,
      "loss": 0.204,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.4375,
      "step": 8670
    },
    {
      "epoch": 0.6265790803436079,
      "grad_norm": 8.645603679282008,
      "learning_rate": 1.8398282275911265e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.625,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.6875,
      "step": 8680
    },
    {
      "epoch": 0.6273009456435429,
      "grad_norm": 9.649146909089938,
      "learning_rate": 1.833754132364003e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.671875,
      "logps/chosen": -696.0,
      "logps/rejected": -1088.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.53125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.5,
      "step": 8690
    },
    {
      "epoch": 0.6280228109434779,
      "grad_norm": 10.143836034702232,
      "learning_rate": 1.8276842678163694e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -716.0,
      "logps/rejected": -1144.0,
      "loss": 0.1694,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.8125,
      "step": 8700
    },
    {
      "epoch": 0.628744676243413,
      "grad_norm": 6.224441882099418,
      "learning_rate": 1.82161867249203e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1184.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.1875,
      "step": 8710
    },
    {
      "epoch": 0.629466541543348,
      "grad_norm": 10.818946600627243,
      "learning_rate": 1.8155573849076829e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1673,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.875,
      "step": 8720
    },
    {
      "epoch": 0.630188406843283,
      "grad_norm": 8.600830363109214,
      "learning_rate": 1.8095004435526673e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.53125,
      "logps/chosen": -752.0,
      "logps/rejected": -1160.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.9375,
      "step": 8730
    },
    {
      "epoch": 0.630910272143218,
      "grad_norm": 7.278694825397051,
      "learning_rate": 1.8034478868887293e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1136.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.9375,
      "step": 8740
    },
    {
      "epoch": 0.631632137443153,
      "grad_norm": 10.061499794489972,
      "learning_rate": 1.7973997533497684e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -728.0,
      "logps/rejected": -1128.0,
      "loss": 0.1721,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.8125,
      "step": 8750
    },
    {
      "epoch": 0.6323540027430882,
      "grad_norm": 12.377386224781029,
      "learning_rate": 1.791356081341597e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -748.0,
      "logps/rejected": -1136.0,
      "loss": 0.152,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.75,
      "step": 8760
    },
    {
      "epoch": 0.6330758680430232,
      "grad_norm": 7.666515466883803,
      "learning_rate": 1.7853169092416992e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.875,
      "step": 8770
    },
    {
      "epoch": 0.6337977333429582,
      "grad_norm": 9.2022093037141,
      "learning_rate": 1.779282275398983e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.9375,
      "step": 8780
    },
    {
      "epoch": 0.6345195986428932,
      "grad_norm": 11.875630627682433,
      "learning_rate": 1.773252218133537e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1184.0,
      "loss": 0.1505,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.1875,
      "step": 8790
    },
    {
      "epoch": 0.6352414639428283,
      "grad_norm": 12.286853044802243,
      "learning_rate": 1.7672267757363917e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.75,
      "logps/chosen": -772.0,
      "logps/rejected": -1128.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.21875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -9.875,
      "step": 8800
    },
    {
      "epoch": 0.6359633292427633,
      "grad_norm": 7.782715805579312,
      "learning_rate": 1.7612059864692696e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.671875,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.1434,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.875,
      "step": 8810
    },
    {
      "epoch": 0.6366851945426983,
      "grad_norm": 10.338787396379217,
      "learning_rate": 1.7551898885643478e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.546875,
      "logps/chosen": -712.0,
      "logps/rejected": -1160.0,
      "loss": 0.1478,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.0625,
      "step": 8820
    },
    {
      "epoch": 0.6374070598426334,
      "grad_norm": 8.641921016163051,
      "learning_rate": 1.749178520224014e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.625,
      "logps/chosen": -728.0,
      "logps/rejected": -1152.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.0625,
      "step": 8830
    },
    {
      "epoch": 0.6381289251425684,
      "grad_norm": 9.128585650717545,
      "learning_rate": 1.7431719196206186e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.6875,
      "logps/chosen": -728.0,
      "logps/rejected": -1136.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.0,
      "step": 8840
    },
    {
      "epoch": 0.6388507904425035,
      "grad_norm": 11.694392580125832,
      "learning_rate": 1.7371701248962426e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1096.0,
      "loss": 0.1925,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 3.640625,
      "rewards/rejected": -9.5625,
      "step": 8850
    },
    {
      "epoch": 0.6395726557424385,
      "grad_norm": 8.29325313521127,
      "learning_rate": 1.7311731741624458e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.46875,
      "logps/chosen": -764.0,
      "logps/rejected": -1184.0,
      "loss": 0.15,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.25,
      "step": 8860
    },
    {
      "epoch": 0.6402945210423735,
      "grad_norm": 8.814547322060793,
      "learning_rate": 1.7251811055000283e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.671875,
      "logps/chosen": -720.0,
      "logps/rejected": -1168.0,
      "loss": 0.1451,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.125,
      "step": 8870
    },
    {
      "epoch": 0.6410163863423085,
      "grad_norm": 9.320265993143382,
      "learning_rate": 1.7191939569587923e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.65625,
      "logps/chosen": -724.0,
      "logps/rejected": -1152.0,
      "loss": 0.13,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.75,
      "step": 8880
    },
    {
      "epoch": 0.6417382516422435,
      "grad_norm": 10.838220312842498,
      "learning_rate": 1.7132117665572932e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.671875,
      "logps/chosen": -736.0,
      "logps/rejected": -1144.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.6875,
      "step": 8890
    },
    {
      "epoch": 0.6424601169421786,
      "grad_norm": 6.55136471958019,
      "learning_rate": 1.7072345722826035e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.59375,
      "logps/chosen": -696.0,
      "logps/rejected": -1104.0,
      "loss": 0.1315,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.4375,
      "step": 8900
    },
    {
      "epoch": 0.6431819822421136,
      "grad_norm": 8.66688500340224,
      "learning_rate": 1.701262412090072e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.796875,
      "logps/chosen": -696.0,
      "logps/rejected": -1112.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.625,
      "step": 8910
    },
    {
      "epoch": 0.6439038475420487,
      "grad_norm": 9.629379415169119,
      "learning_rate": 1.6952953239030779e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.734375,
      "logps/chosen": -720.0,
      "logps/rejected": -1160.0,
      "loss": 0.1416,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.125,
      "step": 8920
    },
    {
      "epoch": 0.6446257128419837,
      "grad_norm": 12.098110662139495,
      "learning_rate": 1.6893333456127946e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.796875,
      "logps/chosen": -752.0,
      "logps/rejected": -1176.0,
      "loss": 0.1484,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.125,
      "step": 8930
    },
    {
      "epoch": 0.6453475781419187,
      "grad_norm": 8.256462573405791,
      "learning_rate": 1.6833765150779484e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.8125,
      "logps/chosen": -724.0,
      "logps/rejected": -1104.0,
      "loss": 0.1609,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.6875,
      "step": 8940
    },
    {
      "epoch": 0.6460694434418538,
      "grad_norm": 11.239144635721424,
      "learning_rate": 1.677424870124575e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.75,
      "logps/chosen": -688.0,
      "logps/rejected": -1096.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.375,
      "step": 8950
    },
    {
      "epoch": 0.6467913087417888,
      "grad_norm": 8.418303487457777,
      "learning_rate": 1.671478448545784e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.765625,
      "logps/chosen": -708.0,
      "logps/rejected": -1128.0,
      "loss": 0.149,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.625,
      "step": 8960
    },
    {
      "epoch": 0.6475131740417238,
      "grad_norm": 9.492259339751383,
      "learning_rate": 1.665537288101514e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -724.0,
      "logps/rejected": -1104.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.625,
      "step": 8970
    },
    {
      "epoch": 0.6482350393416588,
      "grad_norm": 7.953989568068451,
      "learning_rate": 1.6596014265182957e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1683,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.875,
      "step": 8980
    },
    {
      "epoch": 0.6489569046415938,
      "grad_norm": 7.299757937508117,
      "learning_rate": 1.6536709014890147e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -712.0,
      "logps/rejected": -1096.0,
      "loss": 0.1577,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.5,
      "step": 8990
    },
    {
      "epoch": 0.649678769941529,
      "grad_norm": 9.426751785884438,
      "learning_rate": 1.6477457506726655e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -716.0,
      "logps/rejected": -1104.0,
      "loss": 0.1404,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.375,
      "step": 9000
    },
    {
      "epoch": 0.650400635241464,
      "grad_norm": 10.774886673375532,
      "learning_rate": 1.641826011694119e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.75,
      "logps/chosen": -736.0,
      "logps/rejected": -1120.0,
      "loss": 0.1533,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.5,
      "step": 9010
    },
    {
      "epoch": 0.651122500541399,
      "grad_norm": 9.624000604333325,
      "learning_rate": 1.6359117221438784e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -728.0,
      "logps/rejected": -1144.0,
      "loss": 0.1508,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.9375,
      "step": 9020
    },
    {
      "epoch": 0.651844365841334,
      "grad_norm": 9.17261641216877,
      "learning_rate": 1.6300029195778453e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -672.0,
      "logps/rejected": -1112.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.40625,
      "rewards/rejected": -9.625,
      "step": 9030
    },
    {
      "epoch": 0.652566231141269,
      "grad_norm": 10.974103824549372,
      "learning_rate": 1.624099641517078e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.6875,
      "logps/chosen": -692.0,
      "logps/rejected": -1120.0,
      "loss": 0.1616,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.5625,
      "step": 9040
    },
    {
      "epoch": 0.6532880964412041,
      "grad_norm": 6.917901057511971,
      "learning_rate": 1.6182019254475514e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.5625,
      "logps/chosen": -684.0,
      "logps/rejected": -1136.0,
      "loss": 0.1515,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -9.75,
      "step": 9050
    },
    {
      "epoch": 0.6540099617411391,
      "grad_norm": 10.04014608485764,
      "learning_rate": 1.6123098088199267e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.640625,
      "logps/chosen": -728.0,
      "logps/rejected": -1144.0,
      "loss": 0.1476,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0,
      "step": 9060
    },
    {
      "epoch": 0.6547318270410741,
      "grad_norm": 11.307618522526226,
      "learning_rate": 1.6064233290493048e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.640625,
      "logps/chosen": -744.0,
      "logps/rejected": -1160.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.1875,
      "step": 9070
    },
    {
      "epoch": 0.6554536923410091,
      "grad_norm": 11.567312788735324,
      "learning_rate": 1.600542523514992e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -736.0,
      "logps/rejected": -1168.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.0,
      "step": 9080
    },
    {
      "epoch": 0.6561755576409442,
      "grad_norm": 6.469353343465349,
      "learning_rate": 1.5946674295602668e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -688.0,
      "logps/rejected": -1096.0,
      "loss": 0.1644,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.625,
      "step": 9090
    },
    {
      "epoch": 0.6568974229408793,
      "grad_norm": 9.939174768893757,
      "learning_rate": 1.5887980844921338e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -708.0,
      "logps/rejected": -1104.0,
      "loss": 0.1646,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.625,
      "step": 9100
    },
    {
      "epoch": 0.6576192882408143,
      "grad_norm": 10.710746815227962,
      "learning_rate": 1.5829345255810966e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -704.0,
      "logps/rejected": -1096.0,
      "loss": 0.1553,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.5,
      "step": 9110
    },
    {
      "epoch": 0.6583411535407493,
      "grad_norm": 9.589016893008033,
      "learning_rate": 1.5770767900609144e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -696.0,
      "logps/rejected": -1120.0,
      "loss": 0.1403,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.53125,
      "rewards/rejected": -9.75,
      "step": 9120
    },
    {
      "epoch": 0.6590630188406843,
      "grad_norm": 10.535090311502815,
      "learning_rate": 1.5712249151283674e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.65625,
      "logps/chosen": -672.0,
      "logps/rejected": -1120.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -9.6875,
      "step": 9130
    },
    {
      "epoch": 0.6597848841406193,
      "grad_norm": 11.07484592479438,
      "learning_rate": 1.565378937943022e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.875,
      "logps/chosen": -720.0,
      "logps/rejected": -1104.0,
      "loss": 0.1606,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.125,
      "rewards/rejected": -9.625,
      "step": 9140
    },
    {
      "epoch": 0.6605067494405544,
      "grad_norm": 8.804459530019093,
      "learning_rate": 1.559538895626994e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.640625,
      "logps/chosen": -720.0,
      "logps/rejected": -1128.0,
      "loss": 0.163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.8125,
      "step": 9150
    },
    {
      "epoch": 0.6612286147404894,
      "grad_norm": 9.869851207701721,
      "learning_rate": 1.5537048252647102e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.703125,
      "logps/chosen": -728.0,
      "logps/rejected": -1136.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.9375,
      "step": 9160
    },
    {
      "epoch": 0.6619504800404244,
      "grad_norm": 7.227577491074705,
      "learning_rate": 1.547876763902679e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.703125,
      "logps/chosen": -732.0,
      "logps/rejected": -1152.0,
      "loss": 0.1604,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.0,
      "step": 9170
    },
    {
      "epoch": 0.6626723453403595,
      "grad_norm": 8.512681174359171,
      "learning_rate": 1.5420547485492483e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -732.0,
      "logps/rejected": -1176.0,
      "loss": 0.1477,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.0625,
      "step": 9180
    },
    {
      "epoch": 0.6633942106402945,
      "grad_norm": 6.940912396524942,
      "learning_rate": 1.5362388161743743e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.65625,
      "logps/chosen": -728.0,
      "logps/rejected": -1152.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.0,
      "step": 9190
    },
    {
      "epoch": 0.6641160759402296,
      "grad_norm": 7.482638973278819,
      "learning_rate": 1.5304290037093887e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.75,
      "logps/chosen": -732.0,
      "logps/rejected": -1128.0,
      "loss": 0.1642,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.6875,
      "step": 9200
    },
    {
      "epoch": 0.6648379412401646,
      "grad_norm": 9.239067837458254,
      "learning_rate": 1.524625348046758e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.75,
      "logps/chosen": -708.0,
      "logps/rejected": -1136.0,
      "loss": 0.1324,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.875,
      "step": 9210
    },
    {
      "epoch": 0.6655598065400996,
      "grad_norm": 10.312943276531238,
      "learning_rate": 1.5188278860398543e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.75,
      "logps/chosen": -736.0,
      "logps/rejected": -1200.0,
      "loss": 0.1548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.375,
      "step": 9220
    },
    {
      "epoch": 0.6662816718400346,
      "grad_norm": 12.377500015344213,
      "learning_rate": 1.5130366545027215e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.609375,
      "logps/chosen": -736.0,
      "logps/rejected": -1184.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.0,
      "step": 9230
    },
    {
      "epoch": 0.6670035371399696,
      "grad_norm": 11.437994745400522,
      "learning_rate": 1.507251690209837e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.703125,
      "logps/chosen": -708.0,
      "logps/rejected": -1120.0,
      "loss": 0.1722,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.5625,
      "step": 9240
    },
    {
      "epoch": 0.6677254024399047,
      "grad_norm": 9.227052828667116,
      "learning_rate": 1.5014730298958834e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -688.0,
      "logps/rejected": -1088.0,
      "loss": 0.1733,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.4375,
      "step": 9250
    },
    {
      "epoch": 0.6684472677398398,
      "grad_norm": 11.34701390652657,
      "learning_rate": 1.4957007102555101e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.703125,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.625,
      "step": 9260
    },
    {
      "epoch": 0.6691691330397748,
      "grad_norm": 10.840533844995983,
      "learning_rate": 1.489934767943106e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.6875,
      "logps/chosen": -700.0,
      "logps/rejected": -1136.0,
      "loss": 0.1507,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.40625,
      "rewards/rejected": -9.9375,
      "step": 9270
    },
    {
      "epoch": 0.6698909983397098,
      "grad_norm": 5.005020117833695,
      "learning_rate": 1.484175239572563e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.734375,
      "logps/chosen": -732.0,
      "logps/rejected": -1184.0,
      "loss": 0.1538,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.1875,
      "step": 9280
    },
    {
      "epoch": 0.6706128636396449,
      "grad_norm": 9.438942465976393,
      "learning_rate": 1.4784221617170427e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.6875,
      "logps/chosen": -672.0,
      "logps/rejected": -1112.0,
      "loss": 0.1544,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -9.6875,
      "step": 9290
    },
    {
      "epoch": 0.6713347289395799,
      "grad_norm": 8.775509056317704,
      "learning_rate": 1.4726755709087467e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.5625,
      "logps/chosen": -708.0,
      "logps/rejected": -1128.0,
      "loss": 0.1568,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.6875,
      "step": 9300
    },
    {
      "epoch": 0.6720565942395149,
      "grad_norm": 12.815148122117906,
      "learning_rate": 1.466935503638686e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.59375,
      "logps/chosen": -708.0,
      "logps/rejected": -1168.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.0625,
      "step": 9310
    },
    {
      "epoch": 0.6727784595394499,
      "grad_norm": 6.322537492099079,
      "learning_rate": 1.4612019963564425e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.703125,
      "logps/chosen": -716.0,
      "logps/rejected": -1112.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.625,
      "step": 9320
    },
    {
      "epoch": 0.6735003248393849,
      "grad_norm": 8.684754690608713,
      "learning_rate": 1.4554750854699454e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.734375,
      "logps/chosen": -684.0,
      "logps/rejected": -1064.0,
      "loss": 0.1582,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.921875,
      "rewards/rejected": -9.1875,
      "step": 9330
    },
    {
      "epoch": 0.67422219013932,
      "grad_norm": 7.495450070953094,
      "learning_rate": 1.4497548073452353e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.78125,
      "logps/chosen": -688.0,
      "logps/rejected": -1112.0,
      "loss": 0.1404,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -9.625,
      "step": 9340
    },
    {
      "epoch": 0.6749440554392551,
      "grad_norm": 9.818417235804333,
      "learning_rate": 1.444041198306235e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -680.0,
      "logps/rejected": -1088.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.4375,
      "step": 9350
    },
    {
      "epoch": 0.6756659207391901,
      "grad_norm": 10.0125757389507,
      "learning_rate": 1.4383342946345178e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -724.0,
      "logps/rejected": -1184.0,
      "loss": 0.1533,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.25,
      "step": 9360
    },
    {
      "epoch": 0.6763877860391251,
      "grad_norm": 10.056618279215506,
      "learning_rate": 1.432634132569079e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.71875,
      "logps/chosen": -744.0,
      "logps/rejected": -1176.0,
      "loss": 0.1534,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.1875,
      "step": 9370
    },
    {
      "epoch": 0.6771096513390601,
      "grad_norm": 12.261509335918284,
      "learning_rate": 1.4269407483061025e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -736.0,
      "logps/rejected": -1192.0,
      "loss": 0.1616,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.25,
      "step": 9380
    },
    {
      "epoch": 0.6778315166389952,
      "grad_norm": 9.583901573024836,
      "learning_rate": 1.4212541779987358e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.1808,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.875,
      "step": 9390
    },
    {
      "epoch": 0.6785533819389302,
      "grad_norm": 7.602147425073978,
      "learning_rate": 1.415574457756853e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.625,
      "logps/chosen": -720.0,
      "logps/rejected": -1112.0,
      "loss": 0.1429,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.625,
      "step": 9400
    },
    {
      "epoch": 0.6792752472388652,
      "grad_norm": 5.715088913700748,
      "learning_rate": 1.409901623646837e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -708.0,
      "logps/rejected": -1120.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.625,
      "step": 9410
    },
    {
      "epoch": 0.6799971125388002,
      "grad_norm": 7.623897333532847,
      "learning_rate": 1.4042357116913365e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -676.0,
      "logps/rejected": -1112.0,
      "loss": 0.1444,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.625,
      "step": 9420
    },
    {
      "epoch": 0.6807189778387353,
      "grad_norm": 12.781776965070957,
      "learning_rate": 1.3985767578690474e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.640625,
      "logps/chosen": -716.0,
      "logps/rejected": -1088.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -9.4375,
      "step": 9430
    },
    {
      "epoch": 0.6814408431386704,
      "grad_norm": 10.237854404923459,
      "learning_rate": 1.3929247981144845e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.625,
      "logps/chosen": -728.0,
      "logps/rejected": -1160.0,
      "loss": 0.1627,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.1875,
      "step": 9440
    },
    {
      "epoch": 0.6821627084386054,
      "grad_norm": 9.894822207763886,
      "learning_rate": 1.387279868317744e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1224.0,
      "loss": 0.1477,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.4375,
      "step": 9450
    },
    {
      "epoch": 0.6828845737385404,
      "grad_norm": 9.241271465537533,
      "learning_rate": 1.3816420043242843e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.703125,
      "logps/chosen": -708.0,
      "logps/rejected": -1136.0,
      "loss": 0.1488,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.8125,
      "step": 9460
    },
    {
      "epoch": 0.6836064390384754,
      "grad_norm": 10.237589705127691,
      "learning_rate": 1.3760112419346986e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.6875,
      "logps/chosen": -748.0,
      "logps/rejected": -1176.0,
      "loss": 0.156,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.25,
      "step": 9470
    },
    {
      "epoch": 0.6843283043384104,
      "grad_norm": 10.725962046372151,
      "learning_rate": 1.3703876169044802e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.71875,
      "logps/chosen": -756.0,
      "logps/rejected": -1136.0,
      "loss": 0.1511,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.875,
      "step": 9480
    },
    {
      "epoch": 0.6850501696383455,
      "grad_norm": 7.6917366331039325,
      "learning_rate": 1.364771164943802e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -720.0,
      "logps/rejected": -1120.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.75,
      "step": 9490
    },
    {
      "epoch": 0.6857720349382805,
      "grad_norm": 8.728014646166761,
      "learning_rate": 1.3591619217172883e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.578125,
      "logps/chosen": -716.0,
      "logps/rejected": -1144.0,
      "loss": 0.1455,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.5,
      "rewards/rejected": -9.875,
      "step": 9500
    },
    {
      "epoch": 0.6864939002382155,
      "grad_norm": 10.677477412212683,
      "learning_rate": 1.3535599228437867e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.640625,
      "logps/chosen": -716.0,
      "logps/rejected": -1144.0,
      "loss": 0.1626,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.9375,
      "step": 9510
    },
    {
      "epoch": 0.6872157655381506,
      "grad_norm": 9.38419202809167,
      "learning_rate": 1.3479652038961432e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.609375,
      "logps/chosen": -748.0,
      "logps/rejected": -1128.0,
      "loss": 0.1679,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.75,
      "step": 9520
    },
    {
      "epoch": 0.6879376308380856,
      "grad_norm": 8.766505031785163,
      "learning_rate": 1.3423778004009762e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.6875,
      "logps/chosen": -712.0,
      "logps/rejected": -1136.0,
      "loss": 0.1426,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.40625,
      "rewards/rejected": -9.875,
      "step": 9530
    },
    {
      "epoch": 0.6886594961380207,
      "grad_norm": 7.070381554572682,
      "learning_rate": 1.3367977478384513e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.8125,
      "step": 9540
    },
    {
      "epoch": 0.6893813614379557,
      "grad_norm": 13.363793450146552,
      "learning_rate": 1.3312250816420542e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1128.0,
      "loss": 0.1678,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -9.875,
      "step": 9550
    },
    {
      "epoch": 0.6901032267378907,
      "grad_norm": 6.54534750827785,
      "learning_rate": 1.3256598371983686e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.5625,
      "logps/chosen": -716.0,
      "logps/rejected": -1176.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.125,
      "step": 9560
    },
    {
      "epoch": 0.6908250920378257,
      "grad_norm": 8.065890871247143,
      "learning_rate": 1.320102049846849e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.703125,
      "logps/chosen": -712.0,
      "logps/rejected": -1144.0,
      "loss": 0.1447,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.4375,
      "rewards/margins": 4.5,
      "rewards/rejected": -9.9375,
      "step": 9570
    },
    {
      "epoch": 0.6915469573377607,
      "grad_norm": 8.464118508817007,
      "learning_rate": 1.314551754879595e-07,
      "logits/chosen": -3.140625,
      "logits/rejected": -2.796875,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1415,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.0,
      "step": 9580
    },
    {
      "epoch": 0.6922688226376958,
      "grad_norm": 9.854707772376594,
      "learning_rate": 1.3090089875411337e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.6875,
      "logps/chosen": -764.0,
      "logps/rejected": -1256.0,
      "loss": 0.129,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.8125,
      "step": 9590
    },
    {
      "epoch": 0.6929906879376309,
      "grad_norm": 8.672649571433473,
      "learning_rate": 1.30347378302819e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.75,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1698,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.5,
      "step": 9600
    },
    {
      "epoch": 0.6937125532375659,
      "grad_norm": 9.034737955151657,
      "learning_rate": 1.2979461764894608e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.78125,
      "logps/chosen": -768.0,
      "logps/rejected": -1176.0,
      "loss": 0.163,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.1875,
      "step": 9610
    },
    {
      "epoch": 0.6944344185375009,
      "grad_norm": 8.90041609096791,
      "learning_rate": 1.2924262030254022e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.6875,
      "logps/chosen": -728.0,
      "logps/rejected": -1168.0,
      "loss": 0.1499,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.1875,
      "step": 9620
    },
    {
      "epoch": 0.6951562838374359,
      "grad_norm": 8.665967459471112,
      "learning_rate": 1.2869138976879961e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.703125,
      "logps/chosen": -768.0,
      "logps/rejected": -1176.0,
      "loss": 0.1636,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 9630
    },
    {
      "epoch": 0.695878149137371,
      "grad_norm": 7.497535290815869,
      "learning_rate": 1.2814092954805294e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -740.0,
      "logps/rejected": -1176.0,
      "loss": 0.171,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.375,
      "step": 9640
    },
    {
      "epoch": 0.696600014437306,
      "grad_norm": 9.990492914302166,
      "learning_rate": 1.275912431357381e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.703125,
      "logps/chosen": -760.0,
      "logps/rejected": -1136.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.0,
      "step": 9650
    },
    {
      "epoch": 0.697321879737241,
      "grad_norm": 7.823463866960965,
      "learning_rate": 1.2704233402237858e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -728.0,
      "logps/rejected": -1112.0,
      "loss": 0.1707,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.5625,
      "step": 9660
    },
    {
      "epoch": 0.698043745037176,
      "grad_norm": 7.579915303042644,
      "learning_rate": 1.2649420569356217e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.71875,
      "logps/chosen": -696.0,
      "logps/rejected": -1160.0,
      "loss": 0.1607,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.0625,
      "step": 9670
    },
    {
      "epoch": 0.698765610337111,
      "grad_norm": 8.800622349499235,
      "learning_rate": 1.2594686162991914e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.78125,
      "logps/chosen": -700.0,
      "logps/rejected": -1120.0,
      "loss": 0.1402,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.8125,
      "step": 9680
    },
    {
      "epoch": 0.6994874756370462,
      "grad_norm": 9.327395207500006,
      "learning_rate": 1.2540030530709888e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.78125,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.75,
      "step": 9690
    },
    {
      "epoch": 0.7002093409369812,
      "grad_norm": 8.376068930149511,
      "learning_rate": 1.248545401957492e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.59375,
      "logps/chosen": -712.0,
      "logps/rejected": -1144.0,
      "loss": 0.1543,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.875,
      "step": 9700
    },
    {
      "epoch": 0.7009312062369162,
      "grad_norm": 7.936678196185944,
      "learning_rate": 1.2430956976149345e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.6875,
      "logps/chosen": -680.0,
      "logps/rejected": -1136.0,
      "loss": 0.1465,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 4.5625,
      "rewards/rejected": -9.875,
      "step": 9710
    },
    {
      "epoch": 0.7016530715368512,
      "grad_norm": 12.345046600610592,
      "learning_rate": 1.2376539746490878e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.78125,
      "logps/chosen": -740.0,
      "logps/rejected": -1184.0,
      "loss": 0.1713,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.3125,
      "step": 9720
    },
    {
      "epoch": 0.7023749368367862,
      "grad_norm": 9.162811933963393,
      "learning_rate": 1.2322202676150417e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.765625,
      "logps/chosen": -720.0,
      "logps/rejected": -1168.0,
      "loss": 0.1654,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.125,
      "step": 9730
    },
    {
      "epoch": 0.7030968021367213,
      "grad_norm": 10.256111775264358,
      "learning_rate": 1.2267946110169847e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.78125,
      "logps/chosen": -724.0,
      "logps/rejected": -1128.0,
      "loss": 0.1444,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -9.8125,
      "step": 9740
    },
    {
      "epoch": 0.7038186674366563,
      "grad_norm": 8.584246074026865,
      "learning_rate": 1.2213770393079846e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.703125,
      "logps/chosen": -732.0,
      "logps/rejected": -1152.0,
      "loss": 0.1413,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.9375,
      "step": 9750
    },
    {
      "epoch": 0.7045405327365913,
      "grad_norm": 7.583949089234293,
      "learning_rate": 1.21596758688977e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.4375,
      "logps/chosen": -712.0,
      "logps/rejected": -1152.0,
      "loss": 0.1512,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.375,
      "rewards/rejected": -9.9375,
      "step": 9760
    },
    {
      "epoch": 0.7052623980365264,
      "grad_norm": 9.44743736005493,
      "learning_rate": 1.2105662881125112e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.53125,
      "logps/chosen": -720.0,
      "logps/rejected": -1176.0,
      "loss": 0.1652,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.125,
      "step": 9770
    },
    {
      "epoch": 0.7059842633364615,
      "grad_norm": 6.4078324266348705,
      "learning_rate": 1.205173177274604e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -708.0,
      "logps/rejected": -1136.0,
      "loss": 0.1521,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.75,
      "step": 9780
    },
    {
      "epoch": 0.7067061286363965,
      "grad_norm": 7.907790182080389,
      "learning_rate": 1.1997882886224497e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -728.0,
      "logps/rejected": -1144.0,
      "loss": 0.1424,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.0,
      "step": 9790
    },
    {
      "epoch": 0.7074279939363315,
      "grad_norm": 12.95788224606365,
      "learning_rate": 1.194411656350238e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -736.0,
      "logps/rejected": -1144.0,
      "loss": 0.1558,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0,
      "step": 9800
    },
    {
      "epoch": 0.7081498592362665,
      "grad_norm": 9.166806155356193,
      "learning_rate": 1.1890433145997327e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -716.0,
      "logps/rejected": -1120.0,
      "loss": 0.1731,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.8125,
      "step": 9810
    },
    {
      "epoch": 0.7088717245362015,
      "grad_norm": 9.72249270328465,
      "learning_rate": 1.1836832974600483e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.515625,
      "logps/chosen": -720.0,
      "logps/rejected": -1136.0,
      "loss": 0.1635,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.9375,
      "step": 9820
    },
    {
      "epoch": 0.7095935898361366,
      "grad_norm": 7.965526025357869,
      "learning_rate": 1.1783316389674406e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.6875,
      "logps/chosen": -728.0,
      "logps/rejected": -1120.0,
      "loss": 0.1514,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.5625,
      "step": 9830
    },
    {
      "epoch": 0.7103154551360716,
      "grad_norm": 7.929357398986534,
      "learning_rate": 1.1729883731050899e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -720.0,
      "logps/rejected": -1128.0,
      "loss": 0.1315,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -9.875,
      "step": 9840
    },
    {
      "epoch": 0.7110373204360066,
      "grad_norm": 8.634524210955284,
      "learning_rate": 1.1676535338028781e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -776.0,
      "logps/rejected": -1176.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.1875,
      "step": 9850
    },
    {
      "epoch": 0.7117591857359417,
      "grad_norm": 9.065771203848671,
      "learning_rate": 1.1623271549371808e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.625,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1781,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.5,
      "step": 9860
    },
    {
      "epoch": 0.7124810510358767,
      "grad_norm": 7.949345789146648,
      "learning_rate": 1.1570092703306523e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -752.0,
      "logps/rejected": -1200.0,
      "loss": 0.1667,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.3125,
      "step": 9870
    },
    {
      "epoch": 0.7132029163358118,
      "grad_norm": 8.559039194521684,
      "learning_rate": 1.1516999137520023e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -752.0,
      "logps/rejected": -1152.0,
      "loss": 0.1439,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.0625,
      "step": 9880
    },
    {
      "epoch": 0.7139247816357468,
      "grad_norm": 8.830447748470409,
      "learning_rate": 1.1463991189157917e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1152.0,
      "loss": 0.1701,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0,
      "step": 9890
    },
    {
      "epoch": 0.7146466469356818,
      "grad_norm": 7.4835869108530275,
      "learning_rate": 1.1411069194822124e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.71875,
      "logps/chosen": -736.0,
      "logps/rejected": -1168.0,
      "loss": 0.1413,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0625,
      "step": 9900
    },
    {
      "epoch": 0.7153685122356168,
      "grad_norm": 12.056792194890452,
      "learning_rate": 1.1358233490568758e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.640625,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1486,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.0,
      "step": 9910
    },
    {
      "epoch": 0.7160903775355518,
      "grad_norm": 7.752875524749948,
      "learning_rate": 1.1305484411905986e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1224.0,
      "loss": 0.1407,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.6875,
      "step": 9920
    },
    {
      "epoch": 0.716812242835487,
      "grad_norm": 7.773726914737785,
      "learning_rate": 1.12528222937919e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -748.0,
      "logps/rejected": -1184.0,
      "loss": 0.1499,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.4375,
      "step": 9930
    },
    {
      "epoch": 0.717534108135422,
      "grad_norm": 10.91086916613174,
      "learning_rate": 1.1200247470632392e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.484375,
      "logps/chosen": -784.0,
      "logps/rejected": -1200.0,
      "loss": 0.1779,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.4375,
      "step": 9940
    },
    {
      "epoch": 0.718255973435357,
      "grad_norm": 8.136219531246716,
      "learning_rate": 1.1147760276279028e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -772.0,
      "logps/rejected": -1168.0,
      "loss": 0.153,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.0,
      "rewards/rejected": -10.125,
      "step": 9950
    },
    {
      "epoch": 0.718977838735292,
      "grad_norm": 12.213084226027048,
      "learning_rate": 1.1095361044026927e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -760.0,
      "logps/rejected": -1144.0,
      "loss": 0.1488,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.875,
      "step": 9960
    },
    {
      "epoch": 0.719699704035227,
      "grad_norm": 7.447846242333707,
      "learning_rate": 1.1043050106612656e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.671875,
      "logps/chosen": -764.0,
      "logps/rejected": -1136.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.9375,
      "step": 9970
    },
    {
      "epoch": 0.7204215693351621,
      "grad_norm": 11.497519578105333,
      "learning_rate": 1.0990827796212074e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.46875,
      "logps/chosen": -748.0,
      "logps/rejected": -1168.0,
      "loss": 0.163,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.25,
      "step": 9980
    },
    {
      "epoch": 0.7211434346350971,
      "grad_norm": 9.931460815710798,
      "learning_rate": 1.0938694444438307e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.546875,
      "logps/chosen": -720.0,
      "logps/rejected": -1152.0,
      "loss": 0.1411,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.0,
      "step": 9990
    },
    {
      "epoch": 0.7218652999350321,
      "grad_norm": 9.875757410099292,
      "learning_rate": 1.0886650382339566e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.625,
      "logps/chosen": -740.0,
      "logps/rejected": -1152.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.0625,
      "step": 10000
    },
    {
      "epoch": 0.7218652999350321,
      "eval_logits/chosen": -2.9375,
      "eval_logits/rejected": -2.625,
      "eval_logps/chosen": -772.0,
      "eval_logps/rejected": -1168.0,
      "eval_loss": 0.21195410192012787,
      "eval_rewards/accuracies": 0.9136363863945007,
      "eval_rewards/chosen": -6.0,
      "eval_rewards/margins": 4.1875,
      "eval_rewards/rejected": -10.1875,
      "eval_runtime": 3594.9603,
      "eval_samples_per_second": 27.4,
      "eval_steps_per_second": 0.428,
      "step": 10000
    },
    {
      "epoch": 0.7225871652349671,
      "grad_norm": 9.241595039747112,
      "learning_rate": 1.083469594039704e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.640625,
      "logps/chosen": -736.0,
      "logps/rejected": -1184.0,
      "loss": 0.1528,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.25,
      "step": 10010
    },
    {
      "epoch": 0.7233090305349021,
      "grad_norm": 10.979124343100237,
      "learning_rate": 1.0782831448522886e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.640625,
      "logps/chosen": -760.0,
      "logps/rejected": -1184.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.3125,
      "step": 10020
    },
    {
      "epoch": 0.7240308958348373,
      "grad_norm": 9.941856346590853,
      "learning_rate": 1.0731057236058047e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5,
      "logps/chosen": -756.0,
      "logps/rejected": -1152.0,
      "loss": 0.1618,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.0,
      "rewards/rejected": -10.0,
      "step": 10030
    },
    {
      "epoch": 0.7247527611347723,
      "grad_norm": 10.561997010522104,
      "learning_rate": 1.0679373631770162e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.59375,
      "logps/chosen": -736.0,
      "logps/rejected": -1184.0,
      "loss": 0.1482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.25,
      "step": 10040
    },
    {
      "epoch": 0.7254746264347073,
      "grad_norm": 8.458140492638268,
      "learning_rate": 1.0627780963851562e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -740.0,
      "logps/rejected": -1152.0,
      "loss": 0.1639,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.125,
      "step": 10050
    },
    {
      "epoch": 0.7261964917346423,
      "grad_norm": 7.155521990324457,
      "learning_rate": 1.0576279559917081e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -748.0,
      "logps/rejected": -1152.0,
      "loss": 0.1383,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -9.875,
      "step": 10060
    },
    {
      "epoch": 0.7269183570345773,
      "grad_norm": 9.028193158327456,
      "learning_rate": 1.0524869747002041e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.5625,
      "logps/chosen": -688.0,
      "logps/rejected": -1152.0,
      "loss": 0.1405,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.0625,
      "step": 10070
    },
    {
      "epoch": 0.7276402223345124,
      "grad_norm": 7.250067406009502,
      "learning_rate": 1.0473551851560184e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.640625,
      "logps/chosen": -756.0,
      "logps/rejected": -1160.0,
      "loss": 0.1434,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.0625,
      "step": 10080
    },
    {
      "epoch": 0.7283620876344474,
      "grad_norm": 13.036104853884511,
      "learning_rate": 1.0422326199461526e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1176.0,
      "loss": 0.1419,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.1875,
      "step": 10090
    },
    {
      "epoch": 0.7290839529343824,
      "grad_norm": 11.132862883187533,
      "learning_rate": 1.0371193115990362e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1232.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.78125,
      "rewards/margins": 5.0,
      "rewards/rejected": -10.8125,
      "step": 10100
    },
    {
      "epoch": 0.7298058182343174,
      "grad_norm": 11.150633604056837,
      "learning_rate": 1.0320152925843192e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.671875,
      "logps/chosen": -736.0,
      "logps/rejected": -1176.0,
      "loss": 0.1702,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.25,
      "step": 10110
    },
    {
      "epoch": 0.7305276835342525,
      "grad_norm": 9.295923421645263,
      "learning_rate": 1.02692059531266e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -724.0,
      "logps/rejected": -1136.0,
      "loss": 0.1433,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -9.75,
      "step": 10120
    },
    {
      "epoch": 0.7312495488341876,
      "grad_norm": 10.553464062235404,
      "learning_rate": 1.0218352521355262e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1200.0,
      "loss": 0.1472,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.4375,
      "step": 10130
    },
    {
      "epoch": 0.7319714141341226,
      "grad_norm": 7.390985068640905,
      "learning_rate": 1.0167592953449858e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.25,
      "step": 10140
    },
    {
      "epoch": 0.7326932794340576,
      "grad_norm": 9.445556995082786,
      "learning_rate": 1.011692757173504e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.578125,
      "logps/chosen": -768.0,
      "logps/rejected": -1232.0,
      "loss": 0.1839,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.5625,
      "step": 10150
    },
    {
      "epoch": 0.7334151447339926,
      "grad_norm": 12.596474294167802,
      "learning_rate": 1.0066356697937362e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -776.0,
      "logps/rejected": -1232.0,
      "loss": 0.1564,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.875,
      "step": 10160
    },
    {
      "epoch": 0.7341370100339276,
      "grad_norm": 11.44354948885957,
      "learning_rate": 1.0015880653183259e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.3125,
      "step": 10170
    },
    {
      "epoch": 0.7348588753338627,
      "grad_norm": 7.4670272071034836,
      "learning_rate": 9.965499757996997e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1144.0,
      "loss": 0.1356,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.9375,
      "step": 10180
    },
    {
      "epoch": 0.7355807406337977,
      "grad_norm": 9.061126614577189,
      "learning_rate": 9.915214332298638e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.53125,
      "logps/chosen": -740.0,
      "logps/rejected": -1192.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.4375,
      "step": 10190
    },
    {
      "epoch": 0.7363026059337328,
      "grad_norm": 10.142460768085758,
      "learning_rate": 9.865024695402014e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -740.0,
      "logps/rejected": -1176.0,
      "loss": 0.1474,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.0625,
      "step": 10200
    },
    {
      "epoch": 0.7370244712336678,
      "grad_norm": 9.026651543187556,
      "learning_rate": 9.814931166012694e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1423,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.25,
      "step": 10210
    },
    {
      "epoch": 0.7377463365336028,
      "grad_norm": 11.416603725375351,
      "learning_rate": 9.76493406222594e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -772.0,
      "logps/rejected": -1200.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.4375,
      "step": 10220
    },
    {
      "epoch": 0.7384682018335379,
      "grad_norm": 9.385956313764396,
      "learning_rate": 9.715033701524756e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.53125,
      "logps/chosen": -756.0,
      "logps/rejected": -1168.0,
      "loss": 0.1609,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.0,
      "step": 10230
    },
    {
      "epoch": 0.7391900671334729,
      "grad_norm": 12.293936918560657,
      "learning_rate": 9.665230400777793e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1662,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.25,
      "step": 10240
    },
    {
      "epoch": 0.7399119324334079,
      "grad_norm": 9.17603345329017,
      "learning_rate": 9.615524476237358e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -756.0,
      "logps/rejected": -1144.0,
      "loss": 0.1684,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.0,
      "step": 10250
    },
    {
      "epoch": 0.7406337977333429,
      "grad_norm": 8.175509864643324,
      "learning_rate": 9.565916243537434e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.546875,
      "logps/chosen": -744.0,
      "logps/rejected": -1120.0,
      "loss": 0.159,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.625,
      "step": 10260
    },
    {
      "epoch": 0.741355663033278,
      "grad_norm": 9.728008239435237,
      "learning_rate": 9.51640601769168e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5,
      "logps/chosen": -712.0,
      "logps/rejected": -1128.0,
      "loss": 0.1583,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -9.875,
      "step": 10270
    },
    {
      "epoch": 0.742077528333213,
      "grad_norm": 9.336449460312915,
      "learning_rate": 9.466994113091359e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.65625,
      "logps/chosen": -732.0,
      "logps/rejected": -1168.0,
      "loss": 0.1763,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.3125,
      "step": 10280
    },
    {
      "epoch": 0.7427993936331481,
      "grad_norm": 6.970907874552232,
      "learning_rate": 9.417680843503426e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.609375,
      "logps/chosen": -748.0,
      "logps/rejected": -1144.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.9375,
      "step": 10290
    },
    {
      "epoch": 0.7435212589330831,
      "grad_norm": 8.517413689223407,
      "learning_rate": 9.368466522068494e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.6875,
      "logps/chosen": -716.0,
      "logps/rejected": -1168.0,
      "loss": 0.1418,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.0625,
      "step": 10300
    },
    {
      "epoch": 0.7442431242330181,
      "grad_norm": 8.13932194474774,
      "learning_rate": 9.319351461298847e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.578125,
      "logps/chosen": -712.0,
      "logps/rejected": -1144.0,
      "loss": 0.1489,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.875,
      "step": 10310
    },
    {
      "epoch": 0.7449649895329532,
      "grad_norm": 9.845130667757584,
      "learning_rate": 9.270335973076468e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.578125,
      "logps/chosen": -728.0,
      "logps/rejected": -1168.0,
      "loss": 0.1562,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.0625,
      "step": 10320
    },
    {
      "epoch": 0.7456868548328882,
      "grad_norm": 8.005007818856038,
      "learning_rate": 9.221420368651045e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.78125,
      "logps/chosen": -728.0,
      "logps/rejected": -1160.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.1875,
      "step": 10330
    },
    {
      "epoch": 0.7464087201328232,
      "grad_norm": 8.120452643885544,
      "learning_rate": 9.172604958638008e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.609375,
      "logps/chosen": -736.0,
      "logps/rejected": -1160.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.0,
      "step": 10340
    },
    {
      "epoch": 0.7471305854327582,
      "grad_norm": 10.68134693575453,
      "learning_rate": 9.123890053016545e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -724.0,
      "logps/rejected": -1144.0,
      "loss": 0.143,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.9375,
      "step": 10350
    },
    {
      "epoch": 0.7478524507326932,
      "grad_norm": 11.395758028840984,
      "learning_rate": 9.07527596112764e-08,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.78125,
      "logps/chosen": -736.0,
      "logps/rejected": -1152.0,
      "loss": 0.1511,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.25,
      "rewards/rejected": -9.9375,
      "step": 10360
    },
    {
      "epoch": 0.7485743160326284,
      "grad_norm": 8.515447114819407,
      "learning_rate": 9.026762991672105e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.6875,
      "logps/chosen": -752.0,
      "logps/rejected": -1200.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 10370
    },
    {
      "epoch": 0.7492961813325634,
      "grad_norm": 11.086441240036681,
      "learning_rate": 8.978351452708626e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.59375,
      "logps/chosen": -748.0,
      "logps/rejected": -1160.0,
      "loss": 0.1514,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.0625,
      "step": 10380
    },
    {
      "epoch": 0.7500180466324984,
      "grad_norm": 7.288262141466547,
      "learning_rate": 8.930041651651795e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -776.0,
      "logps/rejected": -1232.0,
      "loss": 0.1421,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.625,
      "step": 10390
    },
    {
      "epoch": 0.7507399119324334,
      "grad_norm": 11.30869446863447,
      "learning_rate": 8.881833895270186e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.71875,
      "logps/chosen": -764.0,
      "logps/rejected": -1152.0,
      "loss": 0.1281,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 3.953125,
      "rewards/rejected": -10.0,
      "step": 10400
    },
    {
      "epoch": 0.7514617772323684,
      "grad_norm": 10.142597794950875,
      "learning_rate": 8.833728489684341e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.625,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.138,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.4375,
      "step": 10410
    },
    {
      "epoch": 0.7521836425323035,
      "grad_norm": 8.825883542812928,
      "learning_rate": 8.78572574036493e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.671875,
      "logps/chosen": -760.0,
      "logps/rejected": -1152.0,
      "loss": 0.1413,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.1875,
      "step": 10420
    },
    {
      "epoch": 0.7529055078322385,
      "grad_norm": 9.685940029071732,
      "learning_rate": 8.73782595213072e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.8125,
      "logps/chosen": -744.0,
      "logps/rejected": -1216.0,
      "loss": 0.1498,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.625,
      "step": 10430
    },
    {
      "epoch": 0.7536273731321735,
      "grad_norm": 8.534843301812261,
      "learning_rate": 8.690029429146656e-08,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.828125,
      "logps/chosen": -736.0,
      "logps/rejected": -1152.0,
      "loss": 0.1608,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.125,
      "step": 10440
    },
    {
      "epoch": 0.7543492384321085,
      "grad_norm": 8.26450848095713,
      "learning_rate": 8.642336474922e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.625,
      "logps/chosen": -704.0,
      "logps/rejected": -1192.0,
      "loss": 0.1501,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 5.0625,
      "rewards/rejected": -10.5625,
      "step": 10450
    },
    {
      "epoch": 0.7550711037320436,
      "grad_norm": 7.979518457621037,
      "learning_rate": 8.594747392308288e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.625,
      "logps/chosen": -752.0,
      "logps/rejected": -1176.0,
      "loss": 0.1438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.0625,
      "step": 10460
    },
    {
      "epoch": 0.7557929690319787,
      "grad_norm": 6.740894768789697,
      "learning_rate": 8.5472624834975e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.59375,
      "logps/chosen": -724.0,
      "logps/rejected": -1168.0,
      "loss": 0.1188,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.5,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.1875,
      "step": 10470
    },
    {
      "epoch": 0.7565148343319137,
      "grad_norm": 13.525079704406558,
      "learning_rate": 8.499882050020129e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.640625,
      "logps/chosen": -756.0,
      "logps/rejected": -1232.0,
      "loss": 0.1591,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.625,
      "step": 10480
    },
    {
      "epoch": 0.7572366996318487,
      "grad_norm": 8.368669950131547,
      "learning_rate": 8.4526063927432e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -780.0,
      "logps/rejected": -1136.0,
      "loss": 0.1726,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.21875,
      "rewards/margins": 3.75,
      "rewards/rejected": -10.0,
      "step": 10490
    },
    {
      "epoch": 0.7579585649317837,
      "grad_norm": 6.8452078042054,
      "learning_rate": 8.40543581186843e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.671875,
      "logps/chosen": -768.0,
      "logps/rejected": -1160.0,
      "loss": 0.1398,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -9.9375,
      "step": 10500
    },
    {
      "epoch": 0.7586804302317187,
      "grad_norm": 9.842039787756281,
      "learning_rate": 8.358370606930324e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.71875,
      "logps/chosen": -740.0,
      "logps/rejected": -1152.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.125,
      "step": 10510
    },
    {
      "epoch": 0.7594022955316538,
      "grad_norm": 9.548360738442916,
      "learning_rate": 8.311411076794195e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.6875,
      "logps/chosen": -740.0,
      "logps/rejected": -1144.0,
      "loss": 0.1493,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -9.9375,
      "step": 10520
    },
    {
      "epoch": 0.7601241608315888,
      "grad_norm": 10.51031541236341,
      "learning_rate": 8.264557519654353e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.671875,
      "logps/chosen": -744.0,
      "logps/rejected": -1168.0,
      "loss": 0.1554,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.1875,
      "step": 10530
    },
    {
      "epoch": 0.7608460261315239,
      "grad_norm": 8.514510120912588,
      "learning_rate": 8.217810233032168e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -744.0,
      "logps/rejected": -1176.0,
      "loss": 0.1358,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.1875,
      "step": 10540
    },
    {
      "epoch": 0.7615678914314589,
      "grad_norm": 11.439380123835884,
      "learning_rate": 8.171169513774189e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -760.0,
      "logps/rejected": -1224.0,
      "loss": 0.1591,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 10550
    },
    {
      "epoch": 0.7622897567313939,
      "grad_norm": 10.853410558035229,
      "learning_rate": 8.12463565805026e-08,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.609375,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.375,
      "step": 10560
    },
    {
      "epoch": 0.763011622031329,
      "grad_norm": 10.758545302896312,
      "learning_rate": 8.078208961351637e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -796.0,
      "logps/rejected": -1224.0,
      "loss": 0.1629,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.625,
      "step": 10570
    },
    {
      "epoch": 0.763733487331264,
      "grad_norm": 9.106309959022951,
      "learning_rate": 8.031889718489124e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -764.0,
      "logps/rejected": -1232.0,
      "loss": 0.1498,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.75,
      "step": 10580
    },
    {
      "epoch": 0.764455352631199,
      "grad_norm": 9.985703943059816,
      "learning_rate": 7.985678223591155e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -752.0,
      "logps/rejected": -1216.0,
      "loss": 0.1599,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.375,
      "step": 10590
    },
    {
      "epoch": 0.765177217931134,
      "grad_norm": 10.95962922741963,
      "learning_rate": 7.939574770102011e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -748.0,
      "logps/rejected": -1200.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.5625,
      "step": 10600
    },
    {
      "epoch": 0.765899083231069,
      "grad_norm": 5.8529547106778015,
      "learning_rate": 7.893579650779883e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -740.0,
      "logps/rejected": -1168.0,
      "loss": 0.1209,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.1875,
      "step": 10610
    },
    {
      "epoch": 0.7666209485310042,
      "grad_norm": 12.078898463041797,
      "learning_rate": 7.847693157695015e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.3125,
      "step": 10620
    },
    {
      "epoch": 0.7673428138309392,
      "grad_norm": 17.665428832633168,
      "learning_rate": 7.801915582227916e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.71875,
      "logps/chosen": -756.0,
      "logps/rejected": -1160.0,
      "loss": 0.1655,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.0625,
      "step": 10630
    },
    {
      "epoch": 0.7680646791308742,
      "grad_norm": 10.799427239930818,
      "learning_rate": 7.756247215067444e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -740.0,
      "logps/rejected": -1176.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.125,
      "step": 10640
    },
    {
      "epoch": 0.7687865444308092,
      "grad_norm": 7.898970963786076,
      "learning_rate": 7.710688346208952e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5,
      "logps/chosen": -720.0,
      "logps/rejected": -1136.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.53125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -9.75,
      "step": 10650
    },
    {
      "epoch": 0.7695084097307442,
      "grad_norm": 12.877547551422065,
      "learning_rate": 7.665239264952541e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -692.0,
      "logps/rejected": -1136.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.34375,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.0,
      "step": 10660
    },
    {
      "epoch": 0.7702302750306793,
      "grad_norm": 9.13069580760062,
      "learning_rate": 7.619900259901097e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1383,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.25,
      "step": 10670
    },
    {
      "epoch": 0.7709521403306143,
      "grad_norm": 7.025486658921553,
      "learning_rate": 7.574671618958544e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1311,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.5625,
      "step": 10680
    },
    {
      "epoch": 0.7716740056305493,
      "grad_norm": 8.042577057148172,
      "learning_rate": 7.529553629327994e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.71875,
      "logps/chosen": -792.0,
      "logps/rejected": -1256.0,
      "loss": 0.1337,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.34375,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.875,
      "step": 10690
    },
    {
      "epoch": 0.7723958709304843,
      "grad_norm": 9.406345265786733,
      "learning_rate": 7.484546577509918e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -788.0,
      "logps/rejected": -1216.0,
      "loss": 0.1457,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.625,
      "step": 10700
    },
    {
      "epoch": 0.7731177362304194,
      "grad_norm": 11.471896873198808,
      "learning_rate": 7.439650749300322e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.546875,
      "logps/chosen": -800.0,
      "logps/rejected": -1216.0,
      "loss": 0.1757,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 10710
    },
    {
      "epoch": 0.7738396015303545,
      "grad_norm": 8.152571305874515,
      "learning_rate": 7.394866429788945e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -728.0,
      "logps/rejected": -1208.0,
      "loss": 0.139,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.5625,
      "step": 10720
    },
    {
      "epoch": 0.7745614668302895,
      "grad_norm": 8.210195423512335,
      "learning_rate": 7.350193903357444e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.65625,
      "logps/chosen": -728.0,
      "logps/rejected": -1152.0,
      "loss": 0.163,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.125,
      "step": 10730
    },
    {
      "epoch": 0.7752833321302245,
      "grad_norm": 6.389795872006602,
      "learning_rate": 7.305633453677579e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1232.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.875,
      "step": 10740
    },
    {
      "epoch": 0.7760051974301595,
      "grad_norm": 9.502303401754403,
      "learning_rate": 7.26118536370943e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.609375,
      "logps/chosen": -808.0,
      "logps/rejected": -1248.0,
      "loss": 0.1385,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -11.0,
      "step": 10750
    },
    {
      "epoch": 0.7767270627300946,
      "grad_norm": 7.527767780807479,
      "learning_rate": 7.21684991569958e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.4375,
      "logps/chosen": -780.0,
      "logps/rejected": -1192.0,
      "loss": 0.1411,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.4375,
      "step": 10760
    },
    {
      "epoch": 0.7774489280300296,
      "grad_norm": 13.895751954232171,
      "learning_rate": 7.17262739117934e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1184.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 10770
    },
    {
      "epoch": 0.7781707933299646,
      "grad_norm": 5.314719639389733,
      "learning_rate": 7.128518070962947e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.4375,
      "logps/chosen": -776.0,
      "logps/rejected": -1200.0,
      "loss": 0.1362,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 10780
    },
    {
      "epoch": 0.7788926586298996,
      "grad_norm": 7.382521391430954,
      "learning_rate": 7.084522235145796e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -800.0,
      "logps/rejected": -1168.0,
      "loss": 0.1633,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.3125,
      "step": 10790
    },
    {
      "epoch": 0.7796145239298347,
      "grad_norm": 8.89077829343609,
      "learning_rate": 7.040640163102646e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -760.0,
      "logps/rejected": -1240.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.875,
      "rewards/rejected": -10.9375,
      "step": 10800
    },
    {
      "epoch": 0.7803363892297698,
      "grad_norm": 9.921923991247489,
      "learning_rate": 6.996872133485854e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.421875,
      "logps/chosen": -780.0,
      "logps/rejected": -1176.0,
      "loss": 0.161,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 10810
    },
    {
      "epoch": 0.7810582545297048,
      "grad_norm": 10.225109832732958,
      "learning_rate": 6.953218424223617e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.515625,
      "logps/chosen": -768.0,
      "logps/rejected": -1168.0,
      "loss": 0.1486,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.3125,
      "step": 10820
    },
    {
      "epoch": 0.7817801198296398,
      "grad_norm": 8.45743164183002,
      "learning_rate": 6.909679312518163e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1216.0,
      "loss": 0.1594,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5,
      "step": 10830
    },
    {
      "epoch": 0.7825019851295748,
      "grad_norm": 8.068263202065939,
      "learning_rate": 6.866255074844046e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.484375,
      "logps/chosen": -752.0,
      "logps/rejected": -1176.0,
      "loss": 0.1357,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.25,
      "step": 10840
    },
    {
      "epoch": 0.7832238504295098,
      "grad_norm": 7.516647068103794,
      "learning_rate": 6.822945986946385e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.578125,
      "logps/chosen": -784.0,
      "logps/rejected": -1192.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.5625,
      "step": 10850
    },
    {
      "epoch": 0.7839457157294449,
      "grad_norm": 6.275783146543779,
      "learning_rate": 6.77975232383905e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.515625,
      "logps/chosen": -760.0,
      "logps/rejected": -1288.0,
      "loss": 0.147,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 5.1875,
      "rewards/rejected": -11.1875,
      "step": 10860
    },
    {
      "epoch": 0.78466758102938,
      "grad_norm": 6.33801021737826,
      "learning_rate": 6.736674359802986e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.609375,
      "logps/chosen": -776.0,
      "logps/rejected": -1208.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 10870
    },
    {
      "epoch": 0.785389446329315,
      "grad_norm": 5.535919923586679,
      "learning_rate": 6.693712368384463e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.703125,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.375,
      "step": 10880
    },
    {
      "epoch": 0.78611131162925,
      "grad_norm": 8.838224158602769,
      "learning_rate": 6.650866622393281e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -772.0,
      "logps/rejected": -1208.0,
      "loss": 0.1724,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.28125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.5,
      "step": 10890
    },
    {
      "epoch": 0.786833176929185,
      "grad_norm": 11.077227000211735,
      "learning_rate": 6.608137393901106e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.59375,
      "logps/chosen": -796.0,
      "logps/rejected": -1216.0,
      "loss": 0.1658,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.5625,
      "step": 10900
    },
    {
      "epoch": 0.7875550422291201,
      "grad_norm": 8.408163966199039,
      "learning_rate": 6.565524954239709e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -796.0,
      "logps/rejected": -1200.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.34375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.5,
      "step": 10910
    },
    {
      "epoch": 0.7882769075290551,
      "grad_norm": 6.975124530660155,
      "learning_rate": 6.523029573999247e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.421875,
      "logps/chosen": -764.0,
      "logps/rejected": -1224.0,
      "loss": 0.1341,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.625,
      "step": 10920
    },
    {
      "epoch": 0.7889987728289901,
      "grad_norm": 9.65061642352629,
      "learning_rate": 6.480651523026548e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1184.0,
      "loss": 0.1458,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 10930
    },
    {
      "epoch": 0.7897206381289251,
      "grad_norm": 7.84205002288234,
      "learning_rate": 6.438391070423396e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -784.0,
      "logps/rejected": -1216.0,
      "loss": 0.1422,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.5625,
      "step": 10940
    },
    {
      "epoch": 0.7904425034288601,
      "grad_norm": 9.668441648479394,
      "learning_rate": 6.396248484544817e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1176.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.375,
      "step": 10950
    },
    {
      "epoch": 0.7911643687287953,
      "grad_norm": 14.668012539452555,
      "learning_rate": 6.354224032997391e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -824.0,
      "logps/rejected": -1192.0,
      "loss": 0.1396,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.5,
      "rewards/margins": 3.890625,
      "rewards/rejected": -10.375,
      "step": 10960
    },
    {
      "epoch": 0.7918862340287303,
      "grad_norm": 12.790645204648316,
      "learning_rate": 6.31231798263753e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.703125,
      "logps/chosen": -764.0,
      "logps/rejected": -1216.0,
      "loss": 0.1606,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.625,
      "step": 10970
    },
    {
      "epoch": 0.7926080993286653,
      "grad_norm": 8.256443608309835,
      "learning_rate": 6.27053059956981e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.671875,
      "logps/chosen": -772.0,
      "logps/rejected": -1192.0,
      "loss": 0.1357,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.375,
      "step": 10980
    },
    {
      "epoch": 0.7933299646286003,
      "grad_norm": 7.4278417405838635,
      "learning_rate": 6.228862149145234e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -784.0,
      "logps/rejected": -1176.0,
      "loss": 0.137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.25,
      "step": 10990
    },
    {
      "epoch": 0.7940518299285353,
      "grad_norm": 11.30809033177476,
      "learning_rate": 6.187312895959623e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -764.0,
      "logps/rejected": -1232.0,
      "loss": 0.1531,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.75,
      "rewards/rejected": -10.875,
      "step": 11000
    },
    {
      "epoch": 0.7947736952284704,
      "grad_norm": 8.324969415531836,
      "learning_rate": 6.145883103851876e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.578125,
      "logps/chosen": -792.0,
      "logps/rejected": -1200.0,
      "loss": 0.1482,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.34375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.5625,
      "step": 11010
    },
    {
      "epoch": 0.7954955605284054,
      "grad_norm": 7.879621497542356,
      "learning_rate": 6.10457303590228e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.59375,
      "logps/chosen": -776.0,
      "logps/rejected": -1224.0,
      "loss": 0.1544,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.625,
      "step": 11020
    },
    {
      "epoch": 0.7962174258283404,
      "grad_norm": 13.824352620024788,
      "learning_rate": 6.063382954430921e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.53125,
      "logps/chosen": -776.0,
      "logps/rejected": -1208.0,
      "loss": 0.159,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.6875,
      "step": 11030
    },
    {
      "epoch": 0.7969392911282754,
      "grad_norm": 9.7321719620388,
      "learning_rate": 6.022313120995942e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.53125,
      "logps/chosen": -768.0,
      "logps/rejected": -1208.0,
      "loss": 0.1591,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.625,
      "step": 11040
    },
    {
      "epoch": 0.7976611564282104,
      "grad_norm": 7.721151928722837,
      "learning_rate": 5.981363796391889e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -788.0,
      "logps/rejected": -1192.0,
      "loss": 0.1234,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.28125,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.375,
      "step": 11050
    },
    {
      "epoch": 0.7983830217281456,
      "grad_norm": 9.328263463248103,
      "learning_rate": 5.9405352406481226e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.59375,
      "logps/chosen": -776.0,
      "logps/rejected": -1200.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.5,
      "step": 11060
    },
    {
      "epoch": 0.7991048870280806,
      "grad_norm": 8.988279736482538,
      "learning_rate": 5.899827713027064e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -784.0,
      "logps/rejected": -1160.0,
      "loss": 0.1617,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.0,
      "rewards/rejected": -10.0625,
      "step": 11070
    },
    {
      "epoch": 0.7998267523280156,
      "grad_norm": 11.860283883793564,
      "learning_rate": 5.859241472022633e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -776.0,
      "logps/rejected": -1184.0,
      "loss": 0.1316,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.25,
      "step": 11080
    },
    {
      "epoch": 0.8005486176279506,
      "grad_norm": 8.339756060333015,
      "learning_rate": 5.818776775358586e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.734375,
      "logps/chosen": -796.0,
      "logps/rejected": -1176.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.28125,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.375,
      "step": 11090
    },
    {
      "epoch": 0.8012704829278856,
      "grad_norm": 11.113384556129915,
      "learning_rate": 5.778433879986835e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1176.0,
      "loss": 0.1515,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 11100
    },
    {
      "epoch": 0.8019923482278207,
      "grad_norm": 10.599463504430258,
      "learning_rate": 5.7382130420858794e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -780.0,
      "logps/rejected": -1192.0,
      "loss": 0.1681,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.5,
      "step": 11110
    },
    {
      "epoch": 0.8027142135277557,
      "grad_norm": 10.696808930709432,
      "learning_rate": 5.698114517059135e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -768.0,
      "logps/rejected": -1160.0,
      "loss": 0.1372,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.0625,
      "step": 11120
    },
    {
      "epoch": 0.8034360788276907,
      "grad_norm": 8.944453033412174,
      "learning_rate": 5.658138559533338e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.578125,
      "logps/chosen": -756.0,
      "logps/rejected": -1208.0,
      "loss": 0.1787,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.625,
      "step": 11130
    },
    {
      "epoch": 0.8041579441276258,
      "grad_norm": 8.977764582899535,
      "learning_rate": 5.6182854233569084e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.703125,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.1529,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.3125,
      "step": 11140
    },
    {
      "epoch": 0.8048798094275608,
      "grad_norm": 10.126224395800762,
      "learning_rate": 5.578555361598353e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.453125,
      "logps/chosen": -724.0,
      "logps/rejected": -1176.0,
      "loss": 0.1324,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.25,
      "step": 11150
    },
    {
      "epoch": 0.8056016747274959,
      "grad_norm": 7.398653029233116,
      "learning_rate": 5.538948626544651e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -764.0,
      "logps/rejected": -1168.0,
      "loss": 0.1445,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.25,
      "step": 11160
    },
    {
      "epoch": 0.8063235400274309,
      "grad_norm": 8.672148610321091,
      "learning_rate": 5.4994654696996545e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -752.0,
      "logps/rejected": -1200.0,
      "loss": 0.1573,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 11170
    },
    {
      "epoch": 0.8070454053273659,
      "grad_norm": 7.7083477926519,
      "learning_rate": 5.460106141782492e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -752.0,
      "logps/rejected": -1176.0,
      "loss": 0.1685,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.125,
      "step": 11180
    },
    {
      "epoch": 0.8077672706273009,
      "grad_norm": 11.545406044029406,
      "learning_rate": 5.420870892725965e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -748.0,
      "logps/rejected": -1160.0,
      "loss": 0.158,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.125,
      "step": 11190
    },
    {
      "epoch": 0.8084891359272359,
      "grad_norm": 10.670951866955587,
      "learning_rate": 5.381759971674987e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.46875,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 11200
    },
    {
      "epoch": 0.809211001227171,
      "grad_norm": 11.225540948573572,
      "learning_rate": 5.3427736269849724e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -752.0,
      "logps/rejected": -1144.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.125,
      "step": 11210
    },
    {
      "epoch": 0.809932866527106,
      "grad_norm": 7.266681042559748,
      "learning_rate": 5.303912106220285e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1184.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 11220
    },
    {
      "epoch": 0.8106547318270411,
      "grad_norm": 9.542579667649324,
      "learning_rate": 5.265175656152621e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.703125,
      "logps/chosen": -796.0,
      "logps/rejected": -1168.0,
      "loss": 0.1671,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.4375,
      "rewards/margins": 3.9375,
      "rewards/rejected": -10.375,
      "step": 11230
    },
    {
      "epoch": 0.8113765971269761,
      "grad_norm": 9.545895584965017,
      "learning_rate": 5.226564522759525e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.484375,
      "logps/chosen": -800.0,
      "logps/rejected": -1200.0,
      "loss": 0.1574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.4375,
      "step": 11240
    },
    {
      "epoch": 0.8120984624269111,
      "grad_norm": 8.761578973065786,
      "learning_rate": 5.1880789512227434e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1577,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 3.921875,
      "rewards/rejected": -10.0625,
      "step": 11250
    },
    {
      "epoch": 0.8128203277268462,
      "grad_norm": 10.935949386772302,
      "learning_rate": 5.1497191859267014e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.546875,
      "logps/chosen": -744.0,
      "logps/rejected": -1176.0,
      "loss": 0.1503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.375,
      "step": 11260
    },
    {
      "epoch": 0.8135421930267812,
      "grad_norm": 11.542728432624855,
      "learning_rate": 5.111485470456953e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.609375,
      "logps/chosen": -748.0,
      "logps/rejected": -1200.0,
      "loss": 0.158,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.5,
      "step": 11270
    },
    {
      "epoch": 0.8142640583267162,
      "grad_norm": 9.765483774608478,
      "learning_rate": 5.0733780475986617e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.375,
      "step": 11280
    },
    {
      "epoch": 0.8149859236266512,
      "grad_norm": 9.322831257835603,
      "learning_rate": 5.035397159334984e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -800.0,
      "logps/rejected": -1168.0,
      "loss": 0.1422,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.21875,
      "rewards/margins": 3.96875,
      "rewards/rejected": -10.1875,
      "step": 11290
    },
    {
      "epoch": 0.8157077889265864,
      "grad_norm": 8.623282731830352,
      "learning_rate": 4.997543046845604e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5,
      "logps/chosen": -740.0,
      "logps/rejected": -1224.0,
      "loss": 0.1369,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.5625,
      "step": 11300
    },
    {
      "epoch": 0.8164296542265214,
      "grad_norm": 6.745112334659091,
      "learning_rate": 4.959815950505175e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -756.0,
      "logps/rejected": -1200.0,
      "loss": 0.1443,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.625,
      "step": 11310
    },
    {
      "epoch": 0.8171515195264564,
      "grad_norm": 10.676187874451763,
      "learning_rate": 4.922216109881791e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -764.0,
      "logps/rejected": -1176.0,
      "loss": 0.1458,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.3125,
      "step": 11320
    },
    {
      "epoch": 0.8178733848263914,
      "grad_norm": 7.184364110256313,
      "learning_rate": 4.8847437637354664e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.4375,
      "logps/chosen": -740.0,
      "logps/rejected": -1208.0,
      "loss": 0.1548,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.6875,
      "step": 11330
    },
    {
      "epoch": 0.8185952501263264,
      "grad_norm": 10.120238987000853,
      "learning_rate": 4.8473991500166234e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.375,
      "step": 11340
    },
    {
      "epoch": 0.8193171154262615,
      "grad_norm": 11.946412125314207,
      "learning_rate": 4.810182505864585e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.578125,
      "logps/chosen": -772.0,
      "logps/rejected": -1208.0,
      "loss": 0.1469,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.5,
      "step": 11350
    },
    {
      "epoch": 0.8200389807261965,
      "grad_norm": 12.472625629846199,
      "learning_rate": 4.7730940676060564e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1200.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.5,
      "step": 11360
    },
    {
      "epoch": 0.8207608460261315,
      "grad_norm": 6.685150027611601,
      "learning_rate": 4.736134070753639e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.40625,
      "logps/chosen": -744.0,
      "logps/rejected": -1168.0,
      "loss": 0.1477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.1875,
      "step": 11370
    },
    {
      "epoch": 0.8214827113260665,
      "grad_norm": 10.618261196059562,
      "learning_rate": 4.6993027500043275e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -784.0,
      "logps/rejected": -1256.0,
      "loss": 0.1579,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.9375,
      "step": 11380
    },
    {
      "epoch": 0.8222045766260015,
      "grad_norm": 9.968980323412524,
      "learning_rate": 4.662600339237999e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.46875,
      "logps/chosen": -812.0,
      "logps/rejected": -1168.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.25,
      "rewards/margins": 3.875,
      "rewards/rejected": -10.125,
      "step": 11390
    },
    {
      "epoch": 0.8229264419259367,
      "grad_norm": 9.132558632733545,
      "learning_rate": 4.6260270715159836e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -748.0,
      "logps/rejected": -1192.0,
      "loss": 0.161,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.375,
      "step": 11400
    },
    {
      "epoch": 0.8236483072258717,
      "grad_norm": 9.046561088489032,
      "learning_rate": 4.589583179079531e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.4375,
      "logps/chosen": -740.0,
      "logps/rejected": -1200.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.625,
      "step": 11410
    },
    {
      "epoch": 0.8243701725258067,
      "grad_norm": 9.600138399413229,
      "learning_rate": 4.553268893348339e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.515625,
      "logps/chosen": -764.0,
      "logps/rejected": -1280.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 5.21875,
      "rewards/rejected": -11.0625,
      "step": 11420
    },
    {
      "epoch": 0.8250920378257417,
      "grad_norm": 11.534256892624052,
      "learning_rate": 4.5170844449191364e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -760.0,
      "logps/rejected": -1192.0,
      "loss": 0.1586,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 11430
    },
    {
      "epoch": 0.8258139031256767,
      "grad_norm": 8.265111331997474,
      "learning_rate": 4.481030063564156e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -784.0,
      "logps/rejected": -1184.0,
      "loss": 0.1372,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.4375,
      "step": 11440
    },
    {
      "epoch": 0.8265357684256118,
      "grad_norm": 9.696675663745067,
      "learning_rate": 4.445105978229688e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.515625,
      "logps/chosen": -760.0,
      "logps/rejected": -1224.0,
      "loss": 0.1529,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.90625,
      "rewards/rejected": -10.75,
      "step": 11450
    },
    {
      "epoch": 0.8272576337255468,
      "grad_norm": 11.965530383018477,
      "learning_rate": 4.409312417034683e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.578125,
      "logps/chosen": -728.0,
      "logps/rejected": -1200.0,
      "loss": 0.1372,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.5,
      "step": 11460
    },
    {
      "epoch": 0.8279794990254818,
      "grad_norm": 10.288831340165839,
      "learning_rate": 4.373649607269217e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -792.0,
      "logps/rejected": -1184.0,
      "loss": 0.1517,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.3125,
      "step": 11470
    },
    {
      "epoch": 0.8287013643254169,
      "grad_norm": 10.58525488188931,
      "learning_rate": 4.338117775393107e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.703125,
      "logps/chosen": -792.0,
      "logps/rejected": -1176.0,
      "loss": 0.1647,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.375,
      "step": 11480
    },
    {
      "epoch": 0.8294232296253519,
      "grad_norm": 11.407495223286647,
      "learning_rate": 4.3027171470344767e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -772.0,
      "logps/rejected": -1208.0,
      "loss": 0.1333,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5,
      "step": 11490
    },
    {
      "epoch": 0.830145094925287,
      "grad_norm": 7.549142594426182,
      "learning_rate": 4.267447946988262e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.609375,
      "logps/chosen": -784.0,
      "logps/rejected": -1176.0,
      "loss": 0.1621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.25,
      "step": 11500
    },
    {
      "epoch": 0.830866960225222,
      "grad_norm": 10.39718086616228,
      "learning_rate": 4.232310399214853e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.515625,
      "logps/chosen": -716.0,
      "logps/rejected": -1216.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6875,
      "rewards/margins": 4.875,
      "rewards/rejected": -10.5625,
      "step": 11510
    },
    {
      "epoch": 0.831588825525157,
      "grad_norm": 7.71946842083615,
      "learning_rate": 4.1973047268386545e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -748.0,
      "logps/rejected": -1192.0,
      "loss": 0.1261,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.375,
      "step": 11520
    },
    {
      "epoch": 0.832310690825092,
      "grad_norm": 11.394005391721311,
      "learning_rate": 4.162431152146631e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -788.0,
      "logps/rejected": -1184.0,
      "loss": 0.1835,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -10.125,
      "step": 11530
    },
    {
      "epoch": 0.833032556125027,
      "grad_norm": 9.621474950872127,
      "learning_rate": 4.127689896586936e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.640625,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.141,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.5,
      "step": 11540
    },
    {
      "epoch": 0.8337544214249621,
      "grad_norm": 7.232695813686058,
      "learning_rate": 4.093081180767494e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -736.0,
      "logps/rejected": -1200.0,
      "loss": 0.1313,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.5,
      "step": 11550
    },
    {
      "epoch": 0.8344762867248972,
      "grad_norm": 7.421382018126838,
      "learning_rate": 4.0586052244546e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.734375,
      "logps/chosen": -780.0,
      "logps/rejected": -1144.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 3.984375,
      "rewards/rejected": -10.0625,
      "step": 11560
    },
    {
      "epoch": 0.8351981520248322,
      "grad_norm": 9.602867969798325,
      "learning_rate": 4.0242622465715196e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.59375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1465,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.5,
      "step": 11570
    },
    {
      "epoch": 0.8359200173247672,
      "grad_norm": 9.389336268350176,
      "learning_rate": 3.9900524651970995e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -764.0,
      "logps/rejected": -1200.0,
      "loss": 0.1488,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.5625,
      "step": 11580
    },
    {
      "epoch": 0.8366418826247022,
      "grad_norm": 9.480201806828463,
      "learning_rate": 3.9559760975643897e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.65625,
      "logps/chosen": -748.0,
      "logps/rejected": -1200.0,
      "loss": 0.1461,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 11590
    },
    {
      "epoch": 0.8373637479246373,
      "grad_norm": 7.3419299237458695,
      "learning_rate": 3.922033360059254e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -788.0,
      "logps/rejected": -1192.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.28125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.5,
      "step": 11600
    },
    {
      "epoch": 0.8380856132245723,
      "grad_norm": 9.887973431940114,
      "learning_rate": 3.8882244682190094e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.515625,
      "logps/chosen": -764.0,
      "logps/rejected": -1176.0,
      "loss": 0.146,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.5,
      "step": 11610
    },
    {
      "epoch": 0.8388074785245073,
      "grad_norm": 6.937760256383144,
      "learning_rate": 3.854549636731047e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.6875,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.1391,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.75,
      "step": 11620
    },
    {
      "epoch": 0.8395293438244423,
      "grad_norm": 8.161082030414025,
      "learning_rate": 3.82100907943145e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.65625,
      "logps/chosen": -780.0,
      "logps/rejected": -1208.0,
      "loss": 0.1746,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.6875,
      "step": 11630
    },
    {
      "epoch": 0.8402512091243773,
      "grad_norm": 8.796986939445052,
      "learning_rate": 3.7876030093036945e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -780.0,
      "logps/rejected": -1208.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.75,
      "step": 11640
    },
    {
      "epoch": 0.8409730744243125,
      "grad_norm": 9.161196064107799,
      "learning_rate": 3.7543316384772375e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.5625,
      "logps/chosen": -772.0,
      "logps/rejected": -1216.0,
      "loss": 0.1537,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 11650
    },
    {
      "epoch": 0.8416949397242475,
      "grad_norm": 8.326472553423818,
      "learning_rate": 3.721195178226178e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.609375,
      "logps/chosen": -748.0,
      "logps/rejected": -1200.0,
      "loss": 0.1482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 11660
    },
    {
      "epoch": 0.8424168050241825,
      "grad_norm": 8.11503963287216,
      "learning_rate": 3.68819383896796e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.546875,
      "logps/chosen": -780.0,
      "logps/rejected": -1216.0,
      "loss": 0.1451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.5625,
      "step": 11670
    },
    {
      "epoch": 0.8431386703241175,
      "grad_norm": 8.18334342762608,
      "learning_rate": 3.655327830261995e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -812.0,
      "logps/rejected": -1176.0,
      "loss": 0.1527,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -6.34375,
      "rewards/margins": 3.828125,
      "rewards/rejected": -10.1875,
      "step": 11680
    },
    {
      "epoch": 0.8438605356240525,
      "grad_norm": 9.646827096360845,
      "learning_rate": 3.622597360808324e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -792.0,
      "logps/rejected": -1208.0,
      "loss": 0.1656,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.375,
      "step": 11690
    },
    {
      "epoch": 0.8445824009239876,
      "grad_norm": 8.41592945035557,
      "learning_rate": 3.5900026384463324e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.546875,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.144,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.4375,
      "step": 11700
    },
    {
      "epoch": 0.8453042662239226,
      "grad_norm": 11.091054652826791,
      "learning_rate": 3.557543870153393e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -764.0,
      "logps/rejected": -1224.0,
      "loss": 0.1529,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.6875,
      "step": 11710
    },
    {
      "epoch": 0.8460261315238576,
      "grad_norm": 10.649551216885522,
      "learning_rate": 3.5252212620435764e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.1466,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.375,
      "step": 11720
    },
    {
      "epoch": 0.8467479968237926,
      "grad_norm": 10.591948146106109,
      "learning_rate": 3.493035019366328e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -732.0,
      "logps/rejected": -1200.0,
      "loss": 0.1522,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.4375,
      "step": 11730
    },
    {
      "epoch": 0.8474698621237277,
      "grad_norm": 10.912246863703778,
      "learning_rate": 3.460985346505169e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -788.0,
      "logps/rejected": -1192.0,
      "loss": 0.163,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.25,
      "step": 11740
    },
    {
      "epoch": 0.8481917274236628,
      "grad_norm": 6.539575387536508,
      "learning_rate": 3.4290724469764e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.625,
      "logps/chosen": -772.0,
      "logps/rejected": -1200.0,
      "loss": 0.1443,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.4375,
      "step": 11750
    },
    {
      "epoch": 0.8489135927235978,
      "grad_norm": 12.080456229188744,
      "learning_rate": 3.397296523427806e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -760.0,
      "logps/rejected": -1176.0,
      "loss": 0.1408,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.3125,
      "step": 11760
    },
    {
      "epoch": 0.8496354580235328,
      "grad_norm": 10.737805994163432,
      "learning_rate": 3.3656577776373706e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -772.0,
      "logps/rejected": -1208.0,
      "loss": 0.1402,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.1875,
      "step": 11770
    },
    {
      "epoch": 0.8503573233234678,
      "grad_norm": 8.746433818443641,
      "learning_rate": 3.334156410511993e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.71875,
      "logps/chosen": -752.0,
      "logps/rejected": -1224.0,
      "loss": 0.1619,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.5,
      "step": 11780
    },
    {
      "epoch": 0.8510791886234029,
      "grad_norm": 8.942255435734593,
      "learning_rate": 3.30279262208622e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -796.0,
      "logps/rejected": -1184.0,
      "loss": 0.1355,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.25,
      "step": 11790
    },
    {
      "epoch": 0.8518010539233379,
      "grad_norm": 13.240281042934914,
      "learning_rate": 3.271566611520965e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.78125,
      "logps/chosen": -764.0,
      "logps/rejected": -1144.0,
      "loss": 0.1533,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -10.0625,
      "step": 11800
    },
    {
      "epoch": 0.852522919223273,
      "grad_norm": 7.445169779102206,
      "learning_rate": 3.240478577102254e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.578125,
      "logps/chosen": -812.0,
      "logps/rejected": -1208.0,
      "loss": 0.1539,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.3125,
      "rewards/margins": 3.921875,
      "rewards/rejected": -10.25,
      "step": 11810
    },
    {
      "epoch": 0.853244784523208,
      "grad_norm": 10.391094523428233,
      "learning_rate": 3.2095287162399396e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.5625,
      "logps/chosen": -780.0,
      "logps/rejected": -1200.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.6875,
      "step": 11820
    },
    {
      "epoch": 0.853966649823143,
      "grad_norm": 10.97007525410186,
      "learning_rate": 3.178717225466504e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.515625,
      "logps/chosen": -784.0,
      "logps/rejected": -1224.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.75,
      "step": 11830
    },
    {
      "epoch": 0.8546885151230781,
      "grad_norm": 10.949419924686783,
      "learning_rate": 3.1480443004357535e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.46875,
      "logps/chosen": -748.0,
      "logps/rejected": -1216.0,
      "loss": 0.1419,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.6875,
      "step": 11840
    },
    {
      "epoch": 0.8554103804230131,
      "grad_norm": 7.949017711437389,
      "learning_rate": 3.117510135921589e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.578125,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.127,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5,
      "step": 11850
    },
    {
      "epoch": 0.8561322457229481,
      "grad_norm": 8.211089267034843,
      "learning_rate": 3.087114925816808e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.609375,
      "logps/chosen": -784.0,
      "logps/rejected": -1224.0,
      "loss": 0.1521,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.625,
      "step": 11860
    },
    {
      "epoch": 0.8568541110228831,
      "grad_norm": 7.931702604806061,
      "learning_rate": 3.0568588631318115e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.53125,
      "logps/chosen": -728.0,
      "logps/rejected": -1216.0,
      "loss": 0.1334,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.71875,
      "rewards/margins": 4.90625,
      "rewards/rejected": -10.625,
      "step": 11870
    },
    {
      "epoch": 0.8575759763228181,
      "grad_norm": 13.782675522555701,
      "learning_rate": 3.026742139993427e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -784.0,
      "logps/rejected": -1224.0,
      "loss": 0.1561,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.5625,
      "step": 11880
    },
    {
      "epoch": 0.8582978416227532,
      "grad_norm": 8.568533344202447,
      "learning_rate": 2.996764947643685e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -784.0,
      "logps/rejected": -1256.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.9375,
      "step": 11890
    },
    {
      "epoch": 0.8590197069226883,
      "grad_norm": 12.27253477325102,
      "learning_rate": 2.9669274764385604e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5625,
      "logps/chosen": -784.0,
      "logps/rejected": -1176.0,
      "loss": 0.1662,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.25,
      "step": 11900
    },
    {
      "epoch": 0.8597415722226233,
      "grad_norm": 8.289657053798905,
      "learning_rate": 2.9372299158468144e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.53125,
      "logps/chosen": -764.0,
      "logps/rejected": -1224.0,
      "loss": 0.1465,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.75,
      "rewards/rejected": -10.6875,
      "step": 11910
    },
    {
      "epoch": 0.8604634375225583,
      "grad_norm": 10.205485648725467,
      "learning_rate": 2.907672454448784e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -768.0,
      "logps/rejected": -1232.0,
      "loss": 0.1457,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.6875,
      "step": 11920
    },
    {
      "epoch": 0.8611853028224933,
      "grad_norm": 9.498983941338254,
      "learning_rate": 2.8782552799351404e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.515625,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1494,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.4375,
      "step": 11930
    },
    {
      "epoch": 0.8619071681224284,
      "grad_norm": 6.074944238826367,
      "learning_rate": 2.8489785791057543e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.65625,
      "logps/chosen": -760.0,
      "logps/rejected": -1208.0,
      "loss": 0.1479,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5,
      "step": 11940
    },
    {
      "epoch": 0.8626290334223634,
      "grad_norm": 8.302877488046027,
      "learning_rate": 2.819842537868475e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -796.0,
      "logps/rejected": -1216.0,
      "loss": 0.1538,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.5625,
      "step": 11950
    },
    {
      "epoch": 0.8633508987222984,
      "grad_norm": 11.61178207342639,
      "learning_rate": 2.7908473412379618e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -792.0,
      "logps/rejected": -1192.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.4375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.625,
      "step": 11960
    },
    {
      "epoch": 0.8640727640222334,
      "grad_norm": 9.45795876040694,
      "learning_rate": 2.761993173334509e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5625,
      "logps/chosen": -804.0,
      "logps/rejected": -1192.0,
      "loss": 0.1542,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.3125,
      "step": 11970
    },
    {
      "epoch": 0.8647946293221684,
      "grad_norm": 11.362932343916869,
      "learning_rate": 2.733280217382869e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -764.0,
      "logps/rejected": -1200.0,
      "loss": 0.1466,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.375,
      "step": 11980
    },
    {
      "epoch": 0.8655164946221036,
      "grad_norm": 10.568145168122776,
      "learning_rate": 2.704708655711102e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.609375,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1357,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.3125,
      "step": 11990
    },
    {
      "epoch": 0.8662383599220386,
      "grad_norm": 6.986633654793124,
      "learning_rate": 2.6762786697494016e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -780.0,
      "logps/rejected": -1224.0,
      "loss": 0.1625,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.875,
      "step": 12000
    },
    {
      "epoch": 0.8669602252219736,
      "grad_norm": 7.778882127033462,
      "learning_rate": 2.6479904400289578e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -772.0,
      "logps/rejected": -1200.0,
      "loss": 0.158,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.3125,
      "step": 12010
    },
    {
      "epoch": 0.8676820905219086,
      "grad_norm": 8.539754631291549,
      "learning_rate": 2.6198441461808106e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.421875,
      "logps/chosen": -780.0,
      "logps/rejected": -1264.0,
      "loss": 0.1475,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.84375,
      "rewards/rejected": -11.0,
      "step": 12020
    },
    {
      "epoch": 0.8684039558218436,
      "grad_norm": 7.963728288501986,
      "learning_rate": 2.5918399669346808e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -736.0,
      "logps/rejected": -1200.0,
      "loss": 0.1253,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.5,
      "step": 12030
    },
    {
      "epoch": 0.8691258211217787,
      "grad_norm": 10.257823465025604,
      "learning_rate": 2.5639780801178844e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.1705,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.625,
      "step": 12040
    },
    {
      "epoch": 0.8698476864217137,
      "grad_norm": 8.314704744595538,
      "learning_rate": 2.53625866265417e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -808.0,
      "logps/rejected": -1208.0,
      "loss": 0.1427,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.6875,
      "step": 12050
    },
    {
      "epoch": 0.8705695517216487,
      "grad_norm": 8.499655093782934,
      "learning_rate": 2.508681890562575e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.484375,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.1355,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.375,
      "step": 12060
    },
    {
      "epoch": 0.8712914170215837,
      "grad_norm": 7.835075004860599,
      "learning_rate": 2.4812479389563794e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.484375,
      "logps/chosen": -768.0,
      "logps/rejected": -1232.0,
      "loss": 0.1513,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 12070
    },
    {
      "epoch": 0.8720132823215188,
      "grad_norm": 9.592674313284755,
      "learning_rate": 2.4539569820419213e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.578125,
      "logps/chosen": -772.0,
      "logps/rejected": -1248.0,
      "loss": 0.1653,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.875,
      "step": 12080
    },
    {
      "epoch": 0.8727351476214539,
      "grad_norm": 7.77863378103726,
      "learning_rate": 2.4268091931175128e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.640625,
      "logps/chosen": -764.0,
      "logps/rejected": -1160.0,
      "loss": 0.1351,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.0625,
      "step": 12090
    },
    {
      "epoch": 0.8734570129213889,
      "grad_norm": 5.640158653910894,
      "learning_rate": 2.3998047445723728e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.65625,
      "logps/chosen": -732.0,
      "logps/rejected": -1184.0,
      "loss": 0.1468,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.4375,
      "step": 12100
    },
    {
      "epoch": 0.8741788782213239,
      "grad_norm": 9.254054430708486,
      "learning_rate": 2.3729438078854748e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1208.0,
      "loss": 0.1658,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.28125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.625,
      "step": 12110
    },
    {
      "epoch": 0.8749007435212589,
      "grad_norm": 12.122038328529602,
      "learning_rate": 2.3462265536245085e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -804.0,
      "logps/rejected": -1232.0,
      "loss": 0.1587,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.4375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.8125,
      "step": 12120
    },
    {
      "epoch": 0.8756226088211939,
      "grad_norm": 5.974867571191328,
      "learning_rate": 2.3196531514447643e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1184.0,
      "loss": 0.1273,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5625,
      "step": 12130
    },
    {
      "epoch": 0.876344474121129,
      "grad_norm": 9.921544747499848,
      "learning_rate": 2.293223770088079e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.53125,
      "logps/chosen": -792.0,
      "logps/rejected": -1208.0,
      "loss": 0.1421,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.5,
      "step": 12140
    },
    {
      "epoch": 0.877066339421064,
      "grad_norm": 12.38062445080125,
      "learning_rate": 2.2669385773817467e-08,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.5,
      "logps/chosen": -736.0,
      "logps/rejected": -1216.0,
      "loss": 0.1354,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.625,
      "step": 12150
    },
    {
      "epoch": 0.877788204720999,
      "grad_norm": 12.239735356816436,
      "learning_rate": 2.2407977402374545e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.515625,
      "logps/chosen": -788.0,
      "logps/rejected": -1232.0,
      "loss": 0.148,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.6875,
      "step": 12160
    },
    {
      "epoch": 0.8785100700209341,
      "grad_norm": 9.840653330548708,
      "learning_rate": 2.2148014246502395e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -764.0,
      "logps/rejected": -1168.0,
      "loss": 0.1715,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.125,
      "step": 12170
    },
    {
      "epoch": 0.8792319353208691,
      "grad_norm": 9.716661953638562,
      "learning_rate": 2.1889497956974146e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.1656,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 12180
    },
    {
      "epoch": 0.8799538006208042,
      "grad_norm": 8.417846830214762,
      "learning_rate": 2.1632430175375332e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.625,
      "step": 12190
    },
    {
      "epoch": 0.8806756659207392,
      "grad_norm": 9.941969312589332,
      "learning_rate": 2.1376812534093375e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1176.0,
      "loss": 0.1339,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.1875,
      "step": 12200
    },
    {
      "epoch": 0.8813975312206742,
      "grad_norm": 10.465626887089373,
      "learning_rate": 2.112264665630728e-08,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -744.0,
      "logps/rejected": -1152.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.0,
      "step": 12210
    },
    {
      "epoch": 0.8821193965206092,
      "grad_norm": 10.02996418403007,
      "learning_rate": 2.0869934155977348e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.453125,
      "logps/chosen": -752.0,
      "logps/rejected": -1240.0,
      "loss": 0.1435,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.8125,
      "step": 12220
    },
    {
      "epoch": 0.8828412618205442,
      "grad_norm": 10.447715723395184,
      "learning_rate": 2.0618676637834924e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.578125,
      "logps/chosen": -740.0,
      "logps/rejected": -1192.0,
      "loss": 0.1388,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.4375,
      "step": 12230
    },
    {
      "epoch": 0.8835631271204794,
      "grad_norm": 9.973159448416428,
      "learning_rate": 2.0368875697372028e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1588,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.25,
      "step": 12240
    },
    {
      "epoch": 0.8842849924204144,
      "grad_norm": 7.405712078749057,
      "learning_rate": 2.0120532920831436e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1232.0,
      "loss": 0.1497,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.625,
      "step": 12250
    },
    {
      "epoch": 0.8850068577203494,
      "grad_norm": 8.608940672524163,
      "learning_rate": 1.9873649885196742e-08,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.421875,
      "logps/chosen": -772.0,
      "logps/rejected": -1192.0,
      "loss": 0.149,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.375,
      "step": 12260
    },
    {
      "epoch": 0.8857287230202844,
      "grad_norm": 8.76111445121565,
      "learning_rate": 1.9628228158181853e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.421875,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.158,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.375,
      "step": 12270
    },
    {
      "epoch": 0.8864505883202195,
      "grad_norm": 9.726249616502288,
      "learning_rate": 1.93842692982214e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.46875,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.1367,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.25,
      "step": 12280
    },
    {
      "epoch": 0.8871724536201545,
      "grad_norm": 9.856991665137961,
      "learning_rate": 1.9141774854461e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.53125,
      "logps/chosen": -780.0,
      "logps/rejected": -1216.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.625,
      "step": 12290
    },
    {
      "epoch": 0.8878943189200895,
      "grad_norm": 8.90097517455146,
      "learning_rate": 1.890074636674685e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -784.0,
      "logps/rejected": -1152.0,
      "loss": 0.1622,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.125,
      "step": 12300
    },
    {
      "epoch": 0.8886161842200245,
      "grad_norm": 11.01818428730531,
      "learning_rate": 1.8661185365616478e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1413,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.3125,
      "step": 12310
    },
    {
      "epoch": 0.8893380495199595,
      "grad_norm": 11.263744199520602,
      "learning_rate": 1.842309337228884e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -800.0,
      "logps/rejected": -1192.0,
      "loss": 0.1501,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.4375,
      "step": 12320
    },
    {
      "epoch": 0.8900599148198947,
      "grad_norm": 7.792414941928654,
      "learning_rate": 1.818647189865455e-08,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.75,
      "logps/chosen": -748.0,
      "logps/rejected": -1200.0,
      "loss": 0.1541,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.5,
      "step": 12330
    },
    {
      "epoch": 0.8907817801198297,
      "grad_norm": 8.762492783110734,
      "learning_rate": 1.7951322447266493e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1224.0,
      "loss": 0.1599,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.5625,
      "step": 12340
    },
    {
      "epoch": 0.8915036454197647,
      "grad_norm": 12.278119822877175,
      "learning_rate": 1.7717646511330063e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.484375,
      "logps/chosen": -768.0,
      "logps/rejected": -1216.0,
      "loss": 0.1689,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.5625,
      "step": 12350
    },
    {
      "epoch": 0.8922255107196997,
      "grad_norm": 8.026983202327546,
      "learning_rate": 1.748544557469392e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.46875,
      "logps/chosen": -776.0,
      "logps/rejected": -1208.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 12360
    },
    {
      "epoch": 0.8929473760196347,
      "grad_norm": 8.425357221302765,
      "learning_rate": 1.7254721111840287e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -752.0,
      "logps/rejected": -1216.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.6875,
      "step": 12370
    },
    {
      "epoch": 0.8936692413195698,
      "grad_norm": 9.644004550576527,
      "learning_rate": 1.7025474587875872e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.75,
      "logps/chosen": -760.0,
      "logps/rejected": -1160.0,
      "loss": 0.136,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.125,
      "step": 12380
    },
    {
      "epoch": 0.8943911066195048,
      "grad_norm": 7.131646950070454,
      "learning_rate": 1.6797707458522382e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -776.0,
      "logps/rejected": -1208.0,
      "loss": 0.1411,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.625,
      "step": 12390
    },
    {
      "epoch": 0.8951129719194398,
      "grad_norm": 8.868708466246948,
      "learning_rate": 1.6571421170107236e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.546875,
      "logps/chosen": -768.0,
      "logps/rejected": -1224.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.6875,
      "step": 12400
    },
    {
      "epoch": 0.8958348372193748,
      "grad_norm": 7.487849791891332,
      "learning_rate": 1.6346617159554628e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -780.0,
      "logps/rejected": -1160.0,
      "loss": 0.1536,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.125,
      "step": 12410
    },
    {
      "epoch": 0.8965567025193099,
      "grad_norm": 8.438213927485114,
      "learning_rate": 1.6123296854376217e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.6875,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1648,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.375,
      "step": 12420
    },
    {
      "epoch": 0.897278567819245,
      "grad_norm": 10.754318102307618,
      "learning_rate": 1.5901461672661904e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.546875,
      "logps/chosen": -792.0,
      "logps/rejected": -1184.0,
      "loss": 0.1789,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.34375,
      "rewards/margins": 3.984375,
      "rewards/rejected": -10.3125,
      "step": 12430
    },
    {
      "epoch": 0.89800043311918,
      "grad_norm": 8.21147210255012,
      "learning_rate": 1.5681113023071318e-08,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.46875,
      "logps/chosen": -788.0,
      "logps/rejected": -1232.0,
      "loss": 0.1399,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.625,
      "step": 12440
    },
    {
      "epoch": 0.898722298419115,
      "grad_norm": 8.618776368547858,
      "learning_rate": 1.5462252304824368e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -796.0,
      "logps/rejected": -1232.0,
      "loss": 0.1313,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 12450
    },
    {
      "epoch": 0.89944416371905,
      "grad_norm": 8.41385421629852,
      "learning_rate": 1.524488090769252e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.671875,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1507,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.375,
      "step": 12460
    },
    {
      "epoch": 0.900166029018985,
      "grad_norm": 7.873140216517367,
      "learning_rate": 1.5029000211990216e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -796.0,
      "logps/rejected": -1216.0,
      "loss": 0.1449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.5625,
      "step": 12470
    },
    {
      "epoch": 0.9008878943189201,
      "grad_norm": 9.814168221530482,
      "learning_rate": 1.4814611588565701e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -772.0,
      "logps/rejected": -1192.0,
      "loss": 0.1406,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.5,
      "step": 12480
    },
    {
      "epoch": 0.9016097596188551,
      "grad_norm": 10.539841397759815,
      "learning_rate": 1.4601716398792595e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -804.0,
      "logps/rejected": -1200.0,
      "loss": 0.171,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.34375,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.4375,
      "step": 12490
    },
    {
      "epoch": 0.9023316249187902,
      "grad_norm": 9.225273086300604,
      "learning_rate": 1.4390315994561253e-08,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.53125,
      "logps/chosen": -772.0,
      "logps/rejected": -1160.0,
      "loss": 0.149,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -10.0625,
      "step": 12500
    },
    {
      "epoch": 0.9030534902187252,
      "grad_norm": 8.885167039781129,
      "learning_rate": 1.4180411718269974e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.1346,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.3125,
      "step": 12510
    },
    {
      "epoch": 0.9037753555186602,
      "grad_norm": 9.145979733096528,
      "learning_rate": 1.3972004902816609e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -756.0,
      "logps/rejected": -1176.0,
      "loss": 0.1515,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.25,
      "step": 12520
    },
    {
      "epoch": 0.9044972208185953,
      "grad_norm": 7.295474662843511,
      "learning_rate": 1.3765096871590248e-08,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1160.0,
      "loss": 0.1301,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.8125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.0,
      "step": 12530
    },
    {
      "epoch": 0.9052190861185303,
      "grad_norm": 11.370111074734503,
      "learning_rate": 1.355968893846246e-08,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.1519,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 12540
    },
    {
      "epoch": 0.9059409514184653,
      "grad_norm": 11.025095677328915,
      "learning_rate": 1.3355782407779292e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.5625,
      "logps/chosen": -764.0,
      "logps/rejected": -1232.0,
      "loss": 0.1532,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.75,
      "step": 12550
    },
    {
      "epoch": 0.9066628167184003,
      "grad_norm": 6.549589867605967,
      "learning_rate": 1.3153378574352753e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -784.0,
      "logps/rejected": -1176.0,
      "loss": 0.1384,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.03125,
      "rewards/rejected": -10.25,
      "step": 12560
    },
    {
      "epoch": 0.9073846820183353,
      "grad_norm": 10.439039283166451,
      "learning_rate": 1.2952478723452759e-08,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5,
      "logps/chosen": -772.0,
      "logps/rejected": -1192.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.1875,
      "step": 12570
    },
    {
      "epoch": 0.9081065473182705,
      "grad_norm": 8.066549462633791,
      "learning_rate": 1.2753084130798841e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.484375,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1514,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.25,
      "step": 12580
    },
    {
      "epoch": 0.9088284126182055,
      "grad_norm": 9.16202650125988,
      "learning_rate": 1.2555196062552121e-08,
      "logits/chosen": -2.875,
      "logits/rejected": -2.484375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1475,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.4375,
      "step": 12590
    },
    {
      "epoch": 0.9095502779181405,
      "grad_norm": 9.653491629671723,
      "learning_rate": 1.2358815775307257e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -800.0,
      "logps/rejected": -1240.0,
      "loss": 0.1395,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.5625,
      "step": 12600
    },
    {
      "epoch": 0.9102721432180755,
      "grad_norm": 11.463300528930278,
      "learning_rate": 1.2163944516084434e-08,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.515625,
      "logps/chosen": -760.0,
      "logps/rejected": -1248.0,
      "loss": 0.1523,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.96875,
      "rewards/rejected": -10.9375,
      "step": 12610
    },
    {
      "epoch": 0.9109940085180105,
      "grad_norm": 7.787381760295542,
      "learning_rate": 1.197058352232147e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.578125,
      "logps/chosen": -724.0,
      "logps/rejected": -1232.0,
      "loss": 0.1429,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.8125,
      "rewards/rejected": -10.5625,
      "step": 12620
    },
    {
      "epoch": 0.9117158738179456,
      "grad_norm": 10.056484595488875,
      "learning_rate": 1.1778734021866022e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.625,
      "logps/chosen": -760.0,
      "logps/rejected": -1160.0,
      "loss": 0.1529,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.3125,
      "step": 12630
    },
    {
      "epoch": 0.9124377391178806,
      "grad_norm": 8.755038077817517,
      "learning_rate": 1.1588397232967561e-08,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.515625,
      "logps/chosen": -720.0,
      "logps/rejected": -1200.0,
      "loss": 0.1524,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.25,
      "step": 12640
    },
    {
      "epoch": 0.9131596044178156,
      "grad_norm": 7.008042272492635,
      "learning_rate": 1.1399574364269997e-08,
      "logits/chosen": -3.0,
      "logits/rejected": -2.5625,
      "logps/chosen": -760.0,
      "logps/rejected": -1208.0,
      "loss": 0.145,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.4375,
      "step": 12650
    },
    {
      "epoch": 0.9138814697177506,
      "grad_norm": 8.324802964335403,
      "learning_rate": 1.1212266614803706e-08,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.625,
      "logps/chosen": -768.0,
      "logps/rejected": -1224.0,
      "loss": 0.1378,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.5625,
      "step": 12660
    },
    {
      "epoch": 0.9146033350176856,
      "grad_norm": 10.151515852087343,
      "learning_rate": 1.1026475173977978e-08,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1216.0,
      "loss": 0.1576,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.5625,
      "step": 12670
    },
    {
      "epoch": 0.9153252003176208,
      "grad_norm": 10.157754434801875,
      "learning_rate": 1.0842201221573532e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -764.0,
      "logps/rejected": -1176.0,
      "loss": 0.1472,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.1875,
      "step": 12680
    },
    {
      "epoch": 0.9160470656175558,
      "grad_norm": 7.591920449981847,
      "learning_rate": 1.0659445927735127e-08,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -768.0,
      "logps/rejected": -1216.0,
      "loss": 0.1555,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 12690
    },
    {
      "epoch": 0.9167689309174908,
      "grad_norm": 9.794363619142695,
      "learning_rate": 1.0478210452963737e-08,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.143,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.625,
      "step": 12700
    },
    {
      "epoch": 0.9174907962174258,
      "grad_norm": 9.309110081694072,
      "learning_rate": 1.0298495948109665e-08,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -760.0,
      "logps/rejected": -1184.0,
      "loss": 0.1477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.25,
      "step": 12710
    },
    {
      "epoch": 0.9182126615173608,
      "grad_norm": 7.066887867505908,
      "learning_rate": 1.0120303554364912e-08,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 12720
    },
    {
      "epoch": 0.9189345268172959,
      "grad_norm": 11.301604093462545,
      "learning_rate": 9.943634403256046e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -748.0,
      "logps/rejected": -1232.0,
      "loss": 0.1421,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.6875,
      "step": 12730
    },
    {
      "epoch": 0.9196563921172309,
      "grad_norm": 9.577802355257152,
      "learning_rate": 9.768489616637038e-09,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.625,
      "logps/chosen": -776.0,
      "logps/rejected": -1168.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -6.25,
      "rewards/margins": 3.953125,
      "rewards/rejected": -10.1875,
      "step": 12740
    },
    {
      "epoch": 0.920378257417166,
      "grad_norm": 8.572385841169005,
      "learning_rate": 9.594870306682045e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -740.0,
      "logps/rejected": -1168.0,
      "loss": 0.1459,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.1875,
      "step": 12750
    },
    {
      "epoch": 0.921100122717101,
      "grad_norm": 10.447641927889913,
      "learning_rate": 9.42277757587842e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1200.0,
      "loss": 0.1494,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 12760
    },
    {
      "epoch": 0.9218219880170361,
      "grad_norm": 9.167052590543541,
      "learning_rate": 9.25221251701977e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1569,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.0625,
      "step": 12770
    },
    {
      "epoch": 0.9225438533169711,
      "grad_norm": 9.609226867480382,
      "learning_rate": 9.083176213198874e-09,
      "logits/chosen": -2.75,
      "logits/rejected": -2.4375,
      "logps/chosen": -760.0,
      "logps/rejected": -1192.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.375,
      "step": 12780
    },
    {
      "epoch": 0.9232657186169061,
      "grad_norm": 12.308594013486106,
      "learning_rate": 8.915669737800835e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.609375,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1484,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.4375,
      "step": 12790
    },
    {
      "epoch": 0.9239875839168411,
      "grad_norm": 7.740941902213041,
      "learning_rate": 8.74969415449639e-09,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -776.0,
      "logps/rejected": -1200.0,
      "loss": 0.1305,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.4375,
      "step": 12800
    },
    {
      "epoch": 0.9247094492167761,
      "grad_norm": 9.587184945397313,
      "learning_rate": 8.585250517235048e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -796.0,
      "logps/rejected": -1216.0,
      "loss": 0.1525,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.5625,
      "step": 12810
    },
    {
      "epoch": 0.9254313145167112,
      "grad_norm": 12.236899096306711,
      "learning_rate": 8.422339870238409e-09,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -764.0,
      "logps/rejected": -1192.0,
      "loss": 0.1401,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.3125,
      "step": 12820
    },
    {
      "epoch": 0.9261531798166462,
      "grad_norm": 9.335127467437971,
      "learning_rate": 8.26096324799347e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1642,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 12830
    },
    {
      "epoch": 0.9268750451165813,
      "grad_norm": 10.11030845129818,
      "learning_rate": 8.101121675246293e-09,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1192.0,
      "loss": 0.1364,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.4375,
      "step": 12840
    },
    {
      "epoch": 0.9275969104165163,
      "grad_norm": 9.949772616020681,
      "learning_rate": 7.942816166995187e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -772.0,
      "logps/rejected": -1176.0,
      "loss": 0.1612,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.3125,
      "step": 12850
    },
    {
      "epoch": 0.9283187757164513,
      "grad_norm": 11.04520640854637,
      "learning_rate": 7.786047728484429e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -772.0,
      "logps/rejected": -1216.0,
      "loss": 0.1363,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.3125,
      "step": 12860
    },
    {
      "epoch": 0.9290406410163864,
      "grad_norm": 10.967381676803216,
      "learning_rate": 7.630817355197994e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -748.0,
      "logps/rejected": -1128.0,
      "loss": 0.1426,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.9375,
      "step": 12870
    },
    {
      "epoch": 0.9297625063163214,
      "grad_norm": 9.36940835074148,
      "learning_rate": 7.477126032852888e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -776.0,
      "logps/rejected": -1192.0,
      "loss": 0.1423,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.375,
      "step": 12880
    },
    {
      "epoch": 0.9304843716162564,
      "grad_norm": 6.9344949791967885,
      "learning_rate": 7.324974737393241e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -792.0,
      "logps/rejected": -1200.0,
      "loss": 0.1347,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.625,
      "step": 12890
    },
    {
      "epoch": 0.9312062369161914,
      "grad_norm": 8.914498439457502,
      "learning_rate": 7.174364434984009e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -788.0,
      "logps/rejected": -1192.0,
      "loss": 0.1546,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 12900
    },
    {
      "epoch": 0.9319281022161264,
      "grad_norm": 11.255791378742503,
      "learning_rate": 7.025296082004667e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -764.0,
      "logps/rejected": -1184.0,
      "loss": 0.1693,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.4375,
      "step": 12910
    },
    {
      "epoch": 0.9326499675160616,
      "grad_norm": 10.43918973974646,
      "learning_rate": 6.8777706250433274e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1208.0,
      "loss": 0.1538,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 12920
    },
    {
      "epoch": 0.9333718328159966,
      "grad_norm": 9.338539505172553,
      "learning_rate": 6.731789000890775e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -760.0,
      "logps/rejected": -1192.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.5,
      "step": 12930
    },
    {
      "epoch": 0.9340936981159316,
      "grad_norm": 8.36112263605316,
      "learning_rate": 6.587352136534219e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5625,
      "logps/chosen": -768.0,
      "logps/rejected": -1192.0,
      "loss": 0.1377,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 12940
    },
    {
      "epoch": 0.9348155634158666,
      "grad_norm": 8.797834148383416,
      "learning_rate": 6.444460949151714e-09,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.546875,
      "logps/chosen": -748.0,
      "logps/rejected": -1176.0,
      "loss": 0.1499,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.3125,
      "step": 12950
    },
    {
      "epoch": 0.9355374287158016,
      "grad_norm": 11.053337065925593,
      "learning_rate": 6.303116346106224e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -792.0,
      "logps/rejected": -1216.0,
      "loss": 0.1535,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.5625,
      "step": 12960
    },
    {
      "epoch": 0.9362592940157367,
      "grad_norm": 11.276613104833107,
      "learning_rate": 6.163319224939872e-09,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.65625,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1325,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.3125,
      "step": 12970
    },
    {
      "epoch": 0.9369811593156717,
      "grad_norm": 8.755018080988055,
      "learning_rate": 6.025070473368144e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -760.0,
      "logps/rejected": -1184.0,
      "loss": 0.1502,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.375,
      "step": 12980
    },
    {
      "epoch": 0.9377030246156067,
      "grad_norm": 9.43774867419776,
      "learning_rate": 5.888370969274442e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5,
      "logps/chosen": -772.0,
      "logps/rejected": -1152.0,
      "loss": 0.1572,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.1875,
      "step": 12990
    },
    {
      "epoch": 0.9384248899155417,
      "grad_norm": 9.85434140441404,
      "learning_rate": 5.7532215807043486e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.1319,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.5,
      "step": 13000
    },
    {
      "epoch": 0.9391467552154767,
      "grad_norm": 7.193797068429037,
      "learning_rate": 5.6196231658601764e-09,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.671875,
      "logps/chosen": -812.0,
      "logps/rejected": -1232.0,
      "loss": 0.1416,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.46875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.6875,
      "step": 13010
    },
    {
      "epoch": 0.9398686205154119,
      "grad_norm": 12.631151758721385,
      "learning_rate": 5.48757657309551e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -740.0,
      "logps/rejected": -1176.0,
      "loss": 0.1462,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.3125,
      "step": 13020
    },
    {
      "epoch": 0.9405904858153469,
      "grad_norm": 5.307134460751131,
      "learning_rate": 5.357082640909866e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -744.0,
      "logps/rejected": -1184.0,
      "loss": 0.1358,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.25,
      "step": 13030
    },
    {
      "epoch": 0.9413123511152819,
      "grad_norm": 10.087417447571848,
      "learning_rate": 5.22814219794318e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -784.0,
      "logps/rejected": -1256.0,
      "loss": 0.1444,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.8125,
      "step": 13040
    },
    {
      "epoch": 0.9420342164152169,
      "grad_norm": 7.00443265106439,
      "learning_rate": 5.10075606297089e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1438,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.5625,
      "step": 13050
    },
    {
      "epoch": 0.9427560817151519,
      "grad_norm": 7.793436936582348,
      "learning_rate": 4.974925044898437e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.515625,
      "logps/chosen": -788.0,
      "logps/rejected": -1208.0,
      "loss": 0.1479,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.4375,
      "step": 13060
    },
    {
      "epoch": 0.943477947015087,
      "grad_norm": 6.458652354786663,
      "learning_rate": 4.850649942756135e-09,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.65625,
      "logps/chosen": -780.0,
      "logps/rejected": -1160.0,
      "loss": 0.1339,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.125,
      "rewards/rejected": -10.0625,
      "step": 13070
    },
    {
      "epoch": 0.944199812315022,
      "grad_norm": 7.868041435801801,
      "learning_rate": 4.727931545694397e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1192.0,
      "loss": 0.1424,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.4375,
      "step": 13080
    },
    {
      "epoch": 0.944921677614957,
      "grad_norm": 8.264459996035683,
      "learning_rate": 4.606770632978374e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.4375,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1518,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 13090
    },
    {
      "epoch": 0.945643542914892,
      "grad_norm": 8.88295658076357,
      "learning_rate": 4.4871679739831304e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1216.0,
      "loss": 0.1439,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.375,
      "step": 13100
    },
    {
      "epoch": 0.9463654082148271,
      "grad_norm": 20.8259081865564,
      "learning_rate": 4.369124328188839e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.671875,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1665,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.4375,
      "step": 13110
    },
    {
      "epoch": 0.9470872735147622,
      "grad_norm": 7.245245477976355,
      "learning_rate": 4.252640445175898e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -784.0,
      "logps/rejected": -1216.0,
      "loss": 0.1335,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5,
      "step": 13120
    },
    {
      "epoch": 0.9478091388146972,
      "grad_norm": 9.917996431238413,
      "learning_rate": 4.1377170646201555e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.5,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.6875,
      "step": 13130
    },
    {
      "epoch": 0.9485310041146322,
      "grad_norm": 11.746041159172476,
      "learning_rate": 4.024354916288192e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.609375,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.17,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.21875,
      "rewards/rejected": -10.3125,
      "step": 13140
    },
    {
      "epoch": 0.9492528694145672,
      "grad_norm": 7.791108577213938,
      "learning_rate": 3.912554720032796e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -784.0,
      "logps/rejected": -1192.0,
      "loss": 0.1555,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.5,
      "step": 13150
    },
    {
      "epoch": 0.9499747347145022,
      "grad_norm": 7.770825664424211,
      "learning_rate": 3.8023171857882456e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1192.0,
      "loss": 0.1343,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.5,
      "step": 13160
    },
    {
      "epoch": 0.9506966000144373,
      "grad_norm": 7.654311704309095,
      "learning_rate": 3.693643013565867e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.546875,
      "logps/chosen": -748.0,
      "logps/rejected": -1192.0,
      "loss": 0.1394,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.5,
      "step": 13170
    },
    {
      "epoch": 0.9514184653143724,
      "grad_norm": 10.335438911558372,
      "learning_rate": 3.586532893449706e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.546875,
      "logps/chosen": -720.0,
      "logps/rejected": -1176.0,
      "loss": 0.1659,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.65625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.25,
      "step": 13180
    },
    {
      "epoch": 0.9521403306143074,
      "grad_norm": 9.99678225643105,
      "learning_rate": 3.4809875055918923e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.703125,
      "logps/chosen": -764.0,
      "logps/rejected": -1208.0,
      "loss": 0.1307,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.625,
      "step": 13190
    },
    {
      "epoch": 0.9528621959142424,
      "grad_norm": 6.490173084222523,
      "learning_rate": 3.3770075202085304e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.765625,
      "logps/chosen": -768.0,
      "logps/rejected": -1208.0,
      "loss": 0.1413,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.5,
      "step": 13200
    },
    {
      "epoch": 0.9535840612141774,
      "grad_norm": 8.060395264263372,
      "learning_rate": 3.274593597575398e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.1627,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.25,
      "step": 13210
    },
    {
      "epoch": 0.9543059265141125,
      "grad_norm": 7.41254192877576,
      "learning_rate": 3.173746388023729e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.453125,
      "logps/chosen": -784.0,
      "logps/rejected": -1248.0,
      "loss": 0.1404,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.75,
      "rewards/rejected": -10.8125,
      "step": 13220
    },
    {
      "epoch": 0.9550277918140475,
      "grad_norm": 11.024215888855808,
      "learning_rate": 3.074466531935993e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.65625,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1485,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5,
      "step": 13230
    },
    {
      "epoch": 0.9557496571139825,
      "grad_norm": 11.151738805917098,
      "learning_rate": 2.976754659742037e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5,
      "logps/chosen": -784.0,
      "logps/rejected": -1192.0,
      "loss": 0.1603,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.3125,
      "step": 13240
    },
    {
      "epoch": 0.9564715224139175,
      "grad_norm": 6.773241527202388,
      "learning_rate": 2.88061139191495e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -760.0,
      "logps/rejected": -1200.0,
      "loss": 0.1285,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.4375,
      "step": 13250
    },
    {
      "epoch": 0.9571933877138527,
      "grad_norm": 15.814195118042852,
      "learning_rate": 2.7860373389670398e-09,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.640625,
      "logps/chosen": -744.0,
      "logps/rejected": -1176.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.25,
      "step": 13260
    },
    {
      "epoch": 0.9579152530137877,
      "grad_norm": 10.16676403861108,
      "learning_rate": 2.693033101446196e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -752.0,
      "logps/rejected": -1208.0,
      "loss": 0.1496,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 13270
    },
    {
      "epoch": 0.9586371183137227,
      "grad_norm": 8.485290989034366,
      "learning_rate": 2.6015992699318366e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.484375,
      "logps/chosen": -752.0,
      "logps/rejected": -1160.0,
      "loss": 0.1261,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.25,
      "step": 13280
    },
    {
      "epoch": 0.9593589836136577,
      "grad_norm": 9.904715194680257,
      "learning_rate": 2.5117364250312744e-09,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5,
      "logps/chosen": -760.0,
      "logps/rejected": -1208.0,
      "loss": 0.1563,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.65625,
      "rewards/rejected": -10.625,
      "step": 13290
    },
    {
      "epoch": 0.9600808489135927,
      "grad_norm": 12.293208072952874,
      "learning_rate": 2.4234451373761068e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.734375,
      "logps/chosen": -792.0,
      "logps/rejected": -1248.0,
      "loss": 0.1576,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.9375,
      "step": 13300
    },
    {
      "epoch": 0.9608027142135278,
      "grad_norm": 9.583806839385215,
      "learning_rate": 2.336725967618358e-09,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.154,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.25,
      "step": 13310
    },
    {
      "epoch": 0.9615245795134628,
      "grad_norm": 9.704485535689619,
      "learning_rate": 2.2515794664271502e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.1403,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.625,
      "step": 13320
    },
    {
      "epoch": 0.9622464448133978,
      "grad_norm": 10.451439623478995,
      "learning_rate": 2.168006174485121e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -2.65625,
      "logps/chosen": -768.0,
      "logps/rejected": -1216.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.75,
      "step": 13330
    },
    {
      "epoch": 0.9629683101133328,
      "grad_norm": 8.838700241798461,
      "learning_rate": 2.0860066224848983e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -784.0,
      "logps/rejected": -1208.0,
      "loss": 0.1344,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.3125,
      "step": 13340
    },
    {
      "epoch": 0.9636901754132678,
      "grad_norm": 10.958902722065448,
      "learning_rate": 2.0055813311259383e-09,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1216.0,
      "loss": 0.1581,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.53125,
      "rewards/rejected": -10.625,
      "step": 13350
    },
    {
      "epoch": 0.964412040713203,
      "grad_norm": 6.09181790905039,
      "learning_rate": 1.926730811111027e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1414,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.0625,
      "rewards/rejected": -10.1875,
      "step": 13360
    },
    {
      "epoch": 0.965133906013138,
      "grad_norm": 9.88628572992933,
      "learning_rate": 1.849455563143143e-09,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -784.0,
      "logps/rejected": -1216.0,
      "loss": 0.1567,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.625,
      "step": 13370
    },
    {
      "epoch": 0.965855771313073,
      "grad_norm": 13.53292753569823,
      "learning_rate": 1.773756077922156e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.609375,
      "logps/chosen": -796.0,
      "logps/rejected": -1184.0,
      "loss": 0.1583,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.09375,
      "rewards/rejected": -10.25,
      "step": 13380
    },
    {
      "epoch": 0.966577636613008,
      "grad_norm": 8.793000871412223,
      "learning_rate": 1.69963283614194e-09,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.578125,
      "logps/chosen": -772.0,
      "logps/rejected": -1168.0,
      "loss": 0.155,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.1875,
      "step": 13390
    },
    {
      "epoch": 0.967299501912943,
      "grad_norm": 8.92187666322989,
      "learning_rate": 1.6270863084870967e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -768.0,
      "logps/rejected": -1160.0,
      "loss": 0.1348,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.0625,
      "step": 13400
    },
    {
      "epoch": 0.9680213672128781,
      "grad_norm": 7.727103982802486,
      "learning_rate": 1.5561169556300157e-09,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5,
      "logps/chosen": -756.0,
      "logps/rejected": -1192.0,
      "loss": 0.1426,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.375,
      "step": 13410
    },
    {
      "epoch": 0.9687432325128131,
      "grad_norm": 10.615949449025017,
      "learning_rate": 1.4867252282280974e-09,
      "logits/chosen": -2.875,
      "logits/rejected": -2.46875,
      "logps/chosen": -772.0,
      "logps/rejected": -1224.0,
      "loss": 0.1412,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.625,
      "step": 13420
    },
    {
      "epoch": 0.9694650978127481,
      "grad_norm": 8.424515914002487,
      "learning_rate": 1.4189115669206719e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.703125,
      "logps/chosen": -768.0,
      "logps/rejected": -1176.0,
      "loss": 0.1342,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 13430
    },
    {
      "epoch": 0.9701869631126832,
      "grad_norm": 9.868921088219077,
      "learning_rate": 1.3526764023263082e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.6875,
      "logps/chosen": -768.0,
      "logps/rejected": -1168.0,
      "loss": 0.1618,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.125,
      "step": 13440
    },
    {
      "epoch": 0.9709088284126182,
      "grad_norm": 12.402501755945687,
      "learning_rate": 1.288020155040176e-09,
      "logits/chosen": -3.0,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1224.0,
      "loss": 0.1346,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.375,
      "step": 13450
    },
    {
      "epoch": 0.9716306937125533,
      "grad_norm": 5.426424943097187,
      "learning_rate": 1.2249432356311874e-09,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.734375,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.375,
      "step": 13460
    },
    {
      "epoch": 0.9723525590124883,
      "grad_norm": 9.000433341065145,
      "learning_rate": 1.163446044639499e-09,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.515625,
      "logps/chosen": -768.0,
      "logps/rejected": -1232.0,
      "loss": 0.1623,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.8125,
      "step": 13470
    },
    {
      "epoch": 0.9730744243124233,
      "grad_norm": 8.594542564403534,
      "learning_rate": 1.103528972573986e-09,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.453125,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1483,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.375,
      "step": 13480
    },
    {
      "epoch": 0.9737962896123583,
      "grad_norm": 11.926460876248191,
      "learning_rate": 1.045192399909689e-09,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.46875,
      "logps/chosen": -776.0,
      "logps/rejected": -1232.0,
      "loss": 0.1516,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.5625,
      "step": 13490
    },
    {
      "epoch": 0.9745181549122933,
      "grad_norm": 8.469305025697546,
      "learning_rate": 9.884366970853986e-10,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.59375,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.1545,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.75,
      "step": 13500
    },
    {
      "epoch": 0.9752400202122284,
      "grad_norm": 9.638473562645952,
      "learning_rate": 9.332622245014355e-10,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -764.0,
      "logps/rejected": -1216.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.625,
      "step": 13510
    },
    {
      "epoch": 0.9759618855121635,
      "grad_norm": 10.982840589873115,
      "learning_rate": 8.796693325171522e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.65625,
      "logps/chosen": -768.0,
      "logps/rejected": -1200.0,
      "loss": 0.1414,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.5,
      "step": 13520
    },
    {
      "epoch": 0.9766837508120985,
      "grad_norm": 9.160859372694532,
      "learning_rate": 8.276583614489352e-10,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -780.0,
      "logps/rejected": -1184.0,
      "loss": 0.1397,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.3125,
      "rewards/rejected": -10.375,
      "step": 13530
    },
    {
      "epoch": 0.9774056161120335,
      "grad_norm": 11.936529679612834,
      "learning_rate": 7.772296415678447e-10,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.546875,
      "logps/chosen": -796.0,
      "logps/rejected": -1224.0,
      "loss": 0.1344,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.3125,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.8125,
      "step": 13540
    },
    {
      "epoch": 0.9781274814119685,
      "grad_norm": 7.009572464668358,
      "learning_rate": 7.283834930976451e-10,
      "logits/chosen": -3.0,
      "logits/rejected": -2.625,
      "logps/chosen": -752.0,
      "logps/rejected": -1192.0,
      "loss": 0.1445,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.5625,
      "rewards/rejected": -10.5,
      "step": 13550
    },
    {
      "epoch": 0.9788493467119036,
      "grad_norm": 7.948146385621628,
      "learning_rate": 6.811202262126947e-10,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.59375,
      "logps/chosen": -780.0,
      "logps/rejected": -1240.0,
      "loss": 0.1409,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.75,
      "step": 13560
    },
    {
      "epoch": 0.9795712120118386,
      "grad_norm": 8.433006482064645,
      "learning_rate": 6.354401410360588e-10,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -760.0,
      "logps/rejected": -1176.0,
      "loss": 0.1257,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.4375,
      "step": 13570
    },
    {
      "epoch": 0.9802930773117736,
      "grad_norm": 12.067512536747236,
      "learning_rate": 5.913435276374834e-10,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.65625,
      "logps/chosen": -768.0,
      "logps/rejected": -1184.0,
      "loss": 0.163,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.15625,
      "rewards/rejected": -10.3125,
      "step": 13580
    },
    {
      "epoch": 0.9810149426117086,
      "grad_norm": 8.546742612397637,
      "learning_rate": 5.488306660317299e-10,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -756.0,
      "logps/rejected": -1112.0,
      "loss": 0.1459,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.8125,
      "step": 13590
    },
    {
      "epoch": 0.9817368079116436,
      "grad_norm": 10.59407530013682,
      "learning_rate": 5.079018261766044e-10,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5,
      "logps/chosen": -752.0,
      "logps/rejected": -1216.0,
      "loss": 0.15,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.78125,
      "rewards/margins": 4.6875,
      "rewards/rejected": -10.4375,
      "step": 13600
    },
    {
      "epoch": 0.9824586732115788,
      "grad_norm": 11.513879858085009,
      "learning_rate": 4.685572679713478e-10,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1168.0,
      "loss": 0.153,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.125,
      "step": 13610
    },
    {
      "epoch": 0.9831805385115138,
      "grad_norm": 11.717526217203766,
      "learning_rate": 4.3079724125499875e-10,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -800.0,
      "logps/rejected": -1168.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.28125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -10.1875,
      "step": 13620
    },
    {
      "epoch": 0.9839024038114488,
      "grad_norm": 8.065660623179564,
      "learning_rate": 3.9462198580475505e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.78125,
      "logps/chosen": -784.0,
      "logps/rejected": -1208.0,
      "loss": 0.1549,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.15625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.5625,
      "step": 13630
    },
    {
      "epoch": 0.9846242691113838,
      "grad_norm": 11.648253958259497,
      "learning_rate": 3.6003173133447585e-10,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.46875,
      "logps/chosen": -764.0,
      "logps/rejected": -1192.0,
      "loss": 0.1504,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.4375,
      "step": 13640
    },
    {
      "epoch": 0.9853461344113188,
      "grad_norm": 12.43000507016397,
      "learning_rate": 3.270266974932101e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.546875,
      "logps/chosen": -792.0,
      "logps/rejected": -1256.0,
      "loss": 0.1476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.75,
      "step": 13650
    },
    {
      "epoch": 0.9860679997112539,
      "grad_norm": 10.176611290026623,
      "learning_rate": 2.956070938638644e-10,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.59375,
      "logps/chosen": -736.0,
      "logps/rejected": -1232.0,
      "loss": 0.1565,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.90625,
      "rewards/margins": 5.0,
      "rewards/rejected": -10.875,
      "step": 13660
    },
    {
      "epoch": 0.9867898650111889,
      "grad_norm": 8.384903385562643,
      "learning_rate": 2.6577311996175964e-10,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.625,
      "logps/chosen": -760.0,
      "logps/rejected": -1224.0,
      "loss": 0.1404,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.125,
      "rewards/margins": 4.625,
      "rewards/rejected": -10.75,
      "step": 13670
    },
    {
      "epoch": 0.9875117303111239,
      "grad_norm": 7.9305338057329005,
      "learning_rate": 2.3752496523343767e-10,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.640625,
      "logps/chosen": -756.0,
      "logps/rejected": -1184.0,
      "loss": 0.1398,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.28125,
      "rewards/rejected": -10.3125,
      "step": 13680
    },
    {
      "epoch": 0.988233595611059,
      "grad_norm": 12.291251192914357,
      "learning_rate": 2.1086280905543984e-10,
      "logits/chosen": -2.875,
      "logits/rejected": -2.484375,
      "logps/chosen": -768.0,
      "logps/rejected": -1216.0,
      "loss": 0.1289,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.84375,
      "rewards/rejected": -10.75,
      "step": 13690
    },
    {
      "epoch": 0.988955460910994,
      "grad_norm": 6.115365197215902,
      "learning_rate": 1.8578682073322472e-10,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.640625,
      "logps/chosen": -764.0,
      "logps/rejected": -1184.0,
      "loss": 0.128,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -5.96875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.375,
      "step": 13700
    },
    {
      "epoch": 0.9896773262109291,
      "grad_norm": 11.676458111445147,
      "learning_rate": 1.6229715950000221e-10,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.53125,
      "logps/chosen": -776.0,
      "logps/rejected": -1232.0,
      "loss": 0.1443,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.03125,
      "rewards/margins": 4.71875,
      "rewards/rejected": -10.75,
      "step": 13710
    },
    {
      "epoch": 0.9903991915108641,
      "grad_norm": 8.892462780201626,
      "learning_rate": 1.4039397451573455e-10,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -756.0,
      "logps/rejected": -1152.0,
      "loss": 0.1338,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.25,
      "step": 13720
    },
    {
      "epoch": 0.9911210568107991,
      "grad_norm": 8.896892206063134,
      "learning_rate": 1.2007740486624785e-10,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -776.0,
      "logps/rejected": -1216.0,
      "loss": 0.1626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.25,
      "rewards/margins": 4.34375,
      "rewards/rejected": -10.625,
      "step": 13730
    },
    {
      "epoch": 0.9918429221107341,
      "grad_norm": 8.41403414937158,
      "learning_rate": 1.0134757956234418e-10,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.59375,
      "logps/chosen": -784.0,
      "logps/rejected": -1200.0,
      "loss": 0.1526,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.1875,
      "rewards/margins": 4.1875,
      "rewards/rejected": -10.375,
      "step": 13740
    },
    {
      "epoch": 0.9925647874106691,
      "grad_norm": 8.437786909628086,
      "learning_rate": 8.420461753891327e-11,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -752.0,
      "logps/rejected": -1168.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.84375,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.25,
      "step": 13750
    },
    {
      "epoch": 0.9932866527106042,
      "grad_norm": 9.160480007480118,
      "learning_rate": 6.864862765421087e-11,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.703125,
      "logps/chosen": -752.0,
      "logps/rejected": -1184.0,
      "loss": 0.15,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.4375,
      "step": 13760
    },
    {
      "epoch": 0.9940085180105392,
      "grad_norm": 11.808273487978743,
      "learning_rate": 5.4679708689248095e-11,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.71875,
      "logps/chosen": -756.0,
      "logps/rejected": -1176.0,
      "loss": 0.1254,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.40625,
      "rewards/rejected": -10.3125,
      "step": 13770
    },
    {
      "epoch": 0.9947303833104743,
      "grad_norm": 11.167829268938808,
      "learning_rate": 4.2297949346986606e-11,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -760.0,
      "logps/rejected": -1192.0,
      "loss": 0.1435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.90625,
      "rewards/margins": 4.375,
      "rewards/rejected": -10.25,
      "step": 13780
    },
    {
      "epoch": 0.9954522486104093,
      "grad_norm": 8.328945950486863,
      "learning_rate": 3.150342825197771e-11,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -788.0,
      "logps/rejected": -1232.0,
      "loss": 0.1432,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -6.21875,
      "rewards/margins": 4.4375,
      "rewards/rejected": -10.625,
      "step": 13790
    },
    {
      "epoch": 0.9961741139103444,
      "grad_norm": 8.527974081958734,
      "learning_rate": 2.2296213949696273e-11,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.625,
      "logps/chosen": -788.0,
      "logps/rejected": -1184.0,
      "loss": 0.1509,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.1875,
      "rewards/margins": 3.984375,
      "rewards/rejected": -10.1875,
      "step": 13800
    },
    {
      "epoch": 0.9968959792102794,
      "grad_norm": 5.882710049110608,
      "learning_rate": 1.4676364906235362e-11,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -772.0,
      "logps/rejected": -1184.0,
      "loss": 0.1555,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.9375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.375,
      "step": 13810
    },
    {
      "epoch": 0.9976178445102144,
      "grad_norm": 8.19009325593564,
      "learning_rate": 8.643929507834435e-12,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.53125,
      "logps/chosen": -760.0,
      "logps/rejected": -1216.0,
      "loss": 0.1495,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -6.0625,
      "rewards/margins": 4.59375,
      "rewards/rejected": -10.625,
      "step": 13820
    },
    {
      "epoch": 0.9983397098101494,
      "grad_norm": 8.07778039690577,
      "learning_rate": 4.198946060601782e-12,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.71875,
      "logps/chosen": -776.0,
      "logps/rejected": -1200.0,
      "loss": 0.1558,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.46875,
      "rewards/rejected": -10.5,
      "step": 13830
    },
    {
      "epoch": 0.9990615751100844,
      "grad_norm": 9.049466464418751,
      "learning_rate": 1.3414427903202242e-12,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -776.0,
      "logps/rejected": -1240.0,
      "loss": 0.139,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.09375,
      "rewards/margins": 4.5,
      "rewards/rejected": -10.5625,
      "step": 13840
    },
    {
      "epoch": 0.9997834404100195,
      "grad_norm": 7.329571223136626,
      "learning_rate": 7.143784222507499e-14,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -772.0,
      "logps/rejected": -1192.0,
      "loss": 0.1357,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -6.0,
      "rewards/margins": 4.25,
      "rewards/rejected": -10.25,
      "step": 13850
    },
    {
      "epoch": 1.0,
      "step": 13853,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 39.4392,
      "train_samples_per_second": 44959.109,
      "train_steps_per_second": 351.249
    }
  ],
  "logging_steps": 10,
  "max_steps": 13853,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}