{
  "best_metric": 0.22352416813373566,
  "best_model_checkpoint": "models/qwen2.5-3b-dpo-finegrained/checkpoint-5000",
  "epoch": 0.36093264996751606,
  "eval_steps": 5000,
  "global_step": 5000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 7.218652999350322e-05,
      "grad_norm": 1.852354340577762,
      "learning_rate": 3.6075036075036073e-10,
      "logits/chosen": -2.59375,
      "logits/rejected": -1.6015625,
      "logps/chosen": -146.0,
      "logps/rejected": -166.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0007218652999350321,
      "grad_norm": 1.998199384338708,
      "learning_rate": 3.6075036075036073e-09,
      "logits/chosen": -2.265625,
      "logits/rejected": -1.953125,
      "logps/chosen": -157.0,
      "logps/rejected": -143.0,
      "loss": 0.6922,
      "rewards/accuracies": 0.2638888955116272,
      "rewards/chosen": 0.000766754150390625,
      "rewards/margins": 0.000606536865234375,
      "rewards/rejected": 0.000156402587890625,
      "step": 10
    },
    {
      "epoch": 0.0014437305998700643,
      "grad_norm": 1.859902408963682,
      "learning_rate": 7.215007215007215e-09,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.984375,
      "logps/chosen": -157.0,
      "logps/rejected": -139.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.001129150390625,
      "rewards/margins": 0.00061798095703125,
      "rewards/rejected": 0.000507354736328125,
      "step": 20
    },
    {
      "epoch": 0.0021655958998050965,
      "grad_norm": 1.800945636929482,
      "learning_rate": 1.0822510822510822e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.015625,
      "logps/chosen": -146.0,
      "logps/rejected": -148.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.000640869140625,
      "rewards/margins": -0.0003757476806640625,
      "rewards/rejected": 0.00101470947265625,
      "step": 30
    },
    {
      "epoch": 0.0028874611997401285,
      "grad_norm": 1.9512697515276394,
      "learning_rate": 1.443001443001443e-08,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.0,
      "logps/chosen": -165.0,
      "logps/rejected": -161.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 0.00010967254638671875,
      "rewards/margins": 0.0003910064697265625,
      "rewards/rejected": -0.00028228759765625,
      "step": 40
    },
    {
      "epoch": 0.0036093264996751606,
      "grad_norm": 1.9685077450218404,
      "learning_rate": 1.8037518037518035e-08,
      "logits/chosen": -2.21875,
      "logits/rejected": -2.015625,
      "logps/chosen": -165.0,
      "logps/rejected": -143.0,
      "loss": 0.6929,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": 9.441375732421875e-05,
      "rewards/margins": 0.0001239776611328125,
      "rewards/rejected": -3.0159950256347656e-05,
      "step": 50
    },
    {
      "epoch": 0.004331191799610193,
      "grad_norm": 1.9944209672205195,
      "learning_rate": 2.1645021645021644e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.984375,
      "logps/chosen": -160.0,
      "logps/rejected": -136.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": -0.000232696533203125,
      "rewards/margins": 0.00102996826171875,
      "rewards/rejected": -0.0012664794921875,
      "step": 60
    },
    {
      "epoch": 0.005053057099545225,
      "grad_norm": 1.8602055997261977,
      "learning_rate": 2.525252525252525e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -1.8984375,
      "logps/chosen": -150.0,
      "logps/rejected": -154.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.000438690185546875,
      "rewards/margins": 0.00054931640625,
      "rewards/rejected": -0.00010919570922851562,
      "step": 70
    },
    {
      "epoch": 0.005774922399480257,
      "grad_norm": 1.7129972696287459,
      "learning_rate": 2.886002886002886e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.046875,
      "logps/chosen": -170.0,
      "logps/rejected": -145.0,
      "loss": 0.6928,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.000579833984375,
      "rewards/margins": 0.0004520416259765625,
      "rewards/rejected": 0.0001239776611328125,
      "step": 80
    },
    {
      "epoch": 0.006496787699415289,
      "grad_norm": 1.8246478253492482,
      "learning_rate": 3.246753246753246e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.0625,
      "logps/chosen": -178.0,
      "logps/rejected": -158.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 0.000469207763671875,
      "rewards/margins": 0.0001888275146484375,
      "rewards/rejected": 0.0002803802490234375,
      "step": 90
    },
    {
      "epoch": 0.007218652999350321,
      "grad_norm": 1.7657315202986226,
      "learning_rate": 3.607503607503607e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -1.984375,
      "logps/chosen": -172.0,
      "logps/rejected": -163.0,
      "loss": 0.6927,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 0.000797271728515625,
      "rewards/margins": -0.0003910064697265625,
      "rewards/rejected": 0.001190185546875,
      "step": 100
    },
    {
      "epoch": 0.007940518299285354,
      "grad_norm": 2.0123631368082844,
      "learning_rate": 3.968253968253968e-08,
      "logits/chosen": -2.453125,
      "logits/rejected": -1.9296875,
      "logps/chosen": -153.0,
      "logps/rejected": -140.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": 0.000186920166015625,
      "rewards/margins": 0.00018787384033203125,
      "rewards/rejected": 1.1473894119262695e-06,
      "step": 110
    },
    {
      "epoch": 0.008662383599220386,
      "grad_norm": 1.9667654175700489,
      "learning_rate": 4.329004329004329e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.0,
      "logps/chosen": -146.0,
      "logps/rejected": -137.0,
      "loss": 0.6926,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.00154876708984375,
      "rewards/margins": -0.00103759765625,
      "rewards/rejected": 0.0025787353515625,
      "step": 120
    },
    {
      "epoch": 0.009384248899155418,
      "grad_norm": 1.7516436962452608,
      "learning_rate": 4.68975468975469e-08,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.1875,
      "logps/chosen": -152.0,
      "logps/rejected": -151.0,
      "loss": 0.6925,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.0027313232421875,
      "rewards/margins": 0.00164031982421875,
      "rewards/rejected": 0.0010986328125,
      "step": 130
    },
    {
      "epoch": 0.01010611419909045,
      "grad_norm": 2.0629971445302435,
      "learning_rate": 5.05050505050505e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.046875,
      "logps/chosen": -167.0,
      "logps/rejected": -152.0,
      "loss": 0.6921,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.0018768310546875,
      "rewards/margins": 0.000438690185546875,
      "rewards/rejected": 0.00144195556640625,
      "step": 140
    },
    {
      "epoch": 0.010827979499025482,
      "grad_norm": 2.0455409873804205,
      "learning_rate": 5.411255411255411e-08,
      "logits/chosen": -2.375,
      "logits/rejected": -2.09375,
      "logps/chosen": -165.0,
      "logps/rejected": -147.0,
      "loss": 0.6923,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.001739501953125,
      "rewards/margins": 5.435943603515625e-05,
      "rewards/rejected": 0.001678466796875,
      "step": 150
    },
    {
      "epoch": 0.011549844798960514,
      "grad_norm": 2.003008491192784,
      "learning_rate": 5.772005772005772e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -1.8359375,
      "logps/chosen": -166.0,
      "logps/rejected": -161.0,
      "loss": 0.6919,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0026702880859375,
      "rewards/margins": 0.00093841552734375,
      "rewards/rejected": 0.00173187255859375,
      "step": 160
    },
    {
      "epoch": 0.012271710098895546,
      "grad_norm": 1.8970983946398985,
      "learning_rate": 6.132756132756133e-08,
      "logits/chosen": -2.34375,
      "logits/rejected": -1.9921875,
      "logps/chosen": -163.0,
      "logps/rejected": -142.0,
      "loss": 0.6915,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00445556640625,
      "rewards/margins": 0.00396728515625,
      "rewards/rejected": 0.0004787445068359375,
      "step": 170
    },
    {
      "epoch": 0.012993575398830578,
      "grad_norm": 1.9000812663628288,
      "learning_rate": 6.493506493506492e-08,
      "logits/chosen": -2.390625,
      "logits/rejected": -1.9140625,
      "logps/chosen": -184.0,
      "logps/rejected": -166.0,
      "loss": 0.6913,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004791259765625,
      "rewards/margins": 0.00274658203125,
      "rewards/rejected": 0.0020294189453125,
      "step": 180
    },
    {
      "epoch": 0.01371544069876561,
      "grad_norm": 1.7694192875136483,
      "learning_rate": 6.854256854256854e-08,
      "logits/chosen": -2.375,
      "logits/rejected": -2.015625,
      "logps/chosen": -168.0,
      "logps/rejected": -147.0,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.007598876953125,
      "rewards/margins": 0.003387451171875,
      "rewards/rejected": 0.00421142578125,
      "step": 190
    },
    {
      "epoch": 0.014437305998700642,
      "grad_norm": 1.8022914272788901,
      "learning_rate": 7.215007215007214e-08,
      "logits/chosen": -2.296875,
      "logits/rejected": -1.890625,
      "logps/chosen": -150.0,
      "logps/rejected": -137.0,
      "loss": 0.6907,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.007354736328125,
      "rewards/margins": 0.005584716796875,
      "rewards/rejected": 0.0017547607421875,
      "step": 200
    },
    {
      "epoch": 0.015159171298635674,
      "grad_norm": 1.7754509638333658,
      "learning_rate": 7.575757575757576e-08,
      "logits/chosen": -2.328125,
      "logits/rejected": -2.0,
      "logps/chosen": -155.0,
      "logps/rejected": -151.0,
      "loss": 0.6908,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.007537841796875,
      "rewards/margins": 0.0034942626953125,
      "rewards/rejected": 0.0040283203125,
      "step": 210
    },
    {
      "epoch": 0.015881036598570708,
      "grad_norm": 1.8713087698233093,
      "learning_rate": 7.936507936507936e-08,
      "logits/chosen": -2.546875,
      "logits/rejected": -1.8984375,
      "logps/chosen": -156.0,
      "logps/rejected": -168.0,
      "loss": 0.6903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.00860595703125,
      "rewards/margins": 0.00469970703125,
      "rewards/rejected": 0.00390625,
      "step": 220
    },
    {
      "epoch": 0.01660290189850574,
      "grad_norm": 1.7352700913418746,
      "learning_rate": 8.297258297258297e-08,
      "logits/chosen": -2.359375,
      "logits/rejected": -2.0,
      "logps/chosen": -171.0,
      "logps/rejected": -150.0,
      "loss": 0.6897,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.01153564453125,
      "rewards/margins": 0.00640869140625,
      "rewards/rejected": 0.005157470703125,
      "step": 230
    },
    {
      "epoch": 0.017324767198440772,
      "grad_norm": 1.923679325098434,
      "learning_rate": 8.658008658008658e-08,
      "logits/chosen": -2.40625,
      "logits/rejected": -2.171875,
      "logps/chosen": -168.0,
      "logps/rejected": -153.0,
      "loss": 0.6889,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01507568359375,
      "rewards/margins": 0.00933837890625,
      "rewards/rejected": 0.005706787109375,
      "step": 240
    },
    {
      "epoch": 0.018046632498375802,
      "grad_norm": 1.9336109183415302,
      "learning_rate": 9.018759018759018e-08,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.140625,
      "logps/chosen": -198.0,
      "logps/rejected": -190.0,
      "loss": 0.6883,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.015625,
      "rewards/margins": 0.01055908203125,
      "rewards/rejected": 0.005035400390625,
      "step": 250
    },
    {
      "epoch": 0.018768497798310836,
      "grad_norm": 1.7118991587292718,
      "learning_rate": 9.37950937950938e-08,
      "logits/chosen": -2.484375,
      "logits/rejected": -1.9921875,
      "logps/chosen": -148.0,
      "logps/rejected": -164.0,
      "loss": 0.6873,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.012451171875,
      "rewards/rejected": 0.00469970703125,
      "step": 260
    },
    {
      "epoch": 0.019490363098245866,
      "grad_norm": 2.0498889647791634,
      "learning_rate": 9.74025974025974e-08,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.125,
      "logps/chosen": -168.0,
      "logps/rejected": -150.0,
      "loss": 0.6875,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.0159912109375,
      "rewards/margins": 0.00982666015625,
      "rewards/rejected": 0.006195068359375,
      "step": 270
    },
    {
      "epoch": 0.0202122283981809,
      "grad_norm": 1.7948780770940682,
      "learning_rate": 1.01010101010101e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -1.9140625,
      "logps/chosen": -162.0,
      "logps/rejected": -170.0,
      "loss": 0.6865,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.018310546875,
      "rewards/margins": 0.01513671875,
      "rewards/rejected": 0.0032196044921875,
      "step": 280
    },
    {
      "epoch": 0.02093409369811593,
      "grad_norm": 1.7485728753360685,
      "learning_rate": 1.0461760461760462e-07,
      "logits/chosen": -2.390625,
      "logits/rejected": -2.09375,
      "logps/chosen": -173.0,
      "logps/rejected": -152.0,
      "loss": 0.6865,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0172119140625,
      "rewards/margins": 0.0089111328125,
      "rewards/rejected": 0.0084228515625,
      "step": 290
    },
    {
      "epoch": 0.021655958998050964,
      "grad_norm": 1.7752446010354384,
      "learning_rate": 1.0822510822510822e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.15625,
      "logps/chosen": -162.0,
      "logps/rejected": -165.0,
      "loss": 0.6857,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0238037109375,
      "rewards/margins": 0.015625,
      "rewards/rejected": 0.00823974609375,
      "step": 300
    },
    {
      "epoch": 0.022377824297985995,
      "grad_norm": 1.9527079542055057,
      "learning_rate": 1.1183261183261184e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.078125,
      "logps/chosen": -148.0,
      "logps/rejected": -153.0,
      "loss": 0.6845,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.0203857421875,
      "rewards/margins": 0.0184326171875,
      "rewards/rejected": 0.00201416015625,
      "step": 310
    },
    {
      "epoch": 0.02309968959792103,
      "grad_norm": 1.798683396295616,
      "learning_rate": 1.1544011544011543e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.09375,
      "logps/chosen": -163.0,
      "logps/rejected": -141.0,
      "loss": 0.6844,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.02197265625,
      "rewards/margins": 0.0184326171875,
      "rewards/rejected": 0.003570556640625,
      "step": 320
    },
    {
      "epoch": 0.02382155489785606,
      "grad_norm": 1.822086025310402,
      "learning_rate": 1.1904761904761903e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -1.9765625,
      "logps/chosen": -159.0,
      "logps/rejected": -147.0,
      "loss": 0.6816,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.02490234375,
      "rewards/margins": 0.0225830078125,
      "rewards/rejected": 0.002349853515625,
      "step": 330
    },
    {
      "epoch": 0.024543420197791092,
      "grad_norm": 1.8205804931965275,
      "learning_rate": 1.2265512265512265e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.125,
      "logps/chosen": -163.0,
      "logps/rejected": -168.0,
      "loss": 0.6809,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0240478515625,
      "rewards/margins": 0.02392578125,
      "rewards/rejected": 4.839897155761719e-05,
      "step": 340
    },
    {
      "epoch": 0.025265285497726123,
      "grad_norm": 2.015225469187424,
      "learning_rate": 1.2626262626262626e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.1875,
      "logps/chosen": -168.0,
      "logps/rejected": -138.0,
      "loss": 0.6787,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.023193359375,
      "rewards/margins": 0.02880859375,
      "rewards/rejected": -0.005706787109375,
      "step": 350
    },
    {
      "epoch": 0.025987150797661156,
      "grad_norm": 1.8960338628946363,
      "learning_rate": 1.2987012987012984e-07,
      "logits/chosen": -2.34375,
      "logits/rejected": -2.0625,
      "logps/chosen": -171.0,
      "logps/rejected": -155.0,
      "loss": 0.679,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.024169921875,
      "rewards/margins": 0.029052734375,
      "rewards/rejected": -0.004791259765625,
      "step": 360
    },
    {
      "epoch": 0.02670901609759619,
      "grad_norm": 1.9289106441512014,
      "learning_rate": 1.3347763347763348e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.125,
      "logps/chosen": -143.0,
      "logps/rejected": -147.0,
      "loss": 0.6755,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0277099609375,
      "rewards/margins": 0.036376953125,
      "rewards/rejected": -0.00848388671875,
      "step": 370
    },
    {
      "epoch": 0.02743088139753122,
      "grad_norm": 2.4318763159683168,
      "learning_rate": 1.370851370851371e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.078125,
      "logps/chosen": -161.0,
      "logps/rejected": -154.0,
      "loss": 0.6738,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.016845703125,
      "rewards/margins": 0.0390625,
      "rewards/rejected": -0.022216796875,
      "step": 380
    },
    {
      "epoch": 0.028152746697466254,
      "grad_norm": 1.727518768188907,
      "learning_rate": 1.406926406926407e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.125,
      "logps/chosen": -151.0,
      "logps/rejected": -145.0,
      "loss": 0.6737,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.0238037109375,
      "rewards/margins": 0.04541015625,
      "rewards/rejected": -0.0216064453125,
      "step": 390
    },
    {
      "epoch": 0.028874611997401285,
      "grad_norm": 1.8565593414895172,
      "learning_rate": 1.4430014430014428e-07,
      "logits/chosen": -2.453125,
      "logits/rejected": -2.046875,
      "logps/chosen": -169.0,
      "logps/rejected": -162.0,
      "loss": 0.6703,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.01324462890625,
      "rewards/margins": 0.0458984375,
      "rewards/rejected": -0.03271484375,
      "step": 400
    },
    {
      "epoch": 0.02959647729733632,
      "grad_norm": 1.9588571879597823,
      "learning_rate": 1.479076479076479e-07,
      "logits/chosen": -2.46875,
      "logits/rejected": -2.296875,
      "logps/chosen": -152.0,
      "logps/rejected": -137.0,
      "loss": 0.6681,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.006378173828125,
      "rewards/margins": 0.052734375,
      "rewards/rejected": -0.04638671875,
      "step": 410
    },
    {
      "epoch": 0.03031834259727135,
      "grad_norm": 1.93069247030128,
      "learning_rate": 1.5151515151515152e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.203125,
      "logps/chosen": -172.0,
      "logps/rejected": -144.0,
      "loss": 0.6667,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0025787353515625,
      "rewards/margins": 0.044189453125,
      "rewards/rejected": -0.046875,
      "step": 420
    },
    {
      "epoch": 0.031040207897206382,
      "grad_norm": 1.819742407006859,
      "learning_rate": 1.5512265512265513e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.125,
      "logps/chosen": -172.0,
      "logps/rejected": -152.0,
      "loss": 0.6625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.002685546875,
      "rewards/margins": 0.06396484375,
      "rewards/rejected": -0.06640625,
      "step": 430
    },
    {
      "epoch": 0.031762073197141416,
      "grad_norm": 2.3055219231256108,
      "learning_rate": 1.5873015873015872e-07,
      "logits/chosen": -2.4375,
      "logits/rejected": -2.1875,
      "logps/chosen": -155.0,
      "logps/rejected": -158.0,
      "loss": 0.6555,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.01123046875,
      "rewards/margins": 0.078125,
      "rewards/rejected": -0.08935546875,
      "step": 440
    },
    {
      "epoch": 0.032483938497076446,
      "grad_norm": 1.9129712763765747,
      "learning_rate": 1.6233766233766232e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.140625,
      "logps/chosen": -166.0,
      "logps/rejected": -154.0,
      "loss": 0.6547,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.031494140625,
      "rewards/margins": 0.07666015625,
      "rewards/rejected": -0.10791015625,
      "step": 450
    },
    {
      "epoch": 0.03320580379701148,
      "grad_norm": 1.936887902580248,
      "learning_rate": 1.6594516594516593e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.25,
      "logps/chosen": -166.0,
      "logps/rejected": -163.0,
      "loss": 0.65,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0478515625,
      "rewards/margins": 0.08740234375,
      "rewards/rejected": -0.1357421875,
      "step": 460
    },
    {
      "epoch": 0.03392766909694651,
      "grad_norm": 2.1168551608123725,
      "learning_rate": 1.6955266955266957e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.234375,
      "logps/chosen": -187.0,
      "logps/rejected": -190.0,
      "loss": 0.6399,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.06298828125,
      "rewards/margins": 0.11279296875,
      "rewards/rejected": -0.17578125,
      "step": 470
    },
    {
      "epoch": 0.034649534396881544,
      "grad_norm": 2.4110075564522533,
      "learning_rate": 1.7316017316017315e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.296875,
      "logps/chosen": -171.0,
      "logps/rejected": -167.0,
      "loss": 0.6345,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.09375,
      "rewards/margins": 0.1376953125,
      "rewards/rejected": -0.2314453125,
      "step": 480
    },
    {
      "epoch": 0.035371399696816574,
      "grad_norm": 2.561479324237141,
      "learning_rate": 1.7676767676767676e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -165.0,
      "logps/rejected": -169.0,
      "loss": 0.6272,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.1552734375,
      "rewards/margins": 0.1328125,
      "rewards/rejected": -0.287109375,
      "step": 490
    },
    {
      "epoch": 0.036093264996751605,
      "grad_norm": 2.2009284942320004,
      "learning_rate": 1.8037518037518037e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.1875,
      "logps/chosen": -180.0,
      "logps/rejected": -198.0,
      "loss": 0.6213,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.1826171875,
      "rewards/margins": 0.1806640625,
      "rewards/rejected": -0.36328125,
      "step": 500
    },
    {
      "epoch": 0.036815130296686635,
      "grad_norm": 2.433624086599741,
      "learning_rate": 1.8398268398268395e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.203125,
      "logps/chosen": -182.0,
      "logps/rejected": -181.0,
      "loss": 0.6137,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2294921875,
      "rewards/margins": 0.1787109375,
      "rewards/rejected": -0.408203125,
      "step": 510
    },
    {
      "epoch": 0.03753699559662167,
      "grad_norm": 2.3303081056576396,
      "learning_rate": 1.875901875901876e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.3125,
      "logps/chosen": -183.0,
      "logps/rejected": -198.0,
      "loss": 0.6089,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.28125,
      "rewards/margins": 0.1953125,
      "rewards/rejected": -0.4765625,
      "step": 520
    },
    {
      "epoch": 0.0382588608965567,
      "grad_norm": 7.529255626517495,
      "learning_rate": 1.911976911976912e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.203125,
      "logps/chosen": -188.0,
      "logps/rejected": -206.0,
      "loss": 0.6063,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3046875,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.5390625,
      "step": 530
    },
    {
      "epoch": 0.03898072619649173,
      "grad_norm": 2.2072494405878578,
      "learning_rate": 1.948051948051948e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.3125,
      "logps/chosen": -203.0,
      "logps/rejected": -224.0,
      "loss": 0.5956,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3515625,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -0.6328125,
      "step": 540
    },
    {
      "epoch": 0.03970259149642677,
      "grad_norm": 2.310476086986689,
      "learning_rate": 1.984126984126984e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.484375,
      "logps/chosen": -220.0,
      "logps/rejected": -209.0,
      "loss": 0.5867,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.416015625,
      "rewards/margins": 0.236328125,
      "rewards/rejected": -0.65234375,
      "step": 550
    },
    {
      "epoch": 0.0404244567963618,
      "grad_norm": 2.6001658124806286,
      "learning_rate": 2.02020202020202e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.359375,
      "logps/chosen": -208.0,
      "logps/rejected": -239.0,
      "loss": 0.5754,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.447265625,
      "rewards/margins": 0.31640625,
      "rewards/rejected": -0.76171875,
      "step": 560
    },
    {
      "epoch": 0.04114632209629683,
      "grad_norm": 2.635075269431316,
      "learning_rate": 2.0562770562770563e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -217.0,
      "logps/rejected": -227.0,
      "loss": 0.5719,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.494140625,
      "rewards/margins": 0.33984375,
      "rewards/rejected": -0.83203125,
      "step": 570
    },
    {
      "epoch": 0.04186818739623186,
      "grad_norm": 2.5380650062224066,
      "learning_rate": 2.0923520923520924e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.453125,
      "logps/chosen": -216.0,
      "logps/rejected": -237.0,
      "loss": 0.5794,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 0.349609375,
      "rewards/rejected": -0.8515625,
      "step": 580
    },
    {
      "epoch": 0.0425900526961669,
      "grad_norm": 2.808866971835476,
      "learning_rate": 2.1284271284271282e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -216.0,
      "logps/rejected": -264.0,
      "loss": 0.5557,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.5390625,
      "rewards/margins": 0.50390625,
      "rewards/rejected": -1.046875,
      "step": 590
    },
    {
      "epoch": 0.04331191799610193,
      "grad_norm": 3.660790591856929,
      "learning_rate": 2.1645021645021643e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.734375,
      "logps/chosen": -215.0,
      "logps/rejected": -226.0,
      "loss": 0.5374,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.60546875,
      "rewards/margins": 0.3203125,
      "rewards/rejected": -0.92578125,
      "step": 600
    },
    {
      "epoch": 0.04403378329603696,
      "grad_norm": 3.4012692341173842,
      "learning_rate": 2.2005772005772004e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.671875,
      "logps/chosen": -243.0,
      "logps/rejected": -288.0,
      "loss": 0.5302,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.75,
      "rewards/margins": 0.51953125,
      "rewards/rejected": -1.2734375,
      "step": 610
    },
    {
      "epoch": 0.04475564859597199,
      "grad_norm": 3.1213723367337276,
      "learning_rate": 2.2366522366522368e-07,
      "logits/chosen": -3.265625,
      "logits/rejected": -2.796875,
      "logps/chosen": -252.0,
      "logps/rejected": -290.0,
      "loss": 0.5349,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.82421875,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -1.4375,
      "step": 620
    },
    {
      "epoch": 0.045477513895907026,
      "grad_norm": 3.4139219834989825,
      "learning_rate": 2.2727272727272726e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.8125,
      "logps/chosen": -241.0,
      "logps/rejected": -290.0,
      "loss": 0.5313,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.76953125,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.390625,
      "step": 630
    },
    {
      "epoch": 0.04619937919584206,
      "grad_norm": 3.545601457816913,
      "learning_rate": 2.3088023088023087e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.796875,
      "logps/chosen": -254.0,
      "logps/rejected": -306.0,
      "loss": 0.5163,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.796875,
      "rewards/margins": 0.671875,
      "rewards/rejected": -1.46875,
      "step": 640
    },
    {
      "epoch": 0.04692124449577709,
      "grad_norm": 6.320535114176617,
      "learning_rate": 2.3448773448773448e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.9375,
      "logps/chosen": -258.0,
      "logps/rejected": -312.0,
      "loss": 0.5168,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.625,
      "rewards/rejected": -1.5546875,
      "step": 650
    },
    {
      "epoch": 0.04764310979571212,
      "grad_norm": 4.40606882426551,
      "learning_rate": 2.3809523809523806e-07,
      "logits/chosen": -3.234375,
      "logits/rejected": -2.984375,
      "logps/chosen": -258.0,
      "logps/rejected": -300.0,
      "loss": 0.506,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9375,
      "rewards/margins": 0.6171875,
      "rewards/rejected": -1.5546875,
      "step": 660
    },
    {
      "epoch": 0.048364975095647154,
      "grad_norm": 7.228420353041346,
      "learning_rate": 2.4170274170274167e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.8125,
      "logps/chosen": -230.0,
      "logps/rejected": -308.0,
      "loss": 0.5231,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8359375,
      "rewards/margins": 0.7421875,
      "rewards/rejected": -1.578125,
      "step": 670
    },
    {
      "epoch": 0.049086840395582185,
      "grad_norm": 4.810449590822843,
      "learning_rate": 2.453102453102453e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.75,
      "logps/chosen": -246.0,
      "logps/rejected": -280.0,
      "loss": 0.5107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.88671875,
      "rewards/margins": 0.498046875,
      "rewards/rejected": -1.3828125,
      "step": 680
    },
    {
      "epoch": 0.049808705695517215,
      "grad_norm": 6.916956622472092,
      "learning_rate": 2.4891774891774894e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.9375,
      "logps/chosen": -270.0,
      "logps/rejected": -322.0,
      "loss": 0.5105,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.9765625,
      "rewards/margins": 0.6484375,
      "rewards/rejected": -1.625,
      "step": 690
    },
    {
      "epoch": 0.050530570995452245,
      "grad_norm": 5.072218871042774,
      "learning_rate": 2.525252525252525e-07,
      "logits/chosen": -3.109375,
      "logits/rejected": -2.84375,
      "logps/chosen": -256.0,
      "logps/rejected": -326.0,
      "loss": 0.5065,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.92578125,
      "rewards/margins": 0.76953125,
      "rewards/rejected": -1.6953125,
      "step": 700
    },
    {
      "epoch": 0.05125243629538728,
      "grad_norm": 8.875662849487487,
      "learning_rate": 2.5613275613275616e-07,
      "logits/chosen": -3.09375,
      "logits/rejected": -2.84375,
      "logps/chosen": -254.0,
      "logps/rejected": -306.0,
      "loss": 0.5112,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.94921875,
      "rewards/margins": 0.61328125,
      "rewards/rejected": -1.5625,
      "step": 710
    },
    {
      "epoch": 0.05197430159532231,
      "grad_norm": 5.595849818392523,
      "learning_rate": 2.597402597402597e-07,
      "logits/chosen": -3.15625,
      "logits/rejected": -2.96875,
      "logps/chosen": -264.0,
      "logps/rejected": -340.0,
      "loss": 0.5107,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.0078125,
      "rewards/margins": 0.8125,
      "rewards/rejected": -1.8203125,
      "step": 720
    },
    {
      "epoch": 0.05269616689525734,
      "grad_norm": 5.404147285645744,
      "learning_rate": 2.633477633477633e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.65625,
      "logps/chosen": -268.0,
      "logps/rejected": -368.0,
      "loss": 0.5035,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.984375,
      "rewards/margins": 0.94140625,
      "rewards/rejected": -1.9296875,
      "step": 730
    },
    {
      "epoch": 0.05341803219519238,
      "grad_norm": 8.76041109611346,
      "learning_rate": 2.6695526695526696e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.828125,
      "logps/chosen": -286.0,
      "logps/rejected": -342.0,
      "loss": 0.4887,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.75390625,
      "rewards/rejected": -1.8828125,
      "step": 740
    },
    {
      "epoch": 0.05413989749512741,
      "grad_norm": 7.025589443972428,
      "learning_rate": 2.7056277056277054e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.78125,
      "logps/chosen": -262.0,
      "logps/rejected": -348.0,
      "loss": 0.4823,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0,
      "rewards/margins": 0.94921875,
      "rewards/rejected": -1.953125,
      "step": 750
    },
    {
      "epoch": 0.05486176279506244,
      "grad_norm": 11.975302977639574,
      "learning_rate": 2.741702741702742e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.75,
      "logps/chosen": -276.0,
      "logps/rejected": -322.0,
      "loss": 0.4766,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.140625,
      "rewards/margins": 0.6953125,
      "rewards/rejected": -1.8359375,
      "step": 760
    },
    {
      "epoch": 0.05558362809499747,
      "grad_norm": 6.5118457359444,
      "learning_rate": 2.7777777777777776e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.734375,
      "logps/chosen": -280.0,
      "logps/rejected": -346.0,
      "loss": 0.4846,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1875,
      "rewards/margins": 0.80859375,
      "rewards/rejected": -1.9921875,
      "step": 770
    },
    {
      "epoch": 0.05630549339493251,
      "grad_norm": 6.106781577833728,
      "learning_rate": 2.813852813852814e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.6875,
      "logps/chosen": -260.0,
      "logps/rejected": -356.0,
      "loss": 0.4665,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.046875,
      "rewards/margins": 0.83203125,
      "rewards/rejected": -1.875,
      "step": 780
    },
    {
      "epoch": 0.05702735869486754,
      "grad_norm": 10.75495308386994,
      "learning_rate": 2.8499278499278503e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.625,
      "logps/chosen": -284.0,
      "logps/rejected": -408.0,
      "loss": 0.4695,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.2109375,
      "rewards/margins": 1.171875,
      "rewards/rejected": -2.390625,
      "step": 790
    },
    {
      "epoch": 0.05774922399480257,
      "grad_norm": 5.788421207967755,
      "learning_rate": 2.8860028860028856e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -274.0,
      "logps/rejected": -334.0,
      "loss": 0.4767,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.125,
      "rewards/margins": 0.84765625,
      "rewards/rejected": -1.9765625,
      "step": 800
    },
    {
      "epoch": 0.0584710892947376,
      "grad_norm": 8.78614857490401,
      "learning_rate": 2.922077922077922e-07,
      "logits/chosen": -3.015625,
      "logits/rejected": -2.71875,
      "logps/chosen": -262.0,
      "logps/rejected": -362.0,
      "loss": 0.4722,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.8984375,
      "rewards/rejected": -2.125,
      "step": 810
    },
    {
      "epoch": 0.05919295459467264,
      "grad_norm": 7.3161771388355294,
      "learning_rate": 2.958152958152958e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -272.0,
      "logps/rejected": -376.0,
      "loss": 0.4996,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.1796875,
      "rewards/margins": 0.93359375,
      "rewards/rejected": -2.109375,
      "step": 820
    },
    {
      "epoch": 0.05991481989460767,
      "grad_norm": 8.266553993044619,
      "learning_rate": 2.994227994227994e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -282.0,
      "logps/rejected": -376.0,
      "loss": 0.4545,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.15625,
      "rewards/margins": 1.015625,
      "rewards/rejected": -2.171875,
      "step": 830
    },
    {
      "epoch": 0.0606366851945427,
      "grad_norm": 8.218438777679667,
      "learning_rate": 3.0303030303030305e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.640625,
      "logps/chosen": -272.0,
      "logps/rejected": -382.0,
      "loss": 0.4639,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.25,
      "rewards/margins": 1.0234375,
      "rewards/rejected": -2.28125,
      "step": 840
    },
    {
      "epoch": 0.06135855049447773,
      "grad_norm": 8.61378341353772,
      "learning_rate": 3.0663780663780663e-07,
      "logits/chosen": -3.078125,
      "logits/rejected": -2.640625,
      "logps/chosen": -284.0,
      "logps/rejected": -408.0,
      "loss": 0.4536,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.109375,
      "rewards/rejected": -2.359375,
      "step": 850
    },
    {
      "epoch": 0.062080415794412765,
      "grad_norm": 12.398406848670698,
      "learning_rate": 3.1024531024531027e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -286.0,
      "logps/rejected": -372.0,
      "loss": 0.476,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2578125,
      "rewards/margins": 1.0078125,
      "rewards/rejected": -2.265625,
      "step": 860
    },
    {
      "epoch": 0.0628022810943478,
      "grad_norm": 12.244121744087602,
      "learning_rate": 3.138528138528138e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -294.0,
      "logps/rejected": -388.0,
      "loss": 0.4623,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3671875,
      "rewards/margins": 0.984375,
      "rewards/rejected": -2.359375,
      "step": 870
    },
    {
      "epoch": 0.06352414639428283,
      "grad_norm": 10.812869810884447,
      "learning_rate": 3.1746031746031743e-07,
      "logits/chosen": -3.03125,
      "logits/rejected": -2.796875,
      "logps/chosen": -268.0,
      "logps/rejected": -366.0,
      "loss": 0.4498,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.234375,
      "rewards/margins": 0.99609375,
      "rewards/rejected": -2.234375,
      "step": 880
    },
    {
      "epoch": 0.06424601169421786,
      "grad_norm": 11.229212598361977,
      "learning_rate": 3.2106782106782107e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.75,
      "logps/chosen": -298.0,
      "logps/rejected": -394.0,
      "loss": 0.4373,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3828125,
      "rewards/margins": 1.0859375,
      "rewards/rejected": -2.46875,
      "step": 890
    },
    {
      "epoch": 0.06496787699415289,
      "grad_norm": 7.414765021326351,
      "learning_rate": 3.2467532467532465e-07,
      "logits/chosen": -2.984375,
      "logits/rejected": -2.75,
      "logps/chosen": -294.0,
      "logps/rejected": -394.0,
      "loss": 0.4253,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.34375,
      "rewards/margins": 1.140625,
      "rewards/rejected": -2.484375,
      "step": 900
    },
    {
      "epoch": 0.06568974229408793,
      "grad_norm": 6.656174306105041,
      "learning_rate": 3.282828282828283e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.65625,
      "logps/chosen": -322.0,
      "logps/rejected": -440.0,
      "loss": 0.4366,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.1484375,
      "rewards/rejected": -2.78125,
      "step": 910
    },
    {
      "epoch": 0.06641160759402295,
      "grad_norm": 6.986955189515223,
      "learning_rate": 3.3189033189033187e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -328.0,
      "logps/rejected": -422.0,
      "loss": 0.4335,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.515625,
      "rewards/margins": 1.234375,
      "rewards/rejected": -2.75,
      "step": 920
    },
    {
      "epoch": 0.06713347289395799,
      "grad_norm": 8.704691417598838,
      "learning_rate": 3.354978354978355e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.6875,
      "logps/chosen": -322.0,
      "logps/rejected": -406.0,
      "loss": 0.4048,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.0703125,
      "rewards/rejected": -2.640625,
      "step": 930
    },
    {
      "epoch": 0.06785533819389301,
      "grad_norm": 19.52535270358171,
      "learning_rate": 3.3910533910533914e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -316.0,
      "logps/rejected": -476.0,
      "loss": 0.434,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.6015625,
      "rewards/margins": 1.609375,
      "rewards/rejected": -3.21875,
      "step": 940
    },
    {
      "epoch": 0.06857720349382805,
      "grad_norm": 11.339737953685198,
      "learning_rate": 3.4271284271284267e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.375,
      "logps/chosen": -316.0,
      "logps/rejected": -398.0,
      "loss": 0.4334,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.03125,
      "rewards/rejected": -2.59375,
      "step": 950
    },
    {
      "epoch": 0.06929906879376309,
      "grad_norm": 10.31037341226185,
      "learning_rate": 3.463203463203463e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.4375,
      "logps/chosen": -322.0,
      "logps/rejected": -456.0,
      "loss": 0.4161,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.6328125,
      "rewards/margins": 1.40625,
      "rewards/rejected": -3.046875,
      "step": 960
    },
    {
      "epoch": 0.07002093409369811,
      "grad_norm": 11.130584733664083,
      "learning_rate": 3.499278499278499e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -298.0,
      "logps/rejected": -424.0,
      "loss": 0.4078,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4296875,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -2.796875,
      "step": 970
    },
    {
      "epoch": 0.07074279939363315,
      "grad_norm": 9.06083640139615,
      "learning_rate": 3.535353535353535e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -308.0,
      "logps/rejected": -442.0,
      "loss": 0.4178,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -2.890625,
      "step": 980
    },
    {
      "epoch": 0.07146466469356819,
      "grad_norm": 9.541422651232944,
      "learning_rate": 3.5714285714285716e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -338.0,
      "logps/rejected": -450.0,
      "loss": 0.3953,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.375,
      "rewards/rejected": -3.078125,
      "step": 990
    },
    {
      "epoch": 0.07218652999350321,
      "grad_norm": 9.110551563711448,
      "learning_rate": 3.6075036075036074e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -326.0,
      "logps/rejected": -464.0,
      "loss": 0.4021,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.53125,
      "rewards/margins": 1.453125,
      "rewards/rejected": -2.984375,
      "step": 1000
    },
    {
      "epoch": 0.07290839529343825,
      "grad_norm": 15.071837561832831,
      "learning_rate": 3.643578643578644e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -314.0,
      "logps/rejected": -452.0,
      "loss": 0.4045,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.5625,
      "rewards/margins": 1.3671875,
      "rewards/rejected": -2.921875,
      "step": 1010
    },
    {
      "epoch": 0.07363026059337327,
      "grad_norm": 13.920977378141336,
      "learning_rate": 3.679653679653679e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -334.0,
      "logps/rejected": -492.0,
      "loss": 0.431,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8125,
      "rewards/margins": 1.46875,
      "rewards/rejected": -3.28125,
      "step": 1020
    },
    {
      "epoch": 0.07435212589330831,
      "grad_norm": 11.085289038392293,
      "learning_rate": 3.7157287157287154e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -298.0,
      "logps/rejected": -456.0,
      "loss": 0.3934,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5703125,
      "rewards/margins": 1.390625,
      "rewards/rejected": -2.96875,
      "step": 1030
    },
    {
      "epoch": 0.07507399119324334,
      "grad_norm": 10.088971897286424,
      "learning_rate": 3.751803751803752e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -322.0,
      "logps/rejected": -456.0,
      "loss": 0.4073,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.7109375,
      "rewards/margins": 1.3515625,
      "rewards/rejected": -3.0625,
      "step": 1040
    },
    {
      "epoch": 0.07579585649317837,
      "grad_norm": 43.24163673888106,
      "learning_rate": 3.7878787878787876e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.375,
      "logps/chosen": -348.0,
      "logps/rejected": -496.0,
      "loss": 0.4152,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.890625,
      "rewards/margins": 1.25,
      "rewards/rejected": -3.140625,
      "step": 1050
    },
    {
      "epoch": 0.0765177217931134,
      "grad_norm": 8.537892452168746,
      "learning_rate": 3.823953823953824e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -338.0,
      "logps/rejected": -480.0,
      "loss": 0.3947,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.765625,
      "rewards/margins": 1.5234375,
      "rewards/rejected": -3.296875,
      "step": 1060
    },
    {
      "epoch": 0.07723958709304844,
      "grad_norm": 9.485771716655114,
      "learning_rate": 3.86002886002886e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.375,
      "logps/chosen": -326.0,
      "logps/rejected": -462.0,
      "loss": 0.3837,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.640625,
      "rewards/margins": 1.453125,
      "rewards/rejected": -3.09375,
      "step": 1070
    },
    {
      "epoch": 0.07796145239298347,
      "grad_norm": 9.46365785011617,
      "learning_rate": 3.896103896103896e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.328125,
      "logps/chosen": -386.0,
      "logps/rejected": -536.0,
      "loss": 0.3848,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.15625,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -3.9375,
      "step": 1080
    },
    {
      "epoch": 0.0786833176929185,
      "grad_norm": 13.461346342787172,
      "learning_rate": 3.9321789321789325e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -360.0,
      "logps/rejected": -520.0,
      "loss": 0.3922,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -3.765625,
      "step": 1090
    },
    {
      "epoch": 0.07940518299285354,
      "grad_norm": 12.44634151581957,
      "learning_rate": 3.968253968253968e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -356.0,
      "logps/rejected": -508.0,
      "loss": 0.3727,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.015625,
      "rewards/margins": 1.5,
      "rewards/rejected": -3.515625,
      "step": 1100
    },
    {
      "epoch": 0.08012704829278856,
      "grad_norm": 11.687822831294243,
      "learning_rate": 4.004329004329004e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.4375,
      "logps/chosen": -346.0,
      "logps/rejected": -568.0,
      "loss": 0.3765,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.0625,
      "step": 1110
    },
    {
      "epoch": 0.0808489135927236,
      "grad_norm": 16.193115895860167,
      "learning_rate": 4.04040404040404e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -356.0,
      "logps/rejected": -532.0,
      "loss": 0.3726,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.9921875,
      "rewards/margins": 1.6484375,
      "rewards/rejected": -3.65625,
      "step": 1120
    },
    {
      "epoch": 0.08157077889265862,
      "grad_norm": 10.454280347519187,
      "learning_rate": 4.0764790764790763e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -350.0,
      "logps/rejected": -496.0,
      "loss": 0.4066,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.78125,
      "rewards/margins": 1.515625,
      "rewards/rejected": -3.296875,
      "step": 1130
    },
    {
      "epoch": 0.08229264419259366,
      "grad_norm": 10.033847939954763,
      "learning_rate": 4.1125541125541127e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -334.0,
      "logps/rejected": -462.0,
      "loss": 0.355,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.7890625,
      "rewards/margins": 1.3984375,
      "rewards/rejected": -3.1875,
      "step": 1140
    },
    {
      "epoch": 0.0830145094925287,
      "grad_norm": 13.538065077526934,
      "learning_rate": 4.1486291486291485e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -398.0,
      "logps/rejected": -588.0,
      "loss": 0.3843,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.1875,
      "rewards/margins": 2.078125,
      "rewards/rejected": -4.28125,
      "step": 1150
    },
    {
      "epoch": 0.08373637479246372,
      "grad_norm": 12.06165274825125,
      "learning_rate": 4.184704184704185e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -314.0,
      "logps/rejected": -500.0,
      "loss": 0.3816,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.6796875,
      "rewards/margins": 1.7578125,
      "rewards/rejected": -3.4375,
      "step": 1160
    },
    {
      "epoch": 0.08445824009239876,
      "grad_norm": 14.245004620424362,
      "learning_rate": 4.22077922077922e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.640625,
      "logps/chosen": -376.0,
      "logps/rejected": -612.0,
      "loss": 0.3599,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.21875,
      "rewards/margins": 2.375,
      "rewards/rejected": -4.59375,
      "step": 1170
    },
    {
      "epoch": 0.0851801053923338,
      "grad_norm": 12.93467746542696,
      "learning_rate": 4.2568542568542565e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -330.0,
      "logps/rejected": -476.0,
      "loss": 0.3635,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.75,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -3.34375,
      "step": 1180
    },
    {
      "epoch": 0.08590197069226882,
      "grad_norm": 11.202101570401776,
      "learning_rate": 4.292929292929293e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.546875,
      "logps/chosen": -364.0,
      "logps/rejected": -536.0,
      "loss": 0.3583,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.125,
      "rewards/margins": 1.8359375,
      "rewards/rejected": -3.953125,
      "step": 1190
    },
    {
      "epoch": 0.08662383599220386,
      "grad_norm": 10.771994466129188,
      "learning_rate": 4.3290043290043287e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.5,
      "logps/chosen": -358.0,
      "logps/rejected": -510.0,
      "loss": 0.3465,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.8828125,
      "rewards/margins": 1.578125,
      "rewards/rejected": -3.46875,
      "step": 1200
    },
    {
      "epoch": 0.08734570129213888,
      "grad_norm": 14.841268368598595,
      "learning_rate": 4.365079365079365e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.578125,
      "logps/chosen": -364.0,
      "logps/rejected": -584.0,
      "loss": 0.3477,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.234375,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.40625,
      "step": 1210
    },
    {
      "epoch": 0.08806756659207392,
      "grad_norm": 9.504108427187644,
      "learning_rate": 4.401154401154401e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.546875,
      "logps/chosen": -394.0,
      "logps/rejected": -648.0,
      "loss": 0.3755,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.25,
      "rewards/margins": 2.703125,
      "rewards/rejected": -4.9375,
      "step": 1220
    },
    {
      "epoch": 0.08878943189200895,
      "grad_norm": 12.115514229718103,
      "learning_rate": 4.437229437229437e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -304.0,
      "logps/rejected": -520.0,
      "loss": 0.3704,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.5390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -3.71875,
      "step": 1230
    },
    {
      "epoch": 0.08951129719194398,
      "grad_norm": 10.317228267259349,
      "learning_rate": 4.4733044733044736e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -420.0,
      "logps/rejected": -612.0,
      "loss": 0.3559,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.5,
      "rewards/margins": 1.9453125,
      "rewards/rejected": -4.46875,
      "step": 1240
    },
    {
      "epoch": 0.09023316249187902,
      "grad_norm": 11.197522859178363,
      "learning_rate": 4.509379509379509e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.546875,
      "logps/chosen": -428.0,
      "logps/rejected": -604.0,
      "loss": 0.3219,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.859375,
      "rewards/rejected": -4.59375,
      "step": 1250
    },
    {
      "epoch": 0.09095502779181405,
      "grad_norm": 13.497331724381818,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -412.0,
      "logps/rejected": -604.0,
      "loss": 0.3388,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.546875,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -4.53125,
      "step": 1260
    },
    {
      "epoch": 0.09167689309174908,
      "grad_norm": 11.433124284340494,
      "learning_rate": 4.581529581529581e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -390.0,
      "logps/rejected": -560.0,
      "loss": 0.3458,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.234375,
      "rewards/margins": 1.7734375,
      "rewards/rejected": -4.0,
      "step": 1270
    },
    {
      "epoch": 0.09239875839168411,
      "grad_norm": 12.324384516756012,
      "learning_rate": 4.6176046176046174e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -410.0,
      "logps/rejected": -580.0,
      "loss": 0.3426,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.890625,
      "rewards/rejected": -4.21875,
      "step": 1280
    },
    {
      "epoch": 0.09312062369161915,
      "grad_norm": 10.730987022263216,
      "learning_rate": 4.6536796536796537e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -442.0,
      "logps/rejected": -612.0,
      "loss": 0.3593,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.828125,
      "rewards/margins": 1.7890625,
      "rewards/rejected": -4.625,
      "step": 1290
    },
    {
      "epoch": 0.09384248899155417,
      "grad_norm": 10.132938191746307,
      "learning_rate": 4.6897546897546896e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -408.0,
      "logps/rejected": -564.0,
      "loss": 0.3632,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.359375,
      "rewards/margins": 1.75,
      "rewards/rejected": -4.09375,
      "step": 1300
    },
    {
      "epoch": 0.09456435429148921,
      "grad_norm": 11.988621181927288,
      "learning_rate": 4.725829725829726e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -452.0,
      "logps/rejected": -624.0,
      "loss": 0.3609,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.78125,
      "rewards/rejected": -4.71875,
      "step": 1310
    },
    {
      "epoch": 0.09528621959142423,
      "grad_norm": 8.802084815882347,
      "learning_rate": 4.761904761904761e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -458.0,
      "logps/rejected": -604.0,
      "loss": 0.3197,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9375,
      "rewards/margins": 1.5859375,
      "rewards/rejected": -4.53125,
      "step": 1320
    },
    {
      "epoch": 0.09600808489135927,
      "grad_norm": 13.978035740824382,
      "learning_rate": 4.797979797979798e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -420.0,
      "logps/rejected": -660.0,
      "loss": 0.355,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.703125,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.15625,
      "step": 1330
    },
    {
      "epoch": 0.09672995019129431,
      "grad_norm": 9.329953252240916,
      "learning_rate": 4.834054834054833e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.53125,
      "logps/chosen": -426.0,
      "logps/rejected": -612.0,
      "loss": 0.3382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.703125,
      "rewards/margins": 1.8203125,
      "rewards/rejected": -4.53125,
      "step": 1340
    },
    {
      "epoch": 0.09745181549122933,
      "grad_norm": 14.354032725201476,
      "learning_rate": 4.87012987012987e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -438.0,
      "logps/rejected": -628.0,
      "loss": 0.3375,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.53125,
      "rewards/margins": 2.15625,
      "rewards/rejected": -4.6875,
      "step": 1350
    },
    {
      "epoch": 0.09817368079116437,
      "grad_norm": 11.184165035806638,
      "learning_rate": 4.906204906204906e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -444.0,
      "logps/rejected": -640.0,
      "loss": 0.3528,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 2.015625,
      "rewards/rejected": -4.6875,
      "step": 1360
    },
    {
      "epoch": 0.0988955460910994,
      "grad_norm": 9.32730305340813,
      "learning_rate": 4.942279942279942e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.515625,
      "logps/chosen": -420.0,
      "logps/rejected": -592.0,
      "loss": 0.3343,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.609375,
      "rewards/margins": 1.9375,
      "rewards/rejected": -4.53125,
      "step": 1370
    },
    {
      "epoch": 0.09961741139103443,
      "grad_norm": 17.291834755334516,
      "learning_rate": 4.978354978354979e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.671875,
      "logps/chosen": -450.0,
      "logps/rejected": -636.0,
      "loss": 0.3246,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.9765625,
      "rewards/rejected": -4.6875,
      "step": 1380
    },
    {
      "epoch": 0.10033927669096947,
      "grad_norm": 10.743034154219572,
      "learning_rate": 4.999998729993963e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -430.0,
      "logps/rejected": -604.0,
      "loss": 0.3335,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -4.59375,
      "step": 1390
    },
    {
      "epoch": 0.10106114199090449,
      "grad_norm": 17.190340388139088,
      "learning_rate": 4.999984442440868e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.703125,
      "logps/chosen": -438.0,
      "logps/rejected": -680.0,
      "loss": 0.3204,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.984375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.4375,
      "step": 1400
    },
    {
      "epoch": 0.10178300729083953,
      "grad_norm": 9.641498367561857,
      "learning_rate": 4.99995427991816e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -442.0,
      "logps/rejected": -648.0,
      "loss": 0.3379,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.71875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.875,
      "step": 1410
    },
    {
      "epoch": 0.10250487259077457,
      "grad_norm": 9.768069678095703,
      "learning_rate": 4.999908242617371e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -452.0,
      "logps/rejected": -648.0,
      "loss": 0.34,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.765625,
      "rewards/margins": 1.84375,
      "rewards/rejected": -4.59375,
      "step": 1420
    },
    {
      "epoch": 0.10322673789070959,
      "grad_norm": 13.37422488078675,
      "learning_rate": 4.99984633083084e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -416.0,
      "logps/rejected": -616.0,
      "loss": 0.331,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.6875,
      "rewards/margins": 1.8984375,
      "rewards/rejected": -4.5625,
      "step": 1430
    },
    {
      "epoch": 0.10394860319064463,
      "grad_norm": 11.253293027609532,
      "learning_rate": 4.99976854495171e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -488.0,
      "logps/rejected": -696.0,
      "loss": 0.3218,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.453125,
      "rewards/margins": 2.09375,
      "rewards/rejected": -5.53125,
      "step": 1440
    },
    {
      "epoch": 0.10467046849057966,
      "grad_norm": 10.82613372416563,
      "learning_rate": 4.999674885473922e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.484375,
      "logps/chosen": -458.0,
      "logps/rejected": -656.0,
      "loss": 0.3213,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.921875,
      "rewards/margins": 1.734375,
      "rewards/rejected": -4.65625,
      "step": 1450
    },
    {
      "epoch": 0.10539233379051469,
      "grad_norm": 15.794446739298891,
      "learning_rate": 4.999565352992216e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -502.0,
      "logps/rejected": -712.0,
      "loss": 0.3289,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.296875,
      "rewards/rejected": -5.5,
      "step": 1460
    },
    {
      "epoch": 0.10611419909044972,
      "grad_norm": 10.361664841175285,
      "learning_rate": 4.999439948202127e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -450.0,
      "logps/rejected": -596.0,
      "loss": 0.3261,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.859375,
      "rewards/margins": 1.8515625,
      "rewards/rejected": -4.71875,
      "step": 1470
    },
    {
      "epoch": 0.10683606439038476,
      "grad_norm": 10.555951221100015,
      "learning_rate": 4.999298671899977e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -644.0,
      "loss": 0.3399,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.75,
      "rewards/margins": 2.171875,
      "rewards/rejected": -4.9375,
      "step": 1480
    },
    {
      "epoch": 0.10755792969031978,
      "grad_norm": 9.802204192641314,
      "learning_rate": 4.999141524982877e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -444.0,
      "logps/rejected": -700.0,
      "loss": 0.309,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.53125,
      "step": 1490
    },
    {
      "epoch": 0.10827979499025482,
      "grad_norm": 12.72303480502336,
      "learning_rate": 4.998968508448714e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -432.0,
      "logps/rejected": -632.0,
      "loss": 0.3276,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.15625,
      "rewards/rejected": -4.78125,
      "step": 1500
    },
    {
      "epoch": 0.10900166029018984,
      "grad_norm": 14.570251314538908,
      "learning_rate": 4.998779623396146e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -502.0,
      "logps/rejected": -720.0,
      "loss": 0.2995,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -3.140625,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.65625,
      "step": 1510
    },
    {
      "epoch": 0.10972352559012488,
      "grad_norm": 11.169758259343533,
      "learning_rate": 4.9985748710246e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -476.0,
      "logps/rejected": -656.0,
      "loss": 0.3344,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 1.84375,
      "rewards/rejected": -5.0625,
      "step": 1520
    },
    {
      "epoch": 0.11044539089005992,
      "grad_norm": 10.504554948014457,
      "learning_rate": 4.998354252634257e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.46875,
      "logps/chosen": -486.0,
      "logps/rejected": -668.0,
      "loss": 0.2853,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.078125,
      "rewards/rejected": -5.28125,
      "step": 1530
    },
    {
      "epoch": 0.11116725618999494,
      "grad_norm": 15.073212833972551,
      "learning_rate": 4.998117769626051e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.703125,
      "logps/chosen": -456.0,
      "logps/rejected": -676.0,
      "loss": 0.309,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -2.984375,
      "rewards/margins": 2.25,
      "rewards/rejected": -5.21875,
      "step": 1540
    },
    {
      "epoch": 0.11188912148992998,
      "grad_norm": 10.298569511142073,
      "learning_rate": 4.997865423501657e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -380.0,
      "logps/rejected": -564.0,
      "loss": 0.3261,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.328125,
      "rewards/margins": 1.8828125,
      "rewards/rejected": -4.21875,
      "step": 1550
    },
    {
      "epoch": 0.11261098678986502,
      "grad_norm": 11.022262328486809,
      "learning_rate": 4.997597215863477e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.546875,
      "logps/chosen": -480.0,
      "logps/rejected": -716.0,
      "loss": 0.2878,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.09375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -5.6875,
      "step": 1560
    },
    {
      "epoch": 0.11333285208980004,
      "grad_norm": 12.425423187064853,
      "learning_rate": 4.99731314841464e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -500.0,
      "logps/rejected": -704.0,
      "loss": 0.3008,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.390625,
      "rewards/margins": 2.1875,
      "rewards/rejected": -5.5625,
      "step": 1570
    },
    {
      "epoch": 0.11405471738973508,
      "grad_norm": 10.996930809431353,
      "learning_rate": 4.997013222958978e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.546875,
      "logps/chosen": -496.0,
      "logps/rejected": -732.0,
      "loss": 0.3105,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 2.578125,
      "rewards/rejected": -5.8125,
      "step": 1580
    },
    {
      "epoch": 0.1147765826896701,
      "grad_norm": 11.11449736666811,
      "learning_rate": 4.99669744140103e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -512.0,
      "logps/rejected": -716.0,
      "loss": 0.3043,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 2.09375,
      "rewards/rejected": -5.84375,
      "step": 1590
    },
    {
      "epoch": 0.11549844798960514,
      "grad_norm": 10.155961204890554,
      "learning_rate": 4.996365805746015e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5625,
      "logps/chosen": -474.0,
      "logps/rejected": -752.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.234375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.03125,
      "step": 1600
    },
    {
      "epoch": 0.11622031328954018,
      "grad_norm": 9.079844890608808,
      "learning_rate": 4.996018318099829e-07,
      "logits/chosen": -2.421875,
      "logits/rejected": -2.28125,
      "logps/chosen": -506.0,
      "logps/rejected": -712.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.46875,
      "rewards/margins": 2.171875,
      "rewards/rejected": -5.65625,
      "step": 1610
    },
    {
      "epoch": 0.1169421785894752,
      "grad_norm": 13.337729397169802,
      "learning_rate": 4.995654980669028e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -476.0,
      "logps/rejected": -748.0,
      "loss": 0.2937,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.03125,
      "rewards/margins": 2.875,
      "rewards/rejected": -5.90625,
      "step": 1620
    },
    {
      "epoch": 0.11766404388941024,
      "grad_norm": 9.987891055540013,
      "learning_rate": 4.995275795760816e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.734375,
      "logps/chosen": -454.0,
      "logps/rejected": -660.0,
      "loss": 0.2991,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.109375,
      "rewards/rejected": -5.0625,
      "step": 1630
    },
    {
      "epoch": 0.11838590918934527,
      "grad_norm": 12.804317711855225,
      "learning_rate": 4.994880765783026e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.671875,
      "logps/chosen": -484.0,
      "logps/rejected": -672.0,
      "loss": 0.2883,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.109375,
      "rewards/margins": 2.015625,
      "rewards/rejected": -5.125,
      "step": 1640
    },
    {
      "epoch": 0.1191077744892803,
      "grad_norm": 11.589955875632223,
      "learning_rate": 4.99446989324411e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -470.0,
      "logps/rejected": -756.0,
      "loss": 0.2847,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.953125,
      "rewards/margins": 3.046875,
      "rewards/rejected": -6.0,
      "step": 1650
    },
    {
      "epoch": 0.11982963978921533,
      "grad_norm": 14.992092750756044,
      "learning_rate": 4.99404318075312e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.546875,
      "logps/chosen": -488.0,
      "logps/rejected": -672.0,
      "loss": 0.2926,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.21875,
      "rewards/margins": 2.125,
      "rewards/rejected": -5.34375,
      "step": 1660
    },
    {
      "epoch": 0.12055150508915037,
      "grad_norm": 14.63652475825442,
      "learning_rate": 4.993600631019689e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.578125,
      "logps/chosen": -484.0,
      "logps/rejected": -680.0,
      "loss": 0.3115,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.328125,
      "rewards/margins": 1.984375,
      "rewards/rejected": -5.3125,
      "step": 1670
    },
    {
      "epoch": 0.1212733703890854,
      "grad_norm": 10.01896442134856,
      "learning_rate": 4.993142246854024e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -434.0,
      "logps/rejected": -656.0,
      "loss": 0.2917,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.15625,
      "rewards/rejected": -5.0,
      "step": 1680
    },
    {
      "epoch": 0.12199523568902043,
      "grad_norm": 10.480137264657502,
      "learning_rate": 4.992668031166876e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.703125,
      "logps/chosen": -444.0,
      "logps/rejected": -672.0,
      "loss": 0.3114,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.90625,
      "rewards/margins": 2.421875,
      "rewards/rejected": -5.34375,
      "step": 1690
    },
    {
      "epoch": 0.12271710098895545,
      "grad_norm": 9.543166172183264,
      "learning_rate": 4.992177986969526e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.484375,
      "logps/chosen": -430.0,
      "logps/rejected": -624.0,
      "loss": 0.3235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.71875,
      "rewards/margins": 1.890625,
      "rewards/rejected": -4.59375,
      "step": 1700
    },
    {
      "epoch": 0.12343896628889049,
      "grad_norm": 9.39264085167921,
      "learning_rate": 4.991672117373769e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -452.0,
      "logps/rejected": -712.0,
      "loss": 0.2971,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.796875,
      "rewards/margins": 2.625,
      "rewards/rejected": -5.4375,
      "step": 1710
    },
    {
      "epoch": 0.12416083158882553,
      "grad_norm": 9.35509206624716,
      "learning_rate": 4.991150425591891e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -448.0,
      "logps/rejected": -700.0,
      "loss": 0.2908,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.53125,
      "step": 1720
    },
    {
      "epoch": 0.12488269688876055,
      "grad_norm": 9.262201280713281,
      "learning_rate": 4.99061291493665e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -444.0,
      "logps/rejected": -676.0,
      "loss": 0.2925,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -5.3125,
      "step": 1730
    },
    {
      "epoch": 0.1256045621886956,
      "grad_norm": 10.008148122591312,
      "learning_rate": 4.99005958882125e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -450.0,
      "logps/rejected": -684.0,
      "loss": 0.3133,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.953125,
      "rewards/margins": 2.359375,
      "rewards/rejected": -5.3125,
      "step": 1740
    },
    {
      "epoch": 0.12632642748863063,
      "grad_norm": 8.554136981806407,
      "learning_rate": 4.989490450759331e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -482.0,
      "logps/rejected": -696.0,
      "loss": 0.2649,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -3.125,
      "rewards/margins": 2.3125,
      "rewards/rejected": -5.4375,
      "step": 1750
    },
    {
      "epoch": 0.12704829278856566,
      "grad_norm": 8.929491173417231,
      "learning_rate": 4.988905504364933e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.578125,
      "logps/chosen": -516.0,
      "logps/rejected": -768.0,
      "loss": 0.2715,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.09375,
      "step": 1760
    },
    {
      "epoch": 0.12777015808850067,
      "grad_norm": 19.85583075822656,
      "learning_rate": 4.988304753352482e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -454.0,
      "logps/rejected": -744.0,
      "loss": 0.2838,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.046875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -5.875,
      "step": 1770
    },
    {
      "epoch": 0.1284920233884357,
      "grad_norm": 11.573857511888404,
      "learning_rate": 4.987688201536764e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -404.0,
      "logps/rejected": -636.0,
      "loss": 0.3207,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -4.84375,
      "step": 1780
    },
    {
      "epoch": 0.12921388868837075,
      "grad_norm": 10.767926076980798,
      "learning_rate": 4.987055852832899e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.625,
      "logps/chosen": -504.0,
      "logps/rejected": -784.0,
      "loss": 0.2768,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.296875,
      "rewards/margins": 2.859375,
      "rewards/rejected": -6.15625,
      "step": 1790
    },
    {
      "epoch": 0.12993575398830579,
      "grad_norm": 9.480524665881354,
      "learning_rate": 4.986407711256319e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.46875,
      "logps/chosen": -434.0,
      "logps/rejected": -656.0,
      "loss": 0.2996,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.8125,
      "rewards/margins": 2.203125,
      "rewards/rejected": -5.0,
      "step": 1800
    },
    {
      "epoch": 0.13065761928824082,
      "grad_norm": 11.059595953397832,
      "learning_rate": 4.98574378092274e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.6875,
      "logps/chosen": -506.0,
      "logps/rejected": -760.0,
      "loss": 0.2706,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.40625,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.21875,
      "step": 1810
    },
    {
      "epoch": 0.13137948458817586,
      "grad_norm": 12.559652259820835,
      "learning_rate": 4.985064066048139e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.40625,
      "logps/chosen": -424.0,
      "logps/rejected": -660.0,
      "loss": 0.3016,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.625,
      "rewards/margins": 2.375,
      "rewards/rejected": -5.0,
      "step": 1820
    },
    {
      "epoch": 0.13210134988811087,
      "grad_norm": 16.780300866871737,
      "learning_rate": 4.984368570948724e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.484375,
      "logps/chosen": -444.0,
      "logps/rejected": -704.0,
      "loss": 0.2943,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.828125,
      "rewards/margins": 2.515625,
      "rewards/rejected": -5.34375,
      "step": 1830
    },
    {
      "epoch": 0.1328232151880459,
      "grad_norm": 10.65248837076819,
      "learning_rate": 4.983657300040907e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -472.0,
      "logps/rejected": -704.0,
      "loss": 0.2699,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.25,
      "rewards/margins": 2.484375,
      "rewards/rejected": -5.75,
      "step": 1840
    },
    {
      "epoch": 0.13354508048798094,
      "grad_norm": 10.948003648298677,
      "learning_rate": 4.982930257841278e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.453125,
      "logps/chosen": -456.0,
      "logps/rejected": -728.0,
      "loss": 0.2941,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.96875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -5.75,
      "step": 1850
    },
    {
      "epoch": 0.13426694578791598,
      "grad_norm": 12.047287233162004,
      "learning_rate": 4.982187448966575e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.46875,
      "logps/chosen": -536.0,
      "logps/rejected": -768.0,
      "loss": 0.2898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.640625,
      "rewards/margins": 2.390625,
      "rewards/rejected": -6.03125,
      "step": 1860
    },
    {
      "epoch": 0.13498881108785102,
      "grad_norm": 14.9941388186048,
      "learning_rate": 4.981428878133656e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.4375,
      "logps/chosen": -468.0,
      "logps/rejected": -772.0,
      "loss": 0.2811,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.109375,
      "rewards/margins": 3.09375,
      "rewards/rejected": -6.1875,
      "step": 1870
    },
    {
      "epoch": 0.13571067638778603,
      "grad_norm": 9.951835424657778,
      "learning_rate": 4.980654550159463e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.609375,
      "logps/chosen": -492.0,
      "logps/rejected": -708.0,
      "loss": 0.2732,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.1875,
      "rewards/margins": 2.5,
      "rewards/rejected": -5.6875,
      "step": 1880
    },
    {
      "epoch": 0.13643254168772107,
      "grad_norm": 8.861361984310934,
      "learning_rate": 4.979864469961004e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.578125,
      "logps/chosen": -464.0,
      "logps/rejected": -716.0,
      "loss": 0.2762,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 2.671875,
      "rewards/rejected": -5.75,
      "step": 1890
    },
    {
      "epoch": 0.1371544069876561,
      "grad_norm": 9.692735634395657,
      "learning_rate": 4.979058642555307e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -452.0,
      "logps/rejected": -676.0,
      "loss": 0.2931,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.84375,
      "rewards/margins": 2.46875,
      "rewards/rejected": -5.3125,
      "step": 1900
    },
    {
      "epoch": 0.13787627228759114,
      "grad_norm": 9.565594633496087,
      "learning_rate": 4.978237073059399e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -676.0,
      "loss": 0.2898,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.078125,
      "rewards/margins": 2.171875,
      "rewards/rejected": -5.25,
      "step": 1910
    },
    {
      "epoch": 0.13859813758752618,
      "grad_norm": 11.007909649107406,
      "learning_rate": 4.977399766690269e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -498.0,
      "logps/rejected": -796.0,
      "loss": 0.2902,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.1875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -6.3125,
      "step": 1920
    },
    {
      "epoch": 0.1393200028874612,
      "grad_norm": 9.107456714393756,
      "learning_rate": 4.976546728764836e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -498.0,
      "logps/rejected": -724.0,
      "loss": 0.284,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.3125,
      "rewards/margins": 2.4375,
      "rewards/rejected": -5.75,
      "step": 1930
    },
    {
      "epoch": 0.14004186818739622,
      "grad_norm": 8.927298343498093,
      "learning_rate": 4.975677964699912e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.40625,
      "logps/chosen": -502.0,
      "logps/rejected": -776.0,
      "loss": 0.2524,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.5625,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.34375,
      "step": 1940
    },
    {
      "epoch": 0.14076373348733126,
      "grad_norm": 10.144209347121677,
      "learning_rate": 4.974793480012174e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -464.0,
      "logps/rejected": -716.0,
      "loss": 0.2621,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.859375,
      "rewards/margins": 2.6875,
      "rewards/rejected": -5.5625,
      "step": 1950
    },
    {
      "epoch": 0.1414855987872663,
      "grad_norm": 7.533895015502329,
      "learning_rate": 4.973893280318125e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -492.0,
      "logps/rejected": -764.0,
      "loss": 0.2844,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.34375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.125,
      "step": 1960
    },
    {
      "epoch": 0.14220746408720134,
      "grad_norm": 13.017212573238576,
      "learning_rate": 4.972977371334056e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.5,
      "logps/chosen": -424.0,
      "logps/rejected": -672.0,
      "loss": 0.2873,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.5,
      "rewards/margins": 2.46875,
      "rewards/rejected": -4.96875,
      "step": 1970
    },
    {
      "epoch": 0.14292932938713637,
      "grad_norm": 8.958119059657708,
      "learning_rate": 4.972045758876014e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -488.0,
      "logps/rejected": -724.0,
      "loss": 0.2477,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.359375,
      "rewards/rejected": -5.625,
      "step": 1980
    },
    {
      "epoch": 0.14365119468707138,
      "grad_norm": 10.493203877125167,
      "learning_rate": 4.971098448859766e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.6875,
      "logps/chosen": -488.0,
      "logps/rejected": -692.0,
      "loss": 0.2933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.234375,
      "rewards/rejected": -5.5,
      "step": 1990
    },
    {
      "epoch": 0.14437305998700642,
      "grad_norm": 9.973862051012356,
      "learning_rate": 4.970135447300752e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.3125,
      "logps/chosen": -516.0,
      "logps/rejected": -788.0,
      "loss": 0.2595,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -3.5,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.0625,
      "step": 2000
    },
    {
      "epoch": 0.14509492528694146,
      "grad_norm": 13.28264552294119,
      "learning_rate": 4.969156760314064e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.453125,
      "logps/chosen": -524.0,
      "logps/rejected": -796.0,
      "loss": 0.2904,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.640625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.34375,
      "step": 2010
    },
    {
      "epoch": 0.1458167905868765,
      "grad_norm": 8.872169286182675,
      "learning_rate": 4.968162394114387e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -488.0,
      "logps/rejected": -744.0,
      "loss": 0.2523,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.265625,
      "rewards/margins": 2.703125,
      "rewards/rejected": -5.96875,
      "step": 2020
    },
    {
      "epoch": 0.14653865588681153,
      "grad_norm": 12.757195149734072,
      "learning_rate": 4.967152355015974e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.546875,
      "logps/chosen": -478.0,
      "logps/rejected": -724.0,
      "loss": 0.2783,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.375,
      "rewards/margins": 2.5625,
      "rewards/rejected": -5.9375,
      "step": 2030
    },
    {
      "epoch": 0.14726052118674654,
      "grad_norm": 12.796953074989913,
      "learning_rate": 4.966126649432603e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -496.0,
      "logps/rejected": -780.0,
      "loss": 0.2723,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.53125,
      "rewards/margins": 2.578125,
      "rewards/rejected": -6.125,
      "step": 2040
    },
    {
      "epoch": 0.14798238648668158,
      "grad_norm": 19.108616982264344,
      "learning_rate": 4.96508528387753e-07,
      "logits/chosen": -3.0625,
      "logits/rejected": -2.828125,
      "logps/chosen": -540.0,
      "logps/rejected": -812.0,
      "loss": 0.2763,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.96875,
      "rewards/margins": 2.734375,
      "rewards/rejected": -6.6875,
      "step": 2050
    },
    {
      "epoch": 0.14870425178661661,
      "grad_norm": 9.380783201026548,
      "learning_rate": 4.964028264963456e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.625,
      "logps/chosen": -516.0,
      "logps/rejected": -800.0,
      "loss": 0.2822,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.46875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -6.1875,
      "step": 2060
    },
    {
      "epoch": 0.14942611708655165,
      "grad_norm": 10.81595001993398,
      "learning_rate": 4.962955599402481e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -560.0,
      "logps/rejected": -784.0,
      "loss": 0.266,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.25,
      "rewards/rejected": -6.25,
      "step": 2070
    },
    {
      "epoch": 0.1501479823864867,
      "grad_norm": 11.456291625001526,
      "learning_rate": 4.961867294006059e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.796875,
      "logps/chosen": -608.0,
      "logps/rejected": -836.0,
      "loss": 0.3064,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 2.40625,
      "rewards/rejected": -6.875,
      "step": 2080
    },
    {
      "epoch": 0.15086984768642173,
      "grad_norm": 7.960182686516041,
      "learning_rate": 4.96076335568496e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5625,
      "logps/chosen": -516.0,
      "logps/rejected": -776.0,
      "loss": 0.2467,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.4375,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.21875,
      "step": 2090
    },
    {
      "epoch": 0.15159171298635674,
      "grad_norm": 10.25400623338689,
      "learning_rate": 4.959643791449222e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -564.0,
      "logps/rejected": -788.0,
      "loss": 0.2977,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.375,
      "step": 2100
    },
    {
      "epoch": 0.15231357828629177,
      "grad_norm": 10.09340211565806,
      "learning_rate": 4.958508608408109e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.4375,
      "logps/chosen": -544.0,
      "logps/rejected": -736.0,
      "loss": 0.2829,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.921875,
      "rewards/margins": 2.03125,
      "rewards/rejected": -5.9375,
      "step": 2110
    },
    {
      "epoch": 0.1530354435862268,
      "grad_norm": 8.537784621679538,
      "learning_rate": 4.957357813770064e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -596.0,
      "logps/rejected": -860.0,
      "loss": 0.2479,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.28125,
      "rewards/margins": 2.765625,
      "rewards/rejected": -7.03125,
      "step": 2120
    },
    {
      "epoch": 0.15375730888616185,
      "grad_norm": 8.985222338453365,
      "learning_rate": 4.956191414842665e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.71875,
      "logps/chosen": -572.0,
      "logps/rejected": -860.0,
      "loss": 0.2629,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.03125,
      "step": 2130
    },
    {
      "epoch": 0.15447917418609688,
      "grad_norm": 10.993021927824719,
      "learning_rate": 4.955009419032575e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.515625,
      "logps/chosen": -572.0,
      "logps/rejected": -840.0,
      "loss": 0.2681,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.8125,
      "rewards/rejected": -6.8125,
      "step": 2140
    },
    {
      "epoch": 0.1552010394860319,
      "grad_norm": 11.792550816411442,
      "learning_rate": 4.953811833845503e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -528.0,
      "logps/rejected": -756.0,
      "loss": 0.265,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -3.75,
      "rewards/margins": 2.328125,
      "rewards/rejected": -6.0625,
      "step": 2150
    },
    {
      "epoch": 0.15592290478596693,
      "grad_norm": 13.163433586155344,
      "learning_rate": 4.952598666886145e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -856.0,
      "loss": 0.2516,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.3125,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.0,
      "step": 2160
    },
    {
      "epoch": 0.15664477008590197,
      "grad_norm": 15.36607001210862,
      "learning_rate": 4.951369925858147e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -544.0,
      "logps/rejected": -792.0,
      "loss": 0.2598,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -3.796875,
      "rewards/margins": 2.703125,
      "rewards/rejected": -6.5,
      "step": 2170
    },
    {
      "epoch": 0.157366635385837,
      "grad_norm": 8.701540096554936,
      "learning_rate": 4.950125618564046e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.765625,
      "logps/chosen": -528.0,
      "logps/rejected": -784.0,
      "loss": 0.251,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.625,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.1875,
      "step": 2180
    },
    {
      "epoch": 0.15808850068577204,
      "grad_norm": 10.616304811087307,
      "learning_rate": 4.948865752905228e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.703125,
      "logps/chosen": -576.0,
      "logps/rejected": -828.0,
      "loss": 0.2533,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.65625,
      "rewards/rejected": -6.75,
      "step": 2190
    },
    {
      "epoch": 0.15881036598570708,
      "grad_norm": 10.818629515177625,
      "learning_rate": 4.947590336881874e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -584.0,
      "logps/rejected": -816.0,
      "loss": 0.2593,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.625,
      "rewards/rejected": -6.6875,
      "step": 2200
    },
    {
      "epoch": 0.1595322312856421,
      "grad_norm": 10.595421594796772,
      "learning_rate": 4.946299378592912e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -532.0,
      "logps/rejected": -784.0,
      "loss": 0.2795,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.578125,
      "rewards/margins": 2.546875,
      "rewards/rejected": -6.125,
      "step": 2210
    },
    {
      "epoch": 0.16025409658557713,
      "grad_norm": 13.24386115549115,
      "learning_rate": 4.944992886235961e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -572.0,
      "logps/rejected": -828.0,
      "loss": 0.2521,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.96875,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.71875,
      "step": 2220
    },
    {
      "epoch": 0.16097596188551216,
      "grad_norm": 8.619302388111215,
      "learning_rate": 4.943670868107284e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -532.0,
      "logps/rejected": -832.0,
      "loss": 0.27,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.65625,
      "rewards/margins": 3.078125,
      "rewards/rejected": -6.71875,
      "step": 2230
    },
    {
      "epoch": 0.1616978271854472,
      "grad_norm": 8.272453046055942,
      "learning_rate": 4.942333332601731e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -564.0,
      "logps/rejected": -864.0,
      "loss": 0.2661,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.984375,
      "rewards/margins": 3.1875,
      "rewards/rejected": -7.15625,
      "step": 2240
    },
    {
      "epoch": 0.16241969248538224,
      "grad_norm": 6.171166558818759,
      "learning_rate": 4.940980288212691e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.375,
      "logps/chosen": -576.0,
      "logps/rejected": -800.0,
      "loss": 0.2574,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.03125,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.46875,
      "step": 2250
    },
    {
      "epoch": 0.16314155778531725,
      "grad_norm": 9.42178572046336,
      "learning_rate": 4.939611743532031e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -560.0,
      "logps/rejected": -828.0,
      "loss": 0.2385,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.9375,
      "rewards/margins": 2.828125,
      "rewards/rejected": -6.78125,
      "step": 2260
    },
    {
      "epoch": 0.16386342308525229,
      "grad_norm": 8.438515261081413,
      "learning_rate": 4.93822770725005e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.625,
      "logps/chosen": -544.0,
      "logps/rejected": -776.0,
      "loss": 0.2737,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.734375,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.34375,
      "step": 2270
    },
    {
      "epoch": 0.16458528838518732,
      "grad_norm": 11.043392051990788,
      "learning_rate": 4.936828188155413e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.65625,
      "logps/chosen": -520.0,
      "logps/rejected": -776.0,
      "loss": 0.2496,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.625,
      "rewards/margins": 2.640625,
      "rewards/rejected": -6.28125,
      "step": 2280
    },
    {
      "epoch": 0.16530715368512236,
      "grad_norm": 10.831257780639715,
      "learning_rate": 4.935413195135106e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.6875,
      "logps/chosen": -572.0,
      "logps/rejected": -872.0,
      "loss": 0.2479,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.15625,
      "step": 2290
    },
    {
      "epoch": 0.1660290189850574,
      "grad_norm": 9.747157511340994,
      "learning_rate": 4.933982737174374e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.640625,
      "logps/chosen": -560.0,
      "logps/rejected": -852.0,
      "loss": 0.2701,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.796875,
      "rewards/rejected": -6.875,
      "step": 2300
    },
    {
      "epoch": 0.1667508842849924,
      "grad_norm": 9.870871070277088,
      "learning_rate": 4.932536823356664e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -620.0,
      "logps/rejected": -880.0,
      "loss": 0.2543,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.578125,
      "rewards/rejected": -7.28125,
      "step": 2310
    },
    {
      "epoch": 0.16747274958492744,
      "grad_norm": 8.8575895636107,
      "learning_rate": 4.931075462863567e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.609375,
      "logps/chosen": -604.0,
      "logps/rejected": -912.0,
      "loss": 0.2594,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.4375,
      "step": 2320
    },
    {
      "epoch": 0.16819461488486248,
      "grad_norm": 7.816395810688846,
      "learning_rate": 4.929598664974762e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.609375,
      "logps/chosen": -652.0,
      "logps/rejected": -940.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.90625,
      "step": 2330
    },
    {
      "epoch": 0.16891648018479752,
      "grad_norm": 8.344064880799563,
      "learning_rate": 4.928106439067958e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -816.0,
      "loss": 0.2669,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.53125,
      "rewards/rejected": -6.625,
      "step": 2340
    },
    {
      "epoch": 0.16963834548473256,
      "grad_norm": 10.994252200422187,
      "learning_rate": 4.926598794618829e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -552.0,
      "logps/rejected": -784.0,
      "loss": 0.278,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.921875,
      "rewards/margins": 2.5625,
      "rewards/rejected": -6.46875,
      "step": 2350
    },
    {
      "epoch": 0.1703602107846676,
      "grad_norm": 15.14815293816792,
      "learning_rate": 4.92507574120096e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -604.0,
      "logps/rejected": -852.0,
      "loss": 0.2777,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.609375,
      "rewards/rejected": -7.0,
      "step": 2360
    },
    {
      "epoch": 0.1710820760846026,
      "grad_norm": 7.821370479620557,
      "learning_rate": 4.923537288485779e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -556.0,
      "logps/rejected": -812.0,
      "loss": 0.2638,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.59375,
      "rewards/rejected": -6.6875,
      "step": 2370
    },
    {
      "epoch": 0.17180394138453764,
      "grad_norm": 9.211826498394792,
      "learning_rate": 4.921983446242506e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -644.0,
      "logps/rejected": -944.0,
      "loss": 0.2493,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -7.875,
      "step": 2380
    },
    {
      "epoch": 0.17252580668447268,
      "grad_norm": 8.298256232187494,
      "learning_rate": 4.92041422433808e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -548.0,
      "logps/rejected": -924.0,
      "loss": 0.2348,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.921875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -7.65625,
      "step": 2390
    },
    {
      "epoch": 0.17324767198440771,
      "grad_norm": 10.983625039262604,
      "learning_rate": 4.918829632737103e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.625,
      "logps/chosen": -560.0,
      "logps/rejected": -896.0,
      "loss": 0.2633,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.125,
      "rewards/margins": 3.375,
      "rewards/rejected": -7.5,
      "step": 2400
    },
    {
      "epoch": 0.17396953728434275,
      "grad_norm": 12.61039915689691,
      "learning_rate": 4.917229681501774e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.53125,
      "logps/chosen": -580.0,
      "logps/rejected": -824.0,
      "loss": 0.264,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.0625,
      "rewards/margins": 2.609375,
      "rewards/rejected": -6.6875,
      "step": 2410
    },
    {
      "epoch": 0.17469140258427776,
      "grad_norm": 11.643682996836732,
      "learning_rate": 4.91561438079183e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.34375,
      "logps/chosen": -628.0,
      "logps/rejected": -900.0,
      "loss": 0.2668,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.625,
      "rewards/margins": 2.6875,
      "rewards/rejected": -7.3125,
      "step": 2420
    },
    {
      "epoch": 0.1754132678842128,
      "grad_norm": 13.088952482660957,
      "learning_rate": 4.913983740864473e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.390625,
      "logps/chosen": -632.0,
      "logps/rejected": -920.0,
      "loss": 0.2359,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.859375,
      "rewards/rejected": -7.4375,
      "step": 2430
    },
    {
      "epoch": 0.17613513318414784,
      "grad_norm": 10.188119267424455,
      "learning_rate": 4.91233777207431e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.453125,
      "logps/chosen": -592.0,
      "logps/rejected": -916.0,
      "loss": 0.2748,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -7.6875,
      "step": 2440
    },
    {
      "epoch": 0.17685699848408287,
      "grad_norm": 8.609826083562075,
      "learning_rate": 4.910676484873292e-07,
      "logits/chosen": -2.5,
      "logits/rejected": -2.390625,
      "logps/chosen": -612.0,
      "logps/rejected": -864.0,
      "loss": 0.2658,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.4375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -7.09375,
      "step": 2450
    },
    {
      "epoch": 0.1775788637840179,
      "grad_norm": 10.48253643988837,
      "learning_rate": 4.908999889810633e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -572.0,
      "logps/rejected": -876.0,
      "loss": 0.2331,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.15625,
      "step": 2460
    },
    {
      "epoch": 0.17830072908395295,
      "grad_norm": 17.344352556263694,
      "learning_rate": 4.907307997532761e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -632.0,
      "logps/rejected": -984.0,
      "loss": 0.2354,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 3.484375,
      "rewards/rejected": -8.125,
      "step": 2470
    },
    {
      "epoch": 0.17902259438388796,
      "grad_norm": 9.982854645114383,
      "learning_rate": 4.905600818783237e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -664.0,
      "logps/rejected": -980.0,
      "loss": 0.3273,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.25,
      "step": 2480
    },
    {
      "epoch": 0.179744459683823,
      "grad_norm": 10.68094022282627,
      "learning_rate": 4.903878364402691e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -568.0,
      "logps/rejected": -836.0,
      "loss": 0.2468,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.953125,
      "rewards/margins": 2.953125,
      "rewards/rejected": -6.90625,
      "step": 2490
    },
    {
      "epoch": 0.18046632498375803,
      "grad_norm": 8.296383042780498,
      "learning_rate": 4.902140645328759e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -564.0,
      "logps/rejected": -848.0,
      "loss": 0.2599,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.9375,
      "rewards/rejected": -7.125,
      "step": 2500
    },
    {
      "epoch": 0.18118819028369307,
      "grad_norm": 9.103787824302069,
      "learning_rate": 4.900387672596003e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -528.0,
      "logps/rejected": -816.0,
      "loss": 0.2318,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.671875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.6875,
      "step": 2510
    },
    {
      "epoch": 0.1819100555836281,
      "grad_norm": 16.67988360600736,
      "learning_rate": 4.898619457335848e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.4375,
      "logps/chosen": -568.0,
      "logps/rejected": -844.0,
      "loss": 0.248,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.921875,
      "rewards/rejected": -6.9375,
      "step": 2520
    },
    {
      "epoch": 0.18263192088356311,
      "grad_norm": 10.469848545031832,
      "learning_rate": 4.896836010776509e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.53125,
      "logps/chosen": -644.0,
      "logps/rejected": -1012.0,
      "loss": 0.2595,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.78125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.375,
      "step": 2530
    },
    {
      "epoch": 0.18335378618349815,
      "grad_norm": 9.661637065408652,
      "learning_rate": 4.895037344242921e-07,
      "logits/chosen": -2.484375,
      "logits/rejected": -2.296875,
      "logps/chosen": -580.0,
      "logps/rejected": -856.0,
      "loss": 0.2585,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.125,
      "rewards/margins": 2.953125,
      "rewards/rejected": -7.09375,
      "step": 2540
    },
    {
      "epoch": 0.1840756514834332,
      "grad_norm": 8.853616581850865,
      "learning_rate": 4.893223469156664e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -584.0,
      "logps/rejected": -856.0,
      "loss": 0.249,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -6.9375,
      "step": 2550
    },
    {
      "epoch": 0.18479751678336823,
      "grad_norm": 14.23122096213618,
      "learning_rate": 4.891394397035896e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.53125,
      "logps/chosen": -588.0,
      "logps/rejected": -848.0,
      "loss": 0.2458,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.25,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.0,
      "step": 2560
    },
    {
      "epoch": 0.18551938208330326,
      "grad_norm": 7.165047283079565,
      "learning_rate": 4.889550139495275e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.59375,
      "logps/chosen": -556.0,
      "logps/rejected": -872.0,
      "loss": 0.2193,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.90625,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.15625,
      "step": 2570
    },
    {
      "epoch": 0.1862412473832383,
      "grad_norm": 9.892022723104382,
      "learning_rate": 4.887690708245887e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -540.0,
      "logps/rejected": -860.0,
      "loss": 0.2276,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.765625,
      "rewards/margins": 3.125,
      "rewards/rejected": -6.875,
      "step": 2580
    },
    {
      "epoch": 0.1869631126831733,
      "grad_norm": 9.543867068973059,
      "learning_rate": 4.885816115095171e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -928.0,
      "loss": 0.2391,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.203125,
      "rewards/rejected": -7.5625,
      "step": 2590
    },
    {
      "epoch": 0.18768497798310835,
      "grad_norm": 10.889738728754864,
      "learning_rate": 4.883926371946843e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -608.0,
      "logps/rejected": -872.0,
      "loss": 0.2702,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.375,
      "rewards/margins": 2.671875,
      "rewards/rejected": -7.03125,
      "step": 2600
    },
    {
      "epoch": 0.18840684328304338,
      "grad_norm": 6.728294158129367,
      "learning_rate": 4.882021490800826e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -580.0,
      "logps/rejected": -816.0,
      "loss": 0.2438,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.453125,
      "rewards/rejected": -6.53125,
      "step": 2610
    },
    {
      "epoch": 0.18912870858297842,
      "grad_norm": 9.374612134195704,
      "learning_rate": 4.880101483753167e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.4375,
      "logps/chosen": -560.0,
      "logps/rejected": -896.0,
      "loss": 0.2198,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.34375,
      "step": 2620
    },
    {
      "epoch": 0.18985057388291346,
      "grad_norm": 8.403906697614635,
      "learning_rate": 4.878166362995963e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.578125,
      "logps/chosen": -588.0,
      "logps/rejected": -900.0,
      "loss": 0.2728,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.21875,
      "rewards/margins": 3.015625,
      "rewards/rejected": -7.25,
      "step": 2630
    },
    {
      "epoch": 0.19057243918284847,
      "grad_norm": 9.739452944604345,
      "learning_rate": 4.876216140817285e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.296875,
      "logps/chosen": -568.0,
      "logps/rejected": -856.0,
      "loss": 0.2399,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.09375,
      "rewards/margins": 2.890625,
      "rewards/rejected": -7.0,
      "step": 2640
    },
    {
      "epoch": 0.1912943044827835,
      "grad_norm": 9.412524327526398,
      "learning_rate": 4.874250829601094e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -672.0,
      "logps/rejected": -928.0,
      "loss": 0.2287,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.8125,
      "rewards/margins": 2.796875,
      "rewards/rejected": -7.59375,
      "step": 2650
    },
    {
      "epoch": 0.19201616978271854,
      "grad_norm": 9.67143825795057,
      "learning_rate": 4.872270441827174e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -668.0,
      "logps/rejected": -960.0,
      "loss": 0.2358,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.0,
      "rewards/rejected": -8.0,
      "step": 2660
    },
    {
      "epoch": 0.19273803508265358,
      "grad_norm": 10.225198499138108,
      "learning_rate": 4.870274990071038e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -568.0,
      "logps/rejected": -868.0,
      "loss": 0.2511,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.3125,
      "step": 2670
    },
    {
      "epoch": 0.19345990038258862,
      "grad_norm": 8.93422692887298,
      "learning_rate": 4.868264487003862e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.40625,
      "logps/chosen": -588.0,
      "logps/rejected": -908.0,
      "loss": 0.2518,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.125,
      "rewards/rejected": -7.40625,
      "step": 2680
    },
    {
      "epoch": 0.19418176568252365,
      "grad_norm": 8.108160852039612,
      "learning_rate": 4.866238945392393e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.46875,
      "logps/chosen": -588.0,
      "logps/rejected": -840.0,
      "loss": 0.2465,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.78125,
      "rewards/rejected": -6.96875,
      "step": 2690
    },
    {
      "epoch": 0.19490363098245866,
      "grad_norm": 8.305890952481619,
      "learning_rate": 4.864198378098877e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.46875,
      "logps/chosen": -510.0,
      "logps/rejected": -804.0,
      "loss": 0.237,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.5,
      "rewards/margins": 3.015625,
      "rewards/rejected": -6.53125,
      "step": 2700
    },
    {
      "epoch": 0.1956254962823937,
      "grad_norm": 12.954558843929119,
      "learning_rate": 4.862142798080973e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.578125,
      "logps/chosen": -576.0,
      "logps/rejected": -848.0,
      "loss": 0.2245,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.125,
      "step": 2710
    },
    {
      "epoch": 0.19634736158232874,
      "grad_norm": 6.775612276163364,
      "learning_rate": 4.860072218391669e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -556.0,
      "logps/rejected": -872.0,
      "loss": 0.2153,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -3.90625,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.21875,
      "step": 2720
    },
    {
      "epoch": 0.19706922688226378,
      "grad_norm": 8.770593576643066,
      "learning_rate": 4.857986652179203e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.4375,
      "logps/chosen": -596.0,
      "logps/rejected": -872.0,
      "loss": 0.2534,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.765625,
      "rewards/rejected": -6.9375,
      "step": 2730
    },
    {
      "epoch": 0.1977910921821988,
      "grad_norm": 8.729235176349604,
      "learning_rate": 4.855886112686977e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.59375,
      "logps/chosen": -596.0,
      "logps/rejected": -920.0,
      "loss": 0.2018,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.3125,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.5625,
      "step": 2740
    },
    {
      "epoch": 0.19851295748213382,
      "grad_norm": 10.466383153721448,
      "learning_rate": 4.853770613253476e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.59375,
      "logps/chosen": -564.0,
      "logps/rejected": -884.0,
      "loss": 0.2365,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.15625,
      "rewards/margins": 3.390625,
      "rewards/rejected": -7.53125,
      "step": 2750
    },
    {
      "epoch": 0.19923482278206886,
      "grad_norm": 10.738593768361735,
      "learning_rate": 4.851640167312178e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -568.0,
      "logps/rejected": -868.0,
      "loss": 0.224,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.0,
      "step": 2760
    },
    {
      "epoch": 0.1999566880820039,
      "grad_norm": 9.416300695572696,
      "learning_rate": 4.849494788391473e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.4375,
      "logps/chosen": -664.0,
      "logps/rejected": -1032.0,
      "loss": 0.2284,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.8125,
      "step": 2770
    },
    {
      "epoch": 0.20067855338193893,
      "grad_norm": 9.528669823636646,
      "learning_rate": 4.847334490114576e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5625,
      "logps/chosen": -628.0,
      "logps/rejected": -916.0,
      "loss": 0.2293,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.8125,
      "step": 2780
    },
    {
      "epoch": 0.20140041868187397,
      "grad_norm": 9.797059355953671,
      "learning_rate": 4.84515928619944e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.59375,
      "logps/chosen": -632.0,
      "logps/rejected": -988.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.1875,
      "step": 2790
    },
    {
      "epoch": 0.20212228398180898,
      "grad_norm": 9.406533092835442,
      "learning_rate": 4.84296919045867e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.4375,
      "logps/chosen": -624.0,
      "logps/rejected": -968.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.78125,
      "step": 2800
    },
    {
      "epoch": 0.20284414928174402,
      "grad_norm": 9.254549987899654,
      "learning_rate": 4.840764216799433e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.375,
      "logps/chosen": -608.0,
      "logps/rejected": -904.0,
      "loss": 0.2337,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.46875,
      "step": 2810
    },
    {
      "epoch": 0.20356601458167906,
      "grad_norm": 9.14538857003236,
      "learning_rate": 4.838544379223373e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.671875,
      "logps/chosen": -592.0,
      "logps/rejected": -904.0,
      "loss": 0.2619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.5,
      "step": 2820
    },
    {
      "epoch": 0.2042878798816141,
      "grad_norm": 9.535277544276601,
      "learning_rate": 4.836309691826518e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -556.0,
      "logps/rejected": -840.0,
      "loss": 0.238,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.03125,
      "rewards/margins": 2.875,
      "rewards/rejected": -6.90625,
      "step": 2830
    },
    {
      "epoch": 0.20500974518154913,
      "grad_norm": 8.327834581371992,
      "learning_rate": 4.834060168799195e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.46875,
      "logps/chosen": -592.0,
      "logps/rejected": -856.0,
      "loss": 0.2305,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.15625,
      "rewards/margins": 2.875,
      "rewards/rejected": -7.03125,
      "step": 2840
    },
    {
      "epoch": 0.20573161048148417,
      "grad_norm": 7.7347670431429405,
      "learning_rate": 4.831795824425938e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -620.0,
      "logps/rejected": -916.0,
      "loss": 0.2542,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.625,
      "step": 2850
    },
    {
      "epoch": 0.20645347578141918,
      "grad_norm": 8.15521635648461,
      "learning_rate": 4.829516673085394e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2113,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.4375,
      "step": 2860
    },
    {
      "epoch": 0.20717534108135421,
      "grad_norm": 8.022215432273788,
      "learning_rate": 4.827222729250236e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.46875,
      "logps/chosen": -596.0,
      "logps/rejected": -916.0,
      "loss": 0.2461,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.21875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -7.65625,
      "step": 2870
    },
    {
      "epoch": 0.20789720638128925,
      "grad_norm": 7.199943747449723,
      "learning_rate": 4.824914007487072e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -588.0,
      "logps/rejected": -932.0,
      "loss": 0.2155,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -7.75,
      "step": 2880
    },
    {
      "epoch": 0.2086190716812243,
      "grad_norm": 12.583865202253415,
      "learning_rate": 4.822590522456347e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -560.0,
      "logps/rejected": -844.0,
      "loss": 0.2309,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.0,
      "step": 2890
    },
    {
      "epoch": 0.20934093698115933,
      "grad_norm": 10.061884816438084,
      "learning_rate": 4.820252288912254e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.671875,
      "logps/chosen": -600.0,
      "logps/rejected": -876.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.3125,
      "rewards/margins": 2.875,
      "rewards/rejected": -7.1875,
      "step": 2900
    },
    {
      "epoch": 0.21006280228109434,
      "grad_norm": 8.24310045419468,
      "learning_rate": 4.817899321702642e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.625,
      "logps/chosen": -620.0,
      "logps/rejected": -952.0,
      "loss": 0.2341,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -7.84375,
      "step": 2910
    },
    {
      "epoch": 0.21078466758102937,
      "grad_norm": 9.993736684687924,
      "learning_rate": 4.815531635768916e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.390625,
      "logps/chosen": -628.0,
      "logps/rejected": -952.0,
      "loss": 0.2298,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.78125,
      "step": 2920
    },
    {
      "epoch": 0.2115065328809644,
      "grad_norm": 8.215550634524817,
      "learning_rate": 4.813149246145946e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.625,
      "logps/chosen": -648.0,
      "logps/rejected": -968.0,
      "loss": 0.2413,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.0625,
      "step": 2930
    },
    {
      "epoch": 0.21222839818089945,
      "grad_norm": 6.880787558392017,
      "learning_rate": 4.810752167961971e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.390625,
      "logps/chosen": -624.0,
      "logps/rejected": -904.0,
      "loss": 0.2364,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.71875,
      "rewards/rejected": -7.40625,
      "step": 2940
    },
    {
      "epoch": 0.21295026348083448,
      "grad_norm": 8.022287212940531,
      "learning_rate": 4.808340416438505e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -632.0,
      "logps/rejected": -880.0,
      "loss": 0.2212,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.75,
      "rewards/margins": 2.734375,
      "rewards/rejected": -7.46875,
      "step": 2950
    },
    {
      "epoch": 0.21367212878076952,
      "grad_norm": 10.935811215207599,
      "learning_rate": 4.805914006890234e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -692.0,
      "logps/rejected": -972.0,
      "loss": 0.2454,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.1875,
      "rewards/margins": 2.875,
      "rewards/rejected": -8.0625,
      "step": 2960
    },
    {
      "epoch": 0.21439399408070453,
      "grad_norm": 7.928137164090748,
      "learning_rate": 4.803472954724928e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -676.0,
      "logps/rejected": -944.0,
      "loss": 0.2296,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 2.828125,
      "rewards/rejected": -7.96875,
      "step": 2970
    },
    {
      "epoch": 0.21511585938063957,
      "grad_norm": 10.309732813070605,
      "learning_rate": 4.801017275443331e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -676.0,
      "logps/rejected": -980.0,
      "loss": 0.2371,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.375,
      "step": 2980
    },
    {
      "epoch": 0.2158377246805746,
      "grad_norm": 9.479637952138695,
      "learning_rate": 4.798546984639076e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.4375,
      "logps/chosen": -712.0,
      "logps/rejected": -1040.0,
      "loss": 0.216,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.3125,
      "rewards/rejected": -8.875,
      "step": 2990
    },
    {
      "epoch": 0.21655958998050964,
      "grad_norm": 11.653196066334537,
      "learning_rate": 4.796062097998578e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.375,
      "logps/chosen": -700.0,
      "logps/rejected": -988.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.375,
      "step": 3000
    },
    {
      "epoch": 0.21728145528044468,
      "grad_norm": 12.258514957096317,
      "learning_rate": 4.793562631300932e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.625,
      "logps/chosen": -712.0,
      "logps/rejected": -996.0,
      "loss": 0.2292,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.5,
      "rewards/margins": 2.875,
      "rewards/rejected": -8.375,
      "step": 3010
    },
    {
      "epoch": 0.2180033205803797,
      "grad_norm": 7.907531921224019,
      "learning_rate": 4.791048600417824e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.640625,
      "logps/chosen": -660.0,
      "logps/rejected": -984.0,
      "loss": 0.2244,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -8.375,
      "step": 3020
    },
    {
      "epoch": 0.21872518588031473,
      "grad_norm": 8.125306615010935,
      "learning_rate": 4.788520021313416e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.8125,
      "logps/chosen": -636.0,
      "logps/rejected": -952.0,
      "loss": 0.2078,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.875,
      "step": 3030
    },
    {
      "epoch": 0.21944705118024976,
      "grad_norm": 9.119966563903791,
      "learning_rate": 4.785976910044255e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.703125,
      "logps/chosen": -576.0,
      "logps/rejected": -924.0,
      "loss": 0.2032,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.1875,
      "rewards/margins": 3.625,
      "rewards/rejected": -7.8125,
      "step": 3040
    },
    {
      "epoch": 0.2201689164801848,
      "grad_norm": 12.1919687251649,
      "learning_rate": 4.783419282759168e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.578125,
      "logps/chosen": -624.0,
      "logps/rejected": -928.0,
      "loss": 0.2312,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.84375,
      "step": 3050
    },
    {
      "epoch": 0.22089078178011984,
      "grad_norm": 10.966337886877598,
      "learning_rate": 4.780847155699157e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.484375,
      "logps/chosen": -656.0,
      "logps/rejected": -960.0,
      "loss": 0.2282,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.140625,
      "rewards/rejected": -8.0625,
      "step": 3060
    },
    {
      "epoch": 0.22161264708005488,
      "grad_norm": 10.89546424385192,
      "learning_rate": 4.778260545197301e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.609375,
      "logps/chosen": -628.0,
      "logps/rejected": -916.0,
      "loss": 0.2326,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.78125,
      "rewards/margins": 2.9375,
      "rewards/rejected": -7.71875,
      "step": 3070
    },
    {
      "epoch": 0.22233451237998988,
      "grad_norm": 8.635404642662332,
      "learning_rate": 4.775659467678645e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.71875,
      "logps/chosen": -688.0,
      "logps/rejected": -1004.0,
      "loss": 0.213,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.390625,
      "rewards/rejected": -8.625,
      "step": 3080
    },
    {
      "epoch": 0.22305637767992492,
      "grad_norm": 13.172555524642066,
      "learning_rate": 4.773043939660105e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.671875,
      "logps/chosen": -656.0,
      "logps/rejected": -956.0,
      "loss": 0.2527,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.0625,
      "step": 3090
    },
    {
      "epoch": 0.22377824297985996,
      "grad_norm": 10.053263971609315,
      "learning_rate": 4.770413977750353e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.515625,
      "logps/chosen": -680.0,
      "logps/rejected": -984.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.3125,
      "step": 3100
    },
    {
      "epoch": 0.224500108279795,
      "grad_norm": 7.080435865876819,
      "learning_rate": 4.7677695986497225e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.515625,
      "logps/chosen": -676.0,
      "logps/rejected": -956.0,
      "loss": 0.2285,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 2.90625,
      "rewards/rejected": -7.90625,
      "step": 3110
    },
    {
      "epoch": 0.22522197357973003,
      "grad_norm": 8.489071949818994,
      "learning_rate": 4.7651108191500896e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.578125,
      "logps/chosen": -624.0,
      "logps/rejected": -948.0,
      "loss": 0.2217,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.3125,
      "rewards/rejected": -7.75,
      "step": 3120
    },
    {
      "epoch": 0.22594383887966504,
      "grad_norm": 7.48950614324499,
      "learning_rate": 4.762437656134778e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -648.0,
      "logps/rejected": -956.0,
      "loss": 0.2206,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.28125,
      "rewards/rejected": -8.0625,
      "step": 3130
    },
    {
      "epoch": 0.22666570417960008,
      "grad_norm": 11.147541103318718,
      "learning_rate": 4.7597501265784466e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.484375,
      "logps/chosen": -660.0,
      "logps/rejected": -924.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.0,
      "rewards/margins": 2.75,
      "rewards/rejected": -7.75,
      "step": 3140
    },
    {
      "epoch": 0.22738756947953512,
      "grad_norm": 9.2136696723812,
      "learning_rate": 4.757048247546982e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.453125,
      "logps/chosen": -612.0,
      "logps/rejected": -940.0,
      "loss": 0.2588,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.34375,
      "rewards/rejected": -7.96875,
      "step": 3150
    },
    {
      "epoch": 0.22810943477947015,
      "grad_norm": 12.83236893491969,
      "learning_rate": 4.7543320361973884e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.625,
      "step": 3160
    },
    {
      "epoch": 0.2288313000794052,
      "grad_norm": 9.608775546642143,
      "learning_rate": 4.751601509777683e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.625,
      "logps/chosen": -620.0,
      "logps/rejected": -968.0,
      "loss": 0.219,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.6875,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.125,
      "step": 3170
    },
    {
      "epoch": 0.2295531653793402,
      "grad_norm": 8.893320591024583,
      "learning_rate": 4.748856685626784e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.6875,
      "logps/chosen": -620.0,
      "logps/rejected": -932.0,
      "loss": 0.2299,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -7.6875,
      "step": 3180
    },
    {
      "epoch": 0.23027503067927524,
      "grad_norm": 6.813416083172801,
      "learning_rate": 4.7460975811743986e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.734375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.2321,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.3125,
      "rewards/rejected": -8.25,
      "step": 3190
    },
    {
      "epoch": 0.23099689597921028,
      "grad_norm": 11.910753128704322,
      "learning_rate": 4.743324213940917e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -580.0,
      "logps/rejected": -864.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.1875,
      "rewards/margins": 2.953125,
      "rewards/rejected": -7.15625,
      "step": 3200
    },
    {
      "epoch": 0.2317187612791453,
      "grad_norm": 10.880683342821822,
      "learning_rate": 4.740536601537295e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.71875,
      "logps/chosen": -632.0,
      "logps/rejected": -924.0,
      "loss": 0.226,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.125,
      "rewards/rejected": -7.6875,
      "step": 3210
    },
    {
      "epoch": 0.23244062657908035,
      "grad_norm": 9.387075119803855,
      "learning_rate": 4.73773476166495e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.546875,
      "logps/chosen": -620.0,
      "logps/rejected": -916.0,
      "loss": 0.2052,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.59375,
      "step": 3220
    },
    {
      "epoch": 0.2331624918790154,
      "grad_norm": 9.319384117903645,
      "learning_rate": 4.73491871211564e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -880.0,
      "loss": 0.2334,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.0,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.15625,
      "step": 3230
    },
    {
      "epoch": 0.2338843571789504,
      "grad_norm": 7.92062494143796,
      "learning_rate": 4.7320884707713573e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.375,
      "logps/chosen": -572.0,
      "logps/rejected": -872.0,
      "loss": 0.226,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.09375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.15625,
      "step": 3240
    },
    {
      "epoch": 0.23460622247888543,
      "grad_norm": 7.5905378566734445,
      "learning_rate": 4.7292440556042116e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.421875,
      "logps/chosen": -636.0,
      "logps/rejected": -956.0,
      "loss": 0.2381,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -8.125,
      "step": 3250
    },
    {
      "epoch": 0.23532808777882047,
      "grad_norm": 7.104939281324458,
      "learning_rate": 4.726385484676318e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -600.0,
      "logps/rejected": -876.0,
      "loss": 0.2311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.4375,
      "rewards/margins": 2.84375,
      "rewards/rejected": -7.28125,
      "step": 3260
    },
    {
      "epoch": 0.2360499530787555,
      "grad_norm": 7.112864388089024,
      "learning_rate": 4.723512776139679e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.453125,
      "logps/chosen": -592.0,
      "logps/rejected": -872.0,
      "loss": 0.218,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.5625,
      "rewards/margins": 2.65625,
      "rewards/rejected": -7.21875,
      "step": 3270
    },
    {
      "epoch": 0.23677181837869055,
      "grad_norm": 9.110259152947808,
      "learning_rate": 4.7206259482360734e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -624.0,
      "logps/rejected": -916.0,
      "loss": 0.2342,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.0625,
      "rewards/rejected": -7.65625,
      "step": 3280
    },
    {
      "epoch": 0.23749368367862556,
      "grad_norm": 8.883129926158574,
      "learning_rate": 4.7177250192969364e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -648.0,
      "logps/rejected": -976.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.75,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.25,
      "step": 3290
    },
    {
      "epoch": 0.2382155489785606,
      "grad_norm": 10.573842474018138,
      "learning_rate": 4.714810007743247e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -744.0,
      "logps/rejected": -1080.0,
      "loss": 0.2226,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.609375,
      "rewards/rejected": -9.25,
      "step": 3300
    },
    {
      "epoch": 0.23893741427849563,
      "grad_norm": 9.209497924010089,
      "learning_rate": 4.7118809320854077e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.46875,
      "logps/chosen": -672.0,
      "logps/rejected": -1000.0,
      "loss": 0.2122,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.6875,
      "step": 3310
    },
    {
      "epoch": 0.23965927957843067,
      "grad_norm": 9.372117356581454,
      "learning_rate": 4.7089378109231294e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.40625,
      "logps/chosen": -688.0,
      "logps/rejected": -984.0,
      "loss": 0.2315,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.34375,
      "rewards/margins": 2.96875,
      "rewards/rejected": -8.3125,
      "step": 3320
    },
    {
      "epoch": 0.2403811448783657,
      "grad_norm": 8.352914263375952,
      "learning_rate": 4.7059806629453116e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -648.0,
      "logps/rejected": -920.0,
      "loss": 0.2278,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.6875,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.65625,
      "step": 3330
    },
    {
      "epoch": 0.24110301017830074,
      "grad_norm": 10.512336386108508,
      "learning_rate": 4.7030095069299257e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.546875,
      "logps/chosen": -656.0,
      "logps/rejected": -976.0,
      "loss": 0.2011,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -8.125,
      "step": 3340
    },
    {
      "epoch": 0.24182487547823575,
      "grad_norm": 9.166170654186674,
      "learning_rate": 4.700024361743893e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.59375,
      "logps/chosen": -620.0,
      "logps/rejected": -960.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.5625,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.125,
      "step": 3350
    },
    {
      "epoch": 0.2425467407781708,
      "grad_norm": 10.052841673647455,
      "learning_rate": 4.697025246342967e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5625,
      "logps/chosen": -608.0,
      "logps/rejected": -916.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.40625,
      "rewards/rejected": -7.90625,
      "step": 3360
    },
    {
      "epoch": 0.24326860607810583,
      "grad_norm": 12.87931292960993,
      "learning_rate": 4.6940121797716127e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.5625,
      "logps/chosen": -588.0,
      "logps/rejected": -912.0,
      "loss": 0.227,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.28125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -7.625,
      "step": 3370
    },
    {
      "epoch": 0.24399047137804086,
      "grad_norm": 11.246155735848257,
      "learning_rate": 4.6909851811628853e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.578125,
      "logps/chosen": -644.0,
      "logps/rejected": -948.0,
      "loss": 0.1985,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.96875,
      "step": 3380
    },
    {
      "epoch": 0.2447123366779759,
      "grad_norm": 8.247097831323758,
      "learning_rate": 4.68794426973831e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -584.0,
      "logps/rejected": -892.0,
      "loss": 0.2203,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.375,
      "step": 3390
    },
    {
      "epoch": 0.2454342019779109,
      "grad_norm": 7.982031142460416,
      "learning_rate": 4.684889464807755e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.453125,
      "logps/chosen": -588.0,
      "logps/rejected": -912.0,
      "loss": 0.2141,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.3125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.46875,
      "step": 3400
    },
    {
      "epoch": 0.24615606727784595,
      "grad_norm": 8.859057538137234,
      "learning_rate": 4.681820785769317e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -624.0,
      "logps/rejected": -932.0,
      "loss": 0.2002,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.328125,
      "rewards/rejected": -7.875,
      "step": 3410
    },
    {
      "epoch": 0.24687793257778098,
      "grad_norm": 10.507595703830146,
      "learning_rate": 4.678738252109192e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.515625,
      "logps/chosen": -604.0,
      "logps/rejected": -916.0,
      "loss": 0.2272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.4375,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.71875,
      "step": 3420
    },
    {
      "epoch": 0.24759979787771602,
      "grad_norm": 9.701082860121796,
      "learning_rate": 4.675641883401553e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -624.0,
      "logps/rejected": -936.0,
      "loss": 0.2157,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.625,
      "rewards/margins": 3.15625,
      "rewards/rejected": -7.78125,
      "step": 3430
    },
    {
      "epoch": 0.24832166317765106,
      "grad_norm": 9.249938002509944,
      "learning_rate": 4.672531699308425e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.4375,
      "logps/chosen": -628.0,
      "logps/rejected": -964.0,
      "loss": 0.2167,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.1875,
      "step": 3440
    },
    {
      "epoch": 0.2490435284775861,
      "grad_norm": 10.341717167410792,
      "learning_rate": 4.669407719579562e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.546875,
      "logps/chosen": -604.0,
      "logps/rejected": -920.0,
      "loss": 0.2035,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.71875,
      "step": 3450
    },
    {
      "epoch": 0.2497653937775211,
      "grad_norm": 8.681196124713722,
      "learning_rate": 4.666269964052322e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.53125,
      "logps/chosen": -596.0,
      "logps/rejected": -952.0,
      "loss": 0.2126,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.0625,
      "step": 3460
    },
    {
      "epoch": 0.25048725907745617,
      "grad_norm": 9.212928240257371,
      "learning_rate": 4.6631184526515384e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.65625,
      "logps/chosen": -624.0,
      "logps/rejected": -924.0,
      "loss": 0.2196,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.53125,
      "rewards/margins": 3.140625,
      "rewards/rejected": -7.6875,
      "step": 3470
    },
    {
      "epoch": 0.2512091243773912,
      "grad_norm": 9.229554206053074,
      "learning_rate": 4.6599532053893936e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.53125,
      "logps/chosen": -660.0,
      "logps/rejected": -944.0,
      "loss": 0.2172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.90625,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.875,
      "step": 3480
    },
    {
      "epoch": 0.2519309896773262,
      "grad_norm": 7.2543396832627876,
      "learning_rate": 4.6567742423652955e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -972.0,
      "loss": 0.2158,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.1875,
      "step": 3490
    },
    {
      "epoch": 0.25265285497726125,
      "grad_norm": 8.156746365270518,
      "learning_rate": 4.6535815837657456e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.359375,
      "logps/chosen": -688.0,
      "logps/rejected": -1024.0,
      "loss": 0.2254,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.625,
      "step": 3500
    },
    {
      "epoch": 0.25337472027719626,
      "grad_norm": 9.476042547482471,
      "learning_rate": 4.6503752498642133e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.421875,
      "logps/chosen": -696.0,
      "logps/rejected": -972.0,
      "loss": 0.2247,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 2.953125,
      "rewards/rejected": -8.3125,
      "step": 3510
    },
    {
      "epoch": 0.25409658557713133,
      "grad_norm": 11.068802868497535,
      "learning_rate": 4.6471552610210073e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.640625,
      "logps/chosen": -648.0,
      "logps/rejected": -968.0,
      "loss": 0.2236,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.125,
      "step": 3520
    },
    {
      "epoch": 0.25481845087706634,
      "grad_norm": 9.184379291366328,
      "learning_rate": 4.6439216376831447e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -628.0,
      "logps/rejected": -972.0,
      "loss": 0.2076,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.65625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.0625,
      "step": 3530
    },
    {
      "epoch": 0.25554031617700135,
      "grad_norm": 12.322007532110128,
      "learning_rate": 4.6406744003842213e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -612.0,
      "logps/rejected": -920.0,
      "loss": 0.2124,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.25,
      "rewards/rejected": -7.75,
      "step": 3540
    },
    {
      "epoch": 0.2562621814769364,
      "grad_norm": 7.159317606243659,
      "learning_rate": 4.6374135697442833e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.546875,
      "logps/chosen": -596.0,
      "logps/rejected": -940.0,
      "loss": 0.2098,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.40625,
      "rewards/margins": 3.453125,
      "rewards/rejected": -7.875,
      "step": 3550
    },
    {
      "epoch": 0.2569840467768714,
      "grad_norm": 29.4711769624419,
      "learning_rate": 4.634139166469695e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.390625,
      "logps/chosen": -620.0,
      "logps/rejected": -944.0,
      "loss": 0.2267,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.5,
      "rewards/margins": 3.078125,
      "rewards/rejected": -7.59375,
      "step": 3560
    },
    {
      "epoch": 0.2577059120768065,
      "grad_norm": 7.80532559868148,
      "learning_rate": 4.6308512113530063e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.46875,
      "logps/chosen": -560.0,
      "logps/rejected": -852.0,
      "loss": 0.2187,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.0,
      "rewards/margins": 2.984375,
      "rewards/rejected": -7.0,
      "step": 3570
    },
    {
      "epoch": 0.2584277773767415,
      "grad_norm": 7.185706232473536,
      "learning_rate": 4.627549725272822e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.59375,
      "logps/chosen": -604.0,
      "logps/rejected": -916.0,
      "loss": 0.2362,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.265625,
      "rewards/rejected": -7.59375,
      "step": 3580
    },
    {
      "epoch": 0.2591496426766765,
      "grad_norm": 7.836489814314065,
      "learning_rate": 4.62423472919367e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.421875,
      "logps/chosen": -604.0,
      "logps/rejected": -896.0,
      "loss": 0.2143,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -4.40625,
      "rewards/margins": 3.0,
      "rewards/rejected": -7.40625,
      "step": 3590
    },
    {
      "epoch": 0.25987150797661157,
      "grad_norm": 10.312658330041376,
      "learning_rate": 4.6209062441658654e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.359375,
      "logps/chosen": -604.0,
      "logps/rejected": -904.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.109375,
      "rewards/rejected": -7.5625,
      "step": 3600
    },
    {
      "epoch": 0.2605933732765466,
      "grad_norm": 7.660185299883346,
      "learning_rate": 4.6175642913253783e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -604.0,
      "logps/rejected": -912.0,
      "loss": 0.1951,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.65625,
      "step": 3610
    },
    {
      "epoch": 0.26131523857648165,
      "grad_norm": 9.630986662907384,
      "learning_rate": 4.6142088918937016e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -608.0,
      "logps/rejected": -952.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.4375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -8.0625,
      "step": 3620
    },
    {
      "epoch": 0.26203710387641665,
      "grad_norm": 7.9637824352647195,
      "learning_rate": 4.6108400671777135e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.40625,
      "logps/chosen": -620.0,
      "logps/rejected": -928.0,
      "loss": 0.2137,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.75,
      "step": 3630
    },
    {
      "epoch": 0.2627589691763517,
      "grad_norm": 10.302801071286787,
      "learning_rate": 4.607457838569544e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.296875,
      "logps/chosen": -600.0,
      "logps/rejected": -964.0,
      "loss": 0.208,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.375,
      "rewards/margins": 3.484375,
      "rewards/rejected": -7.84375,
      "step": 3640
    },
    {
      "epoch": 0.26348083447628673,
      "grad_norm": 9.200762753858287,
      "learning_rate": 4.6040622275464355e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -580.0,
      "logps/rejected": -960.0,
      "loss": 0.2058,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.0625,
      "step": 3650
    },
    {
      "epoch": 0.26420269977622174,
      "grad_norm": 8.72614280680941,
      "learning_rate": 4.6006532556706124e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.515625,
      "logps/chosen": -532.0,
      "logps/rejected": -856.0,
      "loss": 0.2263,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -3.875,
      "rewards/margins": 3.296875,
      "rewards/rejected": -7.15625,
      "step": 3660
    },
    {
      "epoch": 0.2649245650761568,
      "grad_norm": 9.95080301383368,
      "learning_rate": 4.5972309445891386e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.515625,
      "logps/chosen": -544.0,
      "logps/rejected": -884.0,
      "loss": 0.2212,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.859375,
      "rewards/margins": 3.609375,
      "rewards/rejected": -7.46875,
      "step": 3670
    },
    {
      "epoch": 0.2656464303760918,
      "grad_norm": 7.898181548134635,
      "learning_rate": 4.593795316033783e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -588.0,
      "logps/rejected": -872.0,
      "loss": 0.2504,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.28125,
      "rewards/margins": 2.96875,
      "rewards/rejected": -7.28125,
      "step": 3680
    },
    {
      "epoch": 0.2663682956760269,
      "grad_norm": 9.27071552978421,
      "learning_rate": 4.5903463918208815e-07,
      "logits/chosen": -2.515625,
      "logits/rejected": -2.296875,
      "logps/chosen": -560.0,
      "logps/rejected": -904.0,
      "loss": 0.2182,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.03125,
      "rewards/margins": 3.46875,
      "rewards/rejected": -7.5,
      "step": 3690
    },
    {
      "epoch": 0.2670901609759619,
      "grad_norm": 7.471297940396982,
      "learning_rate": 4.586884193851197e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.421875,
      "logps/chosen": -568.0,
      "logps/rejected": -864.0,
      "loss": 0.2165,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.125,
      "rewards/margins": 2.921875,
      "rewards/rejected": -7.03125,
      "step": 3700
    },
    {
      "epoch": 0.2678120262758969,
      "grad_norm": 9.198139304204727,
      "learning_rate": 4.5834087441097806e-07,
      "logits/chosen": -2.9375,
      "logits/rejected": -2.5625,
      "logps/chosen": -568.0,
      "logps/rejected": -960.0,
      "loss": 0.1971,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.125,
      "rewards/margins": 3.90625,
      "rewards/rejected": -8.0625,
      "step": 3710
    },
    {
      "epoch": 0.26853389157583196,
      "grad_norm": 7.074538199167009,
      "learning_rate": 4.5799200646658345e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.5,
      "logps/chosen": -568.0,
      "logps/rejected": -900.0,
      "loss": 0.1914,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -4.25,
      "rewards/margins": 3.28125,
      "rewards/rejected": -7.53125,
      "step": 3720
    },
    {
      "epoch": 0.26925575687576697,
      "grad_norm": 7.9353245918675,
      "learning_rate": 4.576418177672568e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.6875,
      "logps/chosen": -680.0,
      "logps/rejected": -1020.0,
      "loss": 0.2194,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.5625,
      "step": 3730
    },
    {
      "epoch": 0.26997762217570204,
      "grad_norm": 7.362232757163594,
      "learning_rate": 4.5729031053670596e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -576.0,
      "logps/rejected": -888.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.0625,
      "rewards/margins": 3.421875,
      "rewards/rejected": -7.46875,
      "step": 3740
    },
    {
      "epoch": 0.27069948747563705,
      "grad_norm": 9.803377167019393,
      "learning_rate": 4.569374870070114e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.65625,
      "logps/chosen": -620.0,
      "logps/rejected": -960.0,
      "loss": 0.2077,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.125,
      "step": 3750
    },
    {
      "epoch": 0.27142135277557206,
      "grad_norm": 6.923819353668018,
      "learning_rate": 4.565833494186122e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.734375,
      "logps/chosen": -644.0,
      "logps/rejected": -968.0,
      "loss": 0.1945,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -8.1875,
      "step": 3760
    },
    {
      "epoch": 0.2721432180755071,
      "grad_norm": 7.409462373704994,
      "learning_rate": 4.562279000202919e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -624.0,
      "logps/rejected": -952.0,
      "loss": 0.1891,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.46875,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.125,
      "step": 3770
    },
    {
      "epoch": 0.27286508337544213,
      "grad_norm": 8.55425171800717,
      "learning_rate": 4.5587114106916366e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.609375,
      "logps/chosen": -664.0,
      "logps/rejected": -1040.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.6875,
      "step": 3780
    },
    {
      "epoch": 0.2735869486753772,
      "grad_norm": 8.283760196639793,
      "learning_rate": 4.5551307483065664e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.65625,
      "logps/chosen": -716.0,
      "logps/rejected": -1064.0,
      "loss": 0.2264,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.125,
      "step": 3790
    },
    {
      "epoch": 0.2743088139753122,
      "grad_norm": 9.334259623082545,
      "learning_rate": 4.5515370357850136e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.640625,
      "logps/chosen": -664.0,
      "logps/rejected": -1056.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.890625,
      "rewards/rejected": -8.875,
      "step": 3800
    },
    {
      "epoch": 0.2750306792752472,
      "grad_norm": 8.790752805238721,
      "learning_rate": 4.547930295947151e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.578125,
      "logps/chosen": -640.0,
      "logps/rejected": -960.0,
      "loss": 0.1949,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.71875,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.25,
      "step": 3810
    },
    {
      "epoch": 0.2757525445751823,
      "grad_norm": 11.07615206391536,
      "learning_rate": 4.5443105516958737e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.640625,
      "logps/chosen": -700.0,
      "logps/rejected": -1072.0,
      "loss": 0.2074,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.34375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.25,
      "step": 3820
    },
    {
      "epoch": 0.2764744098751173,
      "grad_norm": 12.156443108793454,
      "learning_rate": 4.5406778260166586e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.421875,
      "logps/chosen": -648.0,
      "logps/rejected": -984.0,
      "loss": 0.2064,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.5,
      "step": 3830
    },
    {
      "epoch": 0.27719627517505235,
      "grad_norm": 9.054451472292419,
      "learning_rate": 4.5370321419774117e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.390625,
      "logps/chosen": -616.0,
      "logps/rejected": -936.0,
      "loss": 0.2138,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.59375,
      "rewards/margins": 3.171875,
      "rewards/rejected": -7.78125,
      "step": 3840
    },
    {
      "epoch": 0.27791814047498736,
      "grad_norm": 6.698493216999741,
      "learning_rate": 4.5333735227283274e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.5,
      "logps/chosen": -676.0,
      "logps/rejected": -992.0,
      "loss": 0.2083,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.4375,
      "step": 3850
    },
    {
      "epoch": 0.2786400057749224,
      "grad_norm": 9.735985205726063,
      "learning_rate": 4.5297019915017375e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.53125,
      "logps/chosen": -672.0,
      "logps/rejected": -1032.0,
      "loss": 0.1875,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.875,
      "step": 3860
    },
    {
      "epoch": 0.27936187107485744,
      "grad_norm": 8.810687694126111,
      "learning_rate": 4.5260175716119655e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.53125,
      "logps/chosen": -660.0,
      "logps/rejected": -968.0,
      "loss": 0.212,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.71875,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.0625,
      "step": 3870
    },
    {
      "epoch": 0.28008373637479245,
      "grad_norm": 10.6784883112821,
      "learning_rate": 4.522320286455179e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.546875,
      "logps/chosen": -632.0,
      "logps/rejected": -944.0,
      "loss": 0.2101,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.1875,
      "step": 3880
    },
    {
      "epoch": 0.2808056016747275,
      "grad_norm": 9.94830452756906,
      "learning_rate": 4.5186101595092403e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -664.0,
      "logps/rejected": -980.0,
      "loss": 0.2116,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.3125,
      "step": 3890
    },
    {
      "epoch": 0.2815274669746625,
      "grad_norm": 9.418299320110647,
      "learning_rate": 4.5148872143335566e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.578125,
      "logps/chosen": -656.0,
      "logps/rejected": -1016.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.71875,
      "rewards/rejected": -8.625,
      "step": 3900
    },
    {
      "epoch": 0.2822493322745976,
      "grad_norm": 11.904953591713511,
      "learning_rate": 4.5111514745689307e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.4375,
      "logps/chosen": -640.0,
      "logps/rejected": -1024.0,
      "loss": 0.2084,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.890625,
      "rewards/rejected": -8.875,
      "step": 3910
    },
    {
      "epoch": 0.2829711975745326,
      "grad_norm": 11.15522188195295,
      "learning_rate": 4.507402963937414e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -704.0,
      "logps/rejected": -1064.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.0,
      "step": 3920
    },
    {
      "epoch": 0.2836930628744676,
      "grad_norm": 11.36508327799745,
      "learning_rate": 4.5036417062421506e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -748.0,
      "logps/rejected": -1088.0,
      "loss": 0.2226,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.8125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -9.375,
      "step": 3930
    },
    {
      "epoch": 0.28441492817440267,
      "grad_norm": 9.897929517208977,
      "learning_rate": 4.4998677253672297e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.390625,
      "logps/chosen": -732.0,
      "logps/rejected": -1032.0,
      "loss": 0.2208,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.046875,
      "rewards/rejected": -8.6875,
      "step": 3940
    },
    {
      "epoch": 0.2851367934743377,
      "grad_norm": 9.778813209573705,
      "learning_rate": 4.496081045277533e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.359375,
      "logps/chosen": -736.0,
      "logps/rejected": -1080.0,
      "loss": 0.2031,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.875,
      "step": 3950
    },
    {
      "epoch": 0.28585865877427274,
      "grad_norm": 7.8313055942532275,
      "learning_rate": 4.492281690018583e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -696.0,
      "logps/rejected": -1048.0,
      "loss": 0.1818,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.75,
      "rewards/rejected": -9.125,
      "step": 3960
    },
    {
      "epoch": 0.28658052407420775,
      "grad_norm": 8.317207474938273,
      "learning_rate": 4.4884696837163905e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.59375,
      "logps/chosen": -748.0,
      "logps/rejected": -1056.0,
      "loss": 0.2117,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.9375,
      "step": 3970
    },
    {
      "epoch": 0.28730238937414276,
      "grad_norm": 10.26976433894447,
      "learning_rate": 4.484645050577299e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.375,
      "logps/chosen": -724.0,
      "logps/rejected": -1056.0,
      "loss": 0.2079,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.265625,
      "rewards/rejected": -8.9375,
      "step": 3980
    },
    {
      "epoch": 0.28802425467407783,
      "grad_norm": 11.840261029137109,
      "learning_rate": 4.480807814887833e-07,
      "logits/chosen": -2.546875,
      "logits/rejected": -2.40625,
      "logps/chosen": -660.0,
      "logps/rejected": -960.0,
      "loss": 0.214,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.25,
      "step": 3990
    },
    {
      "epoch": 0.28874611997401284,
      "grad_norm": 8.811651662253073,
      "learning_rate": 4.476958001014544e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.4375,
      "logps/chosen": -728.0,
      "logps/rejected": -1064.0,
      "loss": 0.2135,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.125,
      "step": 4000
    },
    {
      "epoch": 0.2894679852739479,
      "grad_norm": 8.267204182411973,
      "learning_rate": 4.4730956334038565e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -736.0,
      "logps/rejected": -1088.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.4375,
      "step": 4010
    },
    {
      "epoch": 0.2901898505738829,
      "grad_norm": 7.77369954894956,
      "learning_rate": 4.46922073658191e-07,
      "logits/chosen": -2.671875,
      "logits/rejected": -2.5,
      "logps/chosen": -708.0,
      "logps/rejected": -1056.0,
      "loss": 0.2009,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.1875,
      "step": 4020
    },
    {
      "epoch": 0.2909117158738179,
      "grad_norm": 8.886847365802758,
      "learning_rate": 4.465333335154406e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.40625,
      "logps/chosen": -684.0,
      "logps/rejected": -1020.0,
      "loss": 0.2115,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.6875,
      "step": 4030
    },
    {
      "epoch": 0.291633581173753,
      "grad_norm": 11.540304268095925,
      "learning_rate": 4.461433453806449e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.34375,
      "logps/chosen": -712.0,
      "logps/rejected": -1032.0,
      "loss": 0.2096,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.6875,
      "step": 4040
    },
    {
      "epoch": 0.292355446473688,
      "grad_norm": 11.614291076457404,
      "learning_rate": 4.457521117302392e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.40625,
      "logps/chosen": -636.0,
      "logps/rejected": -956.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.125,
      "step": 4050
    },
    {
      "epoch": 0.29307731177362306,
      "grad_norm": 11.959955939062517,
      "learning_rate": 4.45359635048568e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.59375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.1875,
      "rewards/rejected": -8.375,
      "step": 4060
    },
    {
      "epoch": 0.29379917707355807,
      "grad_norm": 10.625522954210599,
      "learning_rate": 4.4496591782786883e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.609375,
      "logps/chosen": -780.0,
      "logps/rejected": -1168.0,
      "loss": 0.2562,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -6.25,
      "rewards/margins": 3.890625,
      "rewards/rejected": -10.125,
      "step": 4070
    },
    {
      "epoch": 0.2945210423734931,
      "grad_norm": 8.706370989411305,
      "learning_rate": 4.44570962568257e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.6875,
      "logps/chosen": -736.0,
      "logps/rejected": -1136.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.75,
      "rewards/margins": 4.1875,
      "rewards/rejected": -9.9375,
      "step": 4080
    },
    {
      "epoch": 0.29524290767342815,
      "grad_norm": 10.084781546011934,
      "learning_rate": 4.4417477177770905e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -704.0,
      "logps/rejected": -1056.0,
      "loss": 0.2059,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.609375,
      "rewards/rejected": -9.125,
      "step": 4090
    },
    {
      "epoch": 0.29596477297336315,
      "grad_norm": 10.211730254994997,
      "learning_rate": 4.4377734797204747e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1056.0,
      "loss": 0.1966,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.640625,
      "rewards/rejected": -9.125,
      "step": 4100
    },
    {
      "epoch": 0.2966866382732982,
      "grad_norm": 8.821514902632222,
      "learning_rate": 4.433786936749241e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.46875,
      "logps/chosen": -744.0,
      "logps/rejected": -1080.0,
      "loss": 0.2048,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.3125,
      "step": 4110
    },
    {
      "epoch": 0.29740850357323323,
      "grad_norm": 9.210606252468882,
      "learning_rate": 4.429788114178049e-07,
      "logits/chosen": -2.65625,
      "logits/rejected": -2.421875,
      "logps/chosen": -744.0,
      "logps/rejected": -1072.0,
      "loss": 0.2042,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.40625,
      "rewards/rejected": -9.25,
      "step": 4120
    },
    {
      "epoch": 0.2981303688731683,
      "grad_norm": 8.00555161785549,
      "learning_rate": 4.425777037399529e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.40625,
      "logps/chosen": -744.0,
      "logps/rejected": -1064.0,
      "loss": 0.1886,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.75,
      "rewards/margins": 3.40625,
      "rewards/rejected": -9.1875,
      "step": 4130
    },
    {
      "epoch": 0.2988522341731033,
      "grad_norm": 11.207346805964308,
      "learning_rate": 4.42175373188413e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.375,
      "logps/chosen": -780.0,
      "logps/rejected": -1080.0,
      "loss": 0.2083,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -6.1875,
      "rewards/margins": 3.25,
      "rewards/rejected": -9.4375,
      "step": 4140
    },
    {
      "epoch": 0.2995740994730383,
      "grad_norm": 8.172701676742552,
      "learning_rate": 4.4177182231799513e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.34375,
      "logps/chosen": -716.0,
      "logps/rejected": -1080.0,
      "loss": 0.2072,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.125,
      "step": 4150
    },
    {
      "epoch": 0.3002959647729734,
      "grad_norm": 5.801373290887345,
      "learning_rate": 4.413670536912584e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.46875,
      "logps/chosen": -728.0,
      "logps/rejected": -1048.0,
      "loss": 0.2031,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.40625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -8.875,
      "step": 4160
    },
    {
      "epoch": 0.3010178300729084,
      "grad_norm": 6.307391224443462,
      "learning_rate": 4.4096106987849457e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.515625,
      "logps/chosen": -732.0,
      "logps/rejected": -1120.0,
      "loss": 0.2027,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.96875,
      "rewards/rejected": -9.625,
      "step": 4170
    },
    {
      "epoch": 0.30173969537284345,
      "grad_norm": 11.867957943680011,
      "learning_rate": 4.405538734577121e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.5,
      "logps/chosen": -744.0,
      "logps/rejected": -1136.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.84375,
      "rewards/margins": 3.90625,
      "rewards/rejected": -9.75,
      "step": 4180
    },
    {
      "epoch": 0.30246156067277846,
      "grad_norm": 9.987667285927838,
      "learning_rate": 4.401454670146193e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.71875,
      "logps/chosen": -712.0,
      "logps/rejected": -1080.0,
      "loss": 0.2163,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.25,
      "step": 4190
    },
    {
      "epoch": 0.30318342597271347,
      "grad_norm": 8.977809465407699,
      "learning_rate": 4.3973585314260836e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.53125,
      "logps/chosen": -668.0,
      "logps/rejected": -1048.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.15625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.8125,
      "step": 4200
    },
    {
      "epoch": 0.30390529127264854,
      "grad_norm": 10.825945402642533,
      "learning_rate": 4.393250344427385e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.515625,
      "logps/chosen": -736.0,
      "logps/rejected": -1056.0,
      "loss": 0.2047,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.8125,
      "rewards/margins": 3.34375,
      "rewards/rejected": -9.125,
      "step": 4210
    },
    {
      "epoch": 0.30462715657258355,
      "grad_norm": 8.126172608631684,
      "learning_rate": 4.389130135237196e-07,
      "logits/chosen": -2.890625,
      "logits/rejected": -2.5625,
      "logps/chosen": -692.0,
      "logps/rejected": -1088.0,
      "loss": 0.2207,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.4375,
      "step": 4220
    },
    {
      "epoch": 0.3053490218725186,
      "grad_norm": 7.626491220106016,
      "learning_rate": 4.38499793001896e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.484375,
      "logps/chosen": -672.0,
      "logps/rejected": -992.0,
      "loss": 0.2096,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.375,
      "rewards/rejected": -8.4375,
      "step": 4230
    },
    {
      "epoch": 0.3060708871724536,
      "grad_norm": 9.904614258788264,
      "learning_rate": 4.3808537550122913e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.296875,
      "logps/chosen": -656.0,
      "logps/rejected": -948.0,
      "loss": 0.211,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.109375,
      "rewards/rejected": -8.0625,
      "step": 4240
    },
    {
      "epoch": 0.30679275247238863,
      "grad_norm": 9.163789602486757,
      "learning_rate": 4.376697636532816e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.421875,
      "logps/chosen": -668.0,
      "logps/rejected": -1040.0,
      "loss": 0.1866,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -8.875,
      "step": 4250
    },
    {
      "epoch": 0.3075146177723237,
      "grad_norm": 9.658094123529755,
      "learning_rate": 4.3725296009719985e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.5625,
      "logps/chosen": -672.0,
      "logps/rejected": -1064.0,
      "loss": 0.1865,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.984375,
      "rewards/rejected": -9.125,
      "step": 4260
    },
    {
      "epoch": 0.3082364830722587,
      "grad_norm": 6.663743741837934,
      "learning_rate": 4.3683496747969804e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -668.0,
      "logps/rejected": -1040.0,
      "loss": 0.1913,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.84375,
      "rewards/rejected": -8.9375,
      "step": 4270
    },
    {
      "epoch": 0.30895834837219377,
      "grad_norm": 9.431783701004399,
      "learning_rate": 4.364157884550406e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.609375,
      "logps/chosen": -744.0,
      "logps/rejected": -1120.0,
      "loss": 0.2088,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.875,
      "rewards/rejected": -9.625,
      "step": 4280
    },
    {
      "epoch": 0.3096802136721288,
      "grad_norm": 10.281531783875531,
      "learning_rate": 4.359954256850259e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -692.0,
      "logps/rejected": -1064.0,
      "loss": 0.2274,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.46875,
      "rewards/margins": 3.703125,
      "rewards/rejected": -9.1875,
      "step": 4290
    },
    {
      "epoch": 0.3104020789720638,
      "grad_norm": 9.63227220127926,
      "learning_rate": 4.35573881838969e-07,
      "logits/chosen": -2.6875,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1032.0,
      "loss": 0.2038,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.71875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.9375,
      "step": 4300
    },
    {
      "epoch": 0.31112394427199885,
      "grad_norm": 10.289092693608229,
      "learning_rate": 4.3515115959368476e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.53125,
      "logps/chosen": -716.0,
      "logps/rejected": -1016.0,
      "loss": 0.2013,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.25,
      "rewards/rejected": -8.6875,
      "step": 4310
    },
    {
      "epoch": 0.31184580957193386,
      "grad_norm": 10.974607276803276,
      "learning_rate": 4.3472726163347116e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5625,
      "logps/chosen": -728.0,
      "logps/rejected": -1048.0,
      "loss": 0.2179,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -5.65625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.125,
      "step": 4320
    },
    {
      "epoch": 0.31256767487186893,
      "grad_norm": 6.412992045987879,
      "learning_rate": 4.3430219065009177e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -700.0,
      "logps/rejected": -1040.0,
      "loss": 0.1975,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.4375,
      "rewards/rejected": -8.8125,
      "step": 4330
    },
    {
      "epoch": 0.31328954017180394,
      "grad_norm": 10.403220300532743,
      "learning_rate": 4.3387594934275896e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.59375,
      "logps/chosen": -764.0,
      "logps/rejected": -1112.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -6.0,
      "rewards/margins": 3.515625,
      "rewards/rejected": -9.5,
      "step": 4340
    },
    {
      "epoch": 0.31401140547173895,
      "grad_norm": 6.73518127809711,
      "learning_rate": 4.334485404181167e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -736.0,
      "logps/rejected": -1104.0,
      "loss": 0.186,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.78125,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.5,
      "step": 4350
    },
    {
      "epoch": 0.314733270771674,
      "grad_norm": 10.054893296825982,
      "learning_rate": 4.3301996659022334e-07,
      "logits/chosen": -2.625,
      "logits/rejected": -2.28125,
      "logps/chosen": -664.0,
      "logps/rejected": -1020.0,
      "loss": 0.2176,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.578125,
      "rewards/rejected": -8.5625,
      "step": 4360
    },
    {
      "epoch": 0.315455136071609,
      "grad_norm": 7.828814625819192,
      "learning_rate": 4.3259023058053444e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.328125,
      "logps/chosen": -724.0,
      "logps/rejected": -1040.0,
      "loss": 0.2147,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.234375,
      "rewards/rejected": -8.875,
      "step": 4370
    },
    {
      "epoch": 0.3161770013715441,
      "grad_norm": 7.009502435156667,
      "learning_rate": 4.3215933511788544e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.28125,
      "logps/chosen": -716.0,
      "logps/rejected": -1020.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.15625,
      "rewards/rejected": -8.875,
      "step": 4380
    },
    {
      "epoch": 0.3168988666714791,
      "grad_norm": 9.3284353021479,
      "learning_rate": 4.317272829384743e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.4375,
      "logps/chosen": -704.0,
      "logps/rejected": -1016.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.625,
      "step": 4390
    },
    {
      "epoch": 0.31762073197141416,
      "grad_norm": 8.811880524775491,
      "learning_rate": 4.3129407678584414e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.390625,
      "logps/chosen": -672.0,
      "logps/rejected": -1016.0,
      "loss": 0.2084,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.609375,
      "rewards/rejected": -8.75,
      "step": 4400
    },
    {
      "epoch": 0.31834259727134917,
      "grad_norm": 8.125525464636453,
      "learning_rate": 4.3085971941086585e-07,
      "logits/chosen": -2.578125,
      "logits/rejected": -2.390625,
      "logps/chosen": -728.0,
      "logps/rejected": -1032.0,
      "loss": 0.1915,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.6875,
      "rewards/margins": 3.125,
      "rewards/rejected": -8.8125,
      "step": 4410
    },
    {
      "epoch": 0.3190644625712842,
      "grad_norm": 8.751734979718838,
      "learning_rate": 4.3042421357172076e-07,
      "logits/chosen": -2.640625,
      "logits/rejected": -2.3125,
      "logps/chosen": -708.0,
      "logps/rejected": -1072.0,
      "loss": 0.1923,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.9375,
      "step": 4420
    },
    {
      "epoch": 0.31978632787121924,
      "grad_norm": 7.608014094617588,
      "learning_rate": 4.2998756203388285e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -684.0,
      "logps/rejected": -1032.0,
      "loss": 0.2152,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.0,
      "rewards/margins": 3.8125,
      "rewards/rejected": -8.8125,
      "step": 4430
    },
    {
      "epoch": 0.32050819317115425,
      "grad_norm": 7.639743064757877,
      "learning_rate": 4.2954976757010133e-07,
      "logits/chosen": -2.59375,
      "logits/rejected": -2.421875,
      "logps/chosen": -672.0,
      "logps/rejected": -1032.0,
      "loss": 0.1902,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.0625,
      "rewards/margins": 3.78125,
      "rewards/rejected": -8.875,
      "step": 4440
    },
    {
      "epoch": 0.3212300584710893,
      "grad_norm": 11.460888477745158,
      "learning_rate": 4.2911083296038285e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.546875,
      "logps/chosen": -688.0,
      "logps/rejected": -1056.0,
      "loss": 0.1824,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -9.0625,
      "step": 4450
    },
    {
      "epoch": 0.32195192377102433,
      "grad_norm": 9.224352773153688,
      "learning_rate": 4.2867076099197446e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.5,
      "logps/chosen": -668.0,
      "logps/rejected": -980.0,
      "loss": 0.1895,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.5,
      "rewards/rejected": -8.4375,
      "step": 4460
    },
    {
      "epoch": 0.32267378907095934,
      "grad_norm": 9.244876887068662,
      "learning_rate": 4.2822955445934505e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.5625,
      "logps/chosen": -732.0,
      "logps/rejected": -1080.0,
      "loss": 0.1772,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.765625,
      "rewards/rejected": -9.3125,
      "step": 4470
    },
    {
      "epoch": 0.3233956543708944,
      "grad_norm": 8.906087322512382,
      "learning_rate": 4.277872161641681e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.453125,
      "logps/chosen": -760.0,
      "logps/rejected": -1120.0,
      "loss": 0.1962,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.875,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.5625,
      "step": 4480
    },
    {
      "epoch": 0.3241175196708294,
      "grad_norm": 9.689414725700667,
      "learning_rate": 4.273437489153041e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.390625,
      "logps/chosen": -716.0,
      "logps/rejected": -1048.0,
      "loss": 0.2001,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.4375,
      "rewards/margins": 3.59375,
      "rewards/rejected": -9.0,
      "step": 4490
    },
    {
      "epoch": 0.3248393849707645,
      "grad_norm": 8.996613366246144,
      "learning_rate": 4.2689915552878207e-07,
      "logits/chosen": -2.609375,
      "logits/rejected": -2.4375,
      "logps/chosen": -688.0,
      "logps/rejected": -1008.0,
      "loss": 0.1829,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.390625,
      "rewards/rejected": -8.5,
      "step": 4500
    },
    {
      "epoch": 0.3255612502706995,
      "grad_norm": 9.607472232534843,
      "learning_rate": 4.2645343882778215e-07,
      "logits/chosen": -2.734375,
      "logits/rejected": -2.453125,
      "logps/chosen": -720.0,
      "logps/rejected": -1088.0,
      "loss": 0.2257,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.625,
      "rewards/margins": 3.828125,
      "rewards/rejected": -9.4375,
      "step": 4510
    },
    {
      "epoch": 0.3262831155706345,
      "grad_norm": 8.846494209878479,
      "learning_rate": 4.260066016426177e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -680.0,
      "logps/rejected": -1032.0,
      "loss": 0.1767,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.59375,
      "rewards/rejected": -8.8125,
      "step": 4520
    },
    {
      "epoch": 0.32700498087056956,
      "grad_norm": 11.245824423136213,
      "learning_rate": 4.25558646810717e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.59375,
      "logps/chosen": -688.0,
      "logps/rejected": -1020.0,
      "loss": 0.1856,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.25,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.6875,
      "step": 4530
    },
    {
      "epoch": 0.32772684617050457,
      "grad_norm": 8.60581993865266,
      "learning_rate": 4.251095771766055e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.5,
      "logps/chosen": -672.0,
      "logps/rejected": -1048.0,
      "loss": 0.177,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.75,
      "step": 4540
    },
    {
      "epoch": 0.32844871147043964,
      "grad_norm": 7.6110846227784625,
      "learning_rate": 4.246593955918877e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -716.0,
      "logps/rejected": -1048.0,
      "loss": 0.2188,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -9.0625,
      "step": 4550
    },
    {
      "epoch": 0.32917057677037465,
      "grad_norm": 7.571006600020749,
      "learning_rate": 4.2420810491522896e-07,
      "logits/chosen": -2.5625,
      "logits/rejected": -2.34375,
      "logps/chosen": -672.0,
      "logps/rejected": -988.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.171875,
      "rewards/rejected": -8.4375,
      "step": 4560
    },
    {
      "epoch": 0.32989244207030965,
      "grad_norm": 9.619717180276336,
      "learning_rate": 4.237557080123373e-07,
      "logits/chosen": -2.53125,
      "logits/rejected": -2.4375,
      "logps/chosen": -672.0,
      "logps/rejected": -956.0,
      "loss": 0.2037,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -5.09375,
      "rewards/margins": 3.140625,
      "rewards/rejected": -8.25,
      "step": 4570
    },
    {
      "epoch": 0.3306143073702447,
      "grad_norm": 9.813397994219041,
      "learning_rate": 4.2330220775594567e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.515625,
      "logps/chosen": -660.0,
      "logps/rejected": -972.0,
      "loss": 0.1992,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.328125,
      "rewards/rejected": -8.25,
      "step": 4580
    },
    {
      "epoch": 0.33133617267017973,
      "grad_norm": 8.765749359430536,
      "learning_rate": 4.228476070257929e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.46875,
      "logps/chosen": -680.0,
      "logps/rejected": -992.0,
      "loss": 0.1884,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.203125,
      "rewards/rejected": -8.4375,
      "step": 4590
    },
    {
      "epoch": 0.3320580379701148,
      "grad_norm": 10.82246666876482,
      "learning_rate": 4.223919087086062e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.609375,
      "logps/chosen": -640.0,
      "logps/rejected": -1012.0,
      "loss": 0.1802,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.859375,
      "rewards/rejected": -8.75,
      "step": 4600
    },
    {
      "epoch": 0.3327799032700498,
      "grad_norm": 7.941514770204173,
      "learning_rate": 4.2193511569808225e-07,
      "logits/chosen": -2.859375,
      "logits/rejected": -2.609375,
      "logps/chosen": -616.0,
      "logps/rejected": -988.0,
      "loss": 0.1991,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.5625,
      "rewards/margins": 3.75,
      "rewards/rejected": -8.3125,
      "step": 4610
    },
    {
      "epoch": 0.3335017685699848,
      "grad_norm": 7.912745662185516,
      "learning_rate": 4.214772308948693e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.453125,
      "logps/chosen": -600.0,
      "logps/rejected": -1024.0,
      "loss": 0.2012,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -4.40625,
      "rewards/margins": 4.4375,
      "rewards/rejected": -8.8125,
      "step": 4620
    },
    {
      "epoch": 0.3342236338699199,
      "grad_norm": 10.9521651772932,
      "learning_rate": 4.2101825720654827e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.484375,
      "logps/chosen": -644.0,
      "logps/rejected": -936.0,
      "loss": 0.2081,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.046875,
      "rewards/rejected": -7.90625,
      "step": 4630
    },
    {
      "epoch": 0.3349454991698549,
      "grad_norm": 6.99023599526195,
      "learning_rate": 4.205581975476146e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.421875,
      "logps/chosen": -652.0,
      "logps/rejected": -988.0,
      "loss": 0.2041,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.9375,
      "rewards/margins": 3.453125,
      "rewards/rejected": -8.4375,
      "step": 4640
    },
    {
      "epoch": 0.33566736446978995,
      "grad_norm": 8.603344530140768,
      "learning_rate": 4.200970548394598e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.578125,
      "logps/chosen": -724.0,
      "logps/rejected": -1040.0,
      "loss": 0.1965,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.875,
      "step": 4650
    },
    {
      "epoch": 0.33638922976972496,
      "grad_norm": 9.717823679934542,
      "learning_rate": 4.196348320103527e-07,
      "logits/chosen": -2.78125,
      "logits/rejected": -2.609375,
      "logps/chosen": -656.0,
      "logps/rejected": -1024.0,
      "loss": 0.213,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.515625,
      "rewards/rejected": -8.625,
      "step": 4660
    },
    {
      "epoch": 0.33711109506966,
      "grad_norm": 7.935814772911749,
      "learning_rate": 4.191715319954209e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.53125,
      "logps/chosen": -644.0,
      "logps/rejected": -1008.0,
      "loss": 0.1937,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.625,
      "rewards/rejected": -8.5625,
      "step": 4670
    },
    {
      "epoch": 0.33783296036959504,
      "grad_norm": 8.769171773282734,
      "learning_rate": 4.187071577366321e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.6875,
      "logps/chosen": -684.0,
      "logps/rejected": -1008.0,
      "loss": 0.176,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.75,
      "step": 4680
    },
    {
      "epoch": 0.33855482566953005,
      "grad_norm": 7.096843320580391,
      "learning_rate": 4.182417121827755e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.546875,
      "logps/chosen": -676.0,
      "logps/rejected": -1040.0,
      "loss": 0.1906,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.875,
      "step": 4690
    },
    {
      "epoch": 0.3392766909694651,
      "grad_norm": 10.058671513695627,
      "learning_rate": 4.177751982894433e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.515625,
      "logps/chosen": -600.0,
      "logps/rejected": -916.0,
      "loss": 0.1903,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.34375,
      "rewards/margins": 3.5625,
      "rewards/rejected": -7.90625,
      "step": 4700
    },
    {
      "epoch": 0.3399985562694001,
      "grad_norm": 10.565296750226013,
      "learning_rate": 4.1730761901901135e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -684.0,
      "logps/rejected": -1064.0,
      "loss": 0.1993,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.890625,
      "rewards/rejected": -9.0,
      "step": 4710
    },
    {
      "epoch": 0.3407204215693352,
      "grad_norm": 9.850412180420557,
      "learning_rate": 4.168389773406209e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.5625,
      "logps/chosen": -696.0,
      "logps/rejected": -1032.0,
      "loss": 0.2055,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.875,
      "step": 4720
    },
    {
      "epoch": 0.3414422868692702,
      "grad_norm": 11.906676776121467,
      "learning_rate": 4.1636927623015927e-07,
      "logits/chosen": -2.96875,
      "logits/rejected": -2.578125,
      "logps/chosen": -656.0,
      "logps/rejected": -1016.0,
      "loss": 0.1838,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.6875,
      "step": 4730
    },
    {
      "epoch": 0.3421641521692052,
      "grad_norm": 10.205674489245993,
      "learning_rate": 4.158985186702415e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.4375,
      "logps/chosen": -628.0,
      "logps/rejected": -1056.0,
      "loss": 0.2092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.75,
      "rewards/margins": 4.125,
      "rewards/rejected": -8.875,
      "step": 4740
    },
    {
      "epoch": 0.34288601746914027,
      "grad_norm": 9.260984330253814,
      "learning_rate": 4.1542670765019104e-07,
      "logits/chosen": -2.765625,
      "logits/rejected": -2.5,
      "logps/chosen": -672.0,
      "logps/rejected": -1040.0,
      "loss": 0.1958,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.828125,
      "rewards/rejected": -8.9375,
      "step": 4750
    },
    {
      "epoch": 0.3436078827690753,
      "grad_norm": 8.949027397431555,
      "learning_rate": 4.149538461660206e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -672.0,
      "logps/rejected": -1004.0,
      "loss": 0.2089,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.296875,
      "rewards/rejected": -8.5625,
      "step": 4760
    },
    {
      "epoch": 0.34432974806901034,
      "grad_norm": 6.720014771416349,
      "learning_rate": 4.144799372204136e-07,
      "logits/chosen": -2.921875,
      "logits/rejected": -2.6875,
      "logps/chosen": -644.0,
      "logps/rejected": -956.0,
      "loss": 0.1826,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.21875,
      "rewards/rejected": -8.125,
      "step": 4770
    },
    {
      "epoch": 0.34505161336894535,
      "grad_norm": 8.959024936705582,
      "learning_rate": 4.140049838227049e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.625,
      "logps/chosen": -664.0,
      "logps/rejected": -1012.0,
      "loss": 0.1987,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.90625,
      "rewards/margins": 3.703125,
      "rewards/rejected": -8.5625,
      "step": 4780
    },
    {
      "epoch": 0.34577347866888036,
      "grad_norm": 10.372748362230645,
      "learning_rate": 4.135289889888615e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.65625,
      "logps/chosen": -728.0,
      "logps/rejected": -1088.0,
      "loss": 0.1637,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.78125,
      "rewards/rejected": -9.3125,
      "step": 4790
    },
    {
      "epoch": 0.34649534396881543,
      "grad_norm": 8.270570715124181,
      "learning_rate": 4.130519557414636e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.671875,
      "logps/chosen": -660.0,
      "logps/rejected": -1040.0,
      "loss": 0.2018,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.96875,
      "rewards/margins": 3.90625,
      "rewards/rejected": -8.875,
      "step": 4800
    },
    {
      "epoch": 0.34721720926875044,
      "grad_norm": 9.893879868420212,
      "learning_rate": 4.1257388710968533e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -648.0,
      "logps/rejected": -1008.0,
      "loss": 0.1925,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -8.625,
      "step": 4810
    },
    {
      "epoch": 0.3479390745686855,
      "grad_norm": 12.331591426723168,
      "learning_rate": 4.120947861292757e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.5625,
      "logps/chosen": -716.0,
      "logps/rejected": -1072.0,
      "loss": 0.1966,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -5.5,
      "rewards/margins": 3.671875,
      "rewards/rejected": -9.1875,
      "step": 4820
    },
    {
      "epoch": 0.3486609398686205,
      "grad_norm": 10.172376095851071,
      "learning_rate": 4.11614655842539e-07,
      "logits/chosen": -2.796875,
      "logits/rejected": -2.65625,
      "logps/chosen": -724.0,
      "logps/rejected": -1056.0,
      "loss": 0.2024,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.546875,
      "rewards/rejected": -8.9375,
      "step": 4830
    },
    {
      "epoch": 0.3493828051685555,
      "grad_norm": 11.310294528639917,
      "learning_rate": 4.111334992983156e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.4375,
      "logps/chosen": -640.0,
      "logps/rejected": -992.0,
      "loss": 0.1973,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.8125,
      "rewards/margins": 3.5625,
      "rewards/rejected": -8.375,
      "step": 4840
    },
    {
      "epoch": 0.3501046704684906,
      "grad_norm": 9.138376308040302,
      "learning_rate": 4.1065131955196294e-07,
      "logits/chosen": -2.75,
      "logits/rejected": -2.40625,
      "logps/chosen": -656.0,
      "logps/rejected": -1020.0,
      "loss": 0.1957,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.03125,
      "rewards/margins": 3.65625,
      "rewards/rejected": -8.6875,
      "step": 4850
    },
    {
      "epoch": 0.3508265357684256,
      "grad_norm": 10.948111955434124,
      "learning_rate": 4.1016811966533536e-07,
      "logits/chosen": -2.84375,
      "logits/rejected": -2.53125,
      "logps/chosen": -708.0,
      "logps/rejected": -1104.0,
      "loss": 0.1823,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.59375,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.5625,
      "step": 4860
    },
    {
      "epoch": 0.35154840106836066,
      "grad_norm": 6.8483776654650965,
      "learning_rate": 4.096839027067656e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.625,
      "logps/chosen": -644.0,
      "logps/rejected": -1016.0,
      "loss": 0.2069,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.84375,
      "rewards/margins": 3.796875,
      "rewards/rejected": -8.625,
      "step": 4870
    },
    {
      "epoch": 0.35227026636829567,
      "grad_norm": 7.262308675444548,
      "learning_rate": 4.0919867175104435e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.6875,
      "logps/chosen": -656.0,
      "logps/rejected": -972.0,
      "loss": 0.1921,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.875,
      "rewards/margins": 3.40625,
      "rewards/rejected": -8.25,
      "step": 4880
    },
    {
      "epoch": 0.35299213166823074,
      "grad_norm": 17.175486759838638,
      "learning_rate": 4.0871242987940155e-07,
      "logits/chosen": -2.875,
      "logits/rejected": -2.71875,
      "logps/chosen": -688.0,
      "logps/rejected": -1048.0,
      "loss": 0.1981,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -5.3125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.0625,
      "step": 4890
    },
    {
      "epoch": 0.35371399696816574,
      "grad_norm": 6.418913714694265,
      "learning_rate": 4.082251801794865e-07,
      "logits/chosen": -3.0,
      "logits/rejected": -2.71875,
      "logps/chosen": -684.0,
      "logps/rejected": -1048.0,
      "loss": 0.1888,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.125,
      "rewards/margins": 3.640625,
      "rewards/rejected": -8.75,
      "step": 4900
    },
    {
      "epoch": 0.35443586226810075,
      "grad_norm": 6.8125539564399515,
      "learning_rate": 4.0773692574534795e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.5625,
      "logps/chosen": -684.0,
      "logps/rejected": -1000.0,
      "loss": 0.1871,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -5.1875,
      "rewards/margins": 3.359375,
      "rewards/rejected": -8.5625,
      "step": 4910
    },
    {
      "epoch": 0.3551577275680358,
      "grad_norm": 7.814190613362822,
      "learning_rate": 4.072476696774151e-07,
      "logits/chosen": -2.8125,
      "logits/rejected": -2.546875,
      "logps/chosen": -720.0,
      "logps/rejected": -1112.0,
      "loss": 0.1852,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -5.40625,
      "rewards/margins": 4.03125,
      "rewards/rejected": -9.4375,
      "step": 4920
    },
    {
      "epoch": 0.35587959286797083,
      "grad_norm": 10.826610307359978,
      "learning_rate": 4.0675741508247715e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.59375,
      "logps/chosen": -680.0,
      "logps/rejected": -1144.0,
      "loss": 0.1675,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -5.3125,
      "rewards/margins": 4.78125,
      "rewards/rejected": -10.0625,
      "step": 4930
    },
    {
      "epoch": 0.3566014581679059,
      "grad_norm": 8.596687269801581,
      "learning_rate": 4.062661650736643e-07,
      "logits/chosen": -3.046875,
      "logits/rejected": -2.765625,
      "logps/chosen": -728.0,
      "logps/rejected": -1112.0,
      "loss": 0.1869,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.59375,
      "rewards/margins": 4.0,
      "rewards/rejected": -9.625,
      "step": 4940
    },
    {
      "epoch": 0.3573233234678409,
      "grad_norm": 6.416195679478135,
      "learning_rate": 4.0577392277042766e-07,
      "logits/chosen": -2.90625,
      "logits/rejected": -2.671875,
      "logps/chosen": -700.0,
      "logps/rejected": -1056.0,
      "loss": 0.1855,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.6875,
      "rewards/rejected": -9.0625,
      "step": 4950
    },
    {
      "epoch": 0.3580451887677759,
      "grad_norm": 8.918928643083534,
      "learning_rate": 4.0528069129851914e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.46875,
      "logps/chosen": -712.0,
      "logps/rejected": -1088.0,
      "loss": 0.1846,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.953125,
      "rewards/rejected": -9.25,
      "step": 4960
    },
    {
      "epoch": 0.358767054067711,
      "grad_norm": 8.47545192360217,
      "learning_rate": 4.0478647378997223e-07,
      "logits/chosen": -2.703125,
      "logits/rejected": -2.515625,
      "logps/chosen": -724.0,
      "logps/rejected": -1048.0,
      "loss": 0.2025,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.5625,
      "rewards/margins": 3.46875,
      "rewards/rejected": -9.0625,
      "step": 4970
    },
    {
      "epoch": 0.359488919367646,
      "grad_norm": 8.09646150016637,
      "learning_rate": 4.0429127338308154e-07,
      "logits/chosen": -2.828125,
      "logits/rejected": -2.53125,
      "logps/chosen": -700.0,
      "logps/rejected": -1024.0,
      "loss": 0.206,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.375,
      "rewards/margins": 3.421875,
      "rewards/rejected": -8.8125,
      "step": 4980
    },
    {
      "epoch": 0.36021078466758105,
      "grad_norm": 7.912242395511905,
      "learning_rate": 4.037950932223832e-07,
      "logits/chosen": -2.71875,
      "logits/rejected": -2.5,
      "logps/chosen": -676.0,
      "logps/rejected": -1072.0,
      "loss": 0.186,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -5.21875,
      "rewards/margins": 3.796875,
      "rewards/rejected": -9.0,
      "step": 4990
    },
    {
      "epoch": 0.36093264996751606,
      "grad_norm": 8.510565195808283,
      "learning_rate": 4.032979364586349e-07,
      "logits/chosen": -2.953125,
      "logits/rejected": -2.703125,
      "logps/chosen": -688.0,
      "logps/rejected": -1064.0,
      "loss": 0.2007,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -5.28125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -9.0,
      "step": 5000
    },
    {
      "epoch": 0.36093264996751606,
      "eval_logits/chosen": -2.90625,
      "eval_logits/rejected": -2.65625,
      "eval_logps/chosen": -700.0,
      "eval_logps/rejected": -1040.0,
      "eval_loss": 0.22352416813373566,
      "eval_rewards/accuracies": 0.9081980586051941,
      "eval_rewards/chosen": -5.3125,
      "eval_rewards/margins": 3.546875,
      "eval_rewards/rejected": -8.875,
      "eval_runtime": 3594.5422,
      "eval_samples_per_second": 27.403,
      "eval_steps_per_second": 0.428,
      "step": 5000
    }
  ],
  "logging_steps": 10,
  "max_steps": 13853,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 5000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}