{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 2130,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004694835680751174,
      "grad_norm": 1732.812788499595,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": -0.98046875,
      "logits/rejected": -0.716796875,
      "logps/chosen": -426.79998779296875,
      "logps/rejected": -579.7999877929688,
      "loss": 2.4572,
      "rewards/accuracies": 0.23203125596046448,
      "rewards/chosen": 0.10761718451976776,
      "rewards/margins": 0.2906738221645355,
      "rewards/rejected": -0.18359375,
      "step": 10
    },
    {
      "epoch": 0.009389671361502348,
      "grad_norm": 2189.848459876415,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.71875,
      "logps/chosen": -443.3999938964844,
      "logps/rejected": -568.2000122070312,
      "loss": 2.8891,
      "rewards/accuracies": 0.2835937440395355,
      "rewards/chosen": 0.07880859076976776,
      "rewards/margins": -0.03232421725988388,
      "rewards/rejected": 0.11240234225988388,
      "step": 20
    },
    {
      "epoch": 0.014084507042253521,
      "grad_norm": 2316.395358956773,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": -1.004296898841858,
      "logits/rejected": -0.6507812738418579,
      "logps/chosen": -478.6000061035156,
      "logps/rejected": -654.5999755859375,
      "loss": 3.2099,
      "rewards/accuracies": 0.3023437559604645,
      "rewards/chosen": 0.27983397245407104,
      "rewards/margins": 0.16367188096046448,
      "rewards/rejected": 0.11762695014476776,
      "step": 30
    },
    {
      "epoch": 0.018779342723004695,
      "grad_norm": 1994.3498644795864,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.662109375,
      "logps/chosen": -463.3999938964844,
      "logps/rejected": -659.2000122070312,
      "loss": 3.4081,
      "rewards/accuracies": 0.2789062559604645,
      "rewards/chosen": -0.40966796875,
      "rewards/margins": -0.17539063096046448,
      "rewards/rejected": -0.23398438096046448,
      "step": 40
    },
    {
      "epoch": 0.023474178403755867,
      "grad_norm": 2100.3884265211827,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": -0.9605468511581421,
      "logits/rejected": -0.6216796636581421,
      "logps/chosen": -456.0,
      "logps/rejected": -571.4000244140625,
      "loss": 3.039,
      "rewards/accuracies": 0.3101562559604645,
      "rewards/chosen": -0.14345702528953552,
      "rewards/margins": 0.5108398199081421,
      "rewards/rejected": -0.6509765386581421,
      "step": 50
    },
    {
      "epoch": 0.028169014084507043,
      "grad_norm": 4895.724655957692,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -467.6000061035156,
      "logps/rejected": -629.4000244140625,
      "loss": 2.8639,
      "rewards/accuracies": 0.30546873807907104,
      "rewards/chosen": 0.07888183742761612,
      "rewards/margins": 0.623291015625,
      "rewards/rejected": -0.5448242425918579,
      "step": 60
    },
    {
      "epoch": 0.03286384976525822,
      "grad_norm": 1924.0402161604616,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.729687511920929,
      "logps/chosen": -446.0,
      "logps/rejected": -589.2000122070312,
      "loss": 2.6353,
      "rewards/accuracies": 0.328125,
      "rewards/chosen": 0.681445300579071,
      "rewards/margins": 1.0952637195587158,
      "rewards/rejected": -0.4144531190395355,
      "step": 70
    },
    {
      "epoch": 0.03755868544600939,
      "grad_norm": 1654.257983271788,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": -1.0300781726837158,
      "logits/rejected": -0.6548827886581421,
      "logps/chosen": -449.20001220703125,
      "logps/rejected": -601.2000122070312,
      "loss": 2.8526,
      "rewards/accuracies": 0.3492187559604645,
      "rewards/chosen": 0.989453136920929,
      "rewards/margins": 0.992480456829071,
      "rewards/rejected": -0.0042724609375,
      "step": 80
    },
    {
      "epoch": 0.04225352112676056,
      "grad_norm": 1858.6922571513117,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": -0.918749988079071,
      "logits/rejected": -0.6089843511581421,
      "logps/chosen": -439.3999938964844,
      "logps/rejected": -557.2000122070312,
      "loss": 2.6932,
      "rewards/accuracies": 0.3695312440395355,
      "rewards/chosen": 0.588055431842804,
      "rewards/margins": 1.857421875,
      "rewards/rejected": -1.267187476158142,
      "step": 90
    },
    {
      "epoch": 0.046948356807511735,
      "grad_norm": 1983.0670986791934,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.7251952886581421,
      "logps/chosen": -467.20001220703125,
      "logps/rejected": -552.4000244140625,
      "loss": 3.1394,
      "rewards/accuracies": 0.40546876192092896,
      "rewards/chosen": 1.5703125,
      "rewards/margins": 1.939062476158142,
      "rewards/rejected": -0.36567384004592896,
      "step": 100
    },
    {
      "epoch": 0.051643192488262914,
      "grad_norm": 1788.9914050590635,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": -0.9906250238418579,
      "logits/rejected": -0.681640625,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -603.2000122070312,
      "loss": 2.6849,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 1.875,
      "rewards/margins": 3.637500047683716,
      "rewards/rejected": -1.760156273841858,
      "step": 110
    },
    {
      "epoch": 0.056338028169014086,
      "grad_norm": 2085.491796032139,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": -0.973828136920929,
      "logits/rejected": -0.743359386920929,
      "logps/chosen": -462.6000061035156,
      "logps/rejected": -604.4000244140625,
      "loss": 2.82,
      "rewards/accuracies": 0.46953123807907104,
      "rewards/chosen": 2.742968797683716,
      "rewards/margins": 4.34375,
      "rewards/rejected": -1.5949218273162842,
      "step": 120
    },
    {
      "epoch": 0.06103286384976526,
      "grad_norm": 2389.539687429994,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": -1.0265624523162842,
      "logits/rejected": -0.7275390625,
      "logps/chosen": -475.20001220703125,
      "logps/rejected": -625.0,
      "loss": 3.1287,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": 2.918750047683716,
      "rewards/margins": 4.279687404632568,
      "rewards/rejected": -1.357812523841858,
      "step": 130
    },
    {
      "epoch": 0.06572769953051644,
      "grad_norm": 2724.619040489629,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.7289062738418579,
      "logps/chosen": -454.0,
      "logps/rejected": -627.0,
      "loss": 3.0222,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 3.6109375953674316,
      "rewards/margins": 4.598437309265137,
      "rewards/rejected": -0.9835205078125,
      "step": 140
    },
    {
      "epoch": 0.07042253521126761,
      "grad_norm": 1596.5485634309448,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.6000000238418579,
      "logps/chosen": -459.6000061035156,
      "logps/rejected": -620.4000244140625,
      "loss": 2.9781,
      "rewards/accuracies": 0.49921876192092896,
      "rewards/chosen": 3.4195313453674316,
      "rewards/margins": 6.229687690734863,
      "rewards/rejected": -2.809033155441284,
      "step": 150
    },
    {
      "epoch": 0.07511737089201878,
      "grad_norm": 1629.0288907838165,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.6259765625,
      "logps/chosen": -472.20001220703125,
      "logps/rejected": -626.4000244140625,
      "loss": 2.7898,
      "rewards/accuracies": 0.5054687261581421,
      "rewards/chosen": 3.178906202316284,
      "rewards/margins": 7.040625095367432,
      "rewards/rejected": -3.864062547683716,
      "step": 160
    },
    {
      "epoch": 0.07981220657276995,
      "grad_norm": 1499.1269349286415,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.5874999761581421,
      "logps/chosen": -430.79998779296875,
      "logps/rejected": -604.2000122070312,
      "loss": 3.2504,
      "rewards/accuracies": 0.5335937738418579,
      "rewards/chosen": 3.887500047683716,
      "rewards/margins": 8.379687309265137,
      "rewards/rejected": -4.485009670257568,
      "step": 170
    },
    {
      "epoch": 0.08450704225352113,
      "grad_norm": 1400.802730785054,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": -1.044921875,
      "logits/rejected": -0.7027343511581421,
      "logps/chosen": -415.0,
      "logps/rejected": -590.2000122070312,
      "loss": 2.4878,
      "rewards/accuracies": 0.567187488079071,
      "rewards/chosen": 4.826562404632568,
      "rewards/margins": 9.446874618530273,
      "rewards/rejected": -4.6328125,
      "step": 180
    },
    {
      "epoch": 0.0892018779342723,
      "grad_norm": 1971.798487862077,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": -1.0246093273162842,
      "logits/rejected": -0.676953136920929,
      "logps/chosen": -453.3999938964844,
      "logps/rejected": -570.5999755859375,
      "loss": 3.142,
      "rewards/accuracies": 0.532031238079071,
      "rewards/chosen": 5.400000095367432,
      "rewards/margins": 8.268750190734863,
      "rewards/rejected": -2.8687500953674316,
      "step": 190
    },
    {
      "epoch": 0.09389671361502347,
      "grad_norm": 1675.1798094953417,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": -1.107812523841858,
      "logits/rejected": -0.72265625,
      "logps/chosen": -433.79998779296875,
      "logps/rejected": -588.5999755859375,
      "loss": 3.1102,
      "rewards/accuracies": 0.557812511920929,
      "rewards/chosen": 6.640625,
      "rewards/margins": 8.778124809265137,
      "rewards/rejected": -2.1351561546325684,
      "step": 200
    },
    {
      "epoch": 0.09859154929577464,
      "grad_norm": 1701.402851412957,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": -1.0011718273162842,
      "logits/rejected": -0.6546875238418579,
      "logps/chosen": -450.3999938964844,
      "logps/rejected": -621.5999755859375,
      "loss": 3.0892,
      "rewards/accuracies": 0.551562488079071,
      "rewards/chosen": 5.207421779632568,
      "rewards/margins": 9.418749809265137,
      "rewards/rejected": -4.210253715515137,
      "step": 210
    },
    {
      "epoch": 0.10328638497652583,
      "grad_norm": 1637.995688144809,
      "learning_rate": 4.984350547730829e-07,
      "logits/chosen": -0.96484375,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -467.6000061035156,
      "logps/rejected": -605.5999755859375,
      "loss": 3.5522,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 4.857031345367432,
      "rewards/margins": 10.15625,
      "rewards/rejected": -5.298437595367432,
      "step": 220
    },
    {
      "epoch": 0.107981220657277,
      "grad_norm": 1335.9118287415,
      "learning_rate": 4.958268127282212e-07,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.7427734136581421,
      "logps/chosen": -449.79998779296875,
      "logps/rejected": -620.2000122070312,
      "loss": 3.0907,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 4.057031154632568,
      "rewards/margins": 10.265625,
      "rewards/rejected": -6.206250190734863,
      "step": 230
    },
    {
      "epoch": 0.11267605633802817,
      "grad_norm": 1766.2349059749727,
      "learning_rate": 4.932185706833594e-07,
      "logits/chosen": -1.076562523841858,
      "logits/rejected": -0.751953125,
      "logps/chosen": -407.6000061035156,
      "logps/rejected": -509.6000061035156,
      "loss": 2.4439,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 4.246874809265137,
      "rewards/margins": 8.287500381469727,
      "rewards/rejected": -4.034765720367432,
      "step": 240
    },
    {
      "epoch": 0.11737089201877934,
      "grad_norm": 1519.2707975056328,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.6148437261581421,
      "logps/chosen": -453.6000061035156,
      "logps/rejected": -632.5999755859375,
      "loss": 2.8675,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 3.594531297683716,
      "rewards/margins": 12.071874618530273,
      "rewards/rejected": -8.464062690734863,
      "step": 250
    },
    {
      "epoch": 0.12206572769953052,
      "grad_norm": 1445.8083782962885,
      "learning_rate": 4.880020865936358e-07,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.712890625,
      "logps/chosen": -423.0,
      "logps/rejected": -579.7999877929688,
      "loss": 3.0069,
      "rewards/accuracies": 0.5648437738418579,
      "rewards/chosen": 3.411328077316284,
      "rewards/margins": 10.324999809265137,
      "rewards/rejected": -6.900000095367432,
      "step": 260
    },
    {
      "epoch": 0.1267605633802817,
      "grad_norm": 2219.7418850286153,
      "learning_rate": 4.853938445487741e-07,
      "logits/chosen": -1.0207030773162842,
      "logits/rejected": -0.693164050579071,
      "logps/chosen": -451.79998779296875,
      "logps/rejected": -612.4000244140625,
      "loss": 3.391,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 3.848437547683716,
      "rewards/margins": 11.453125,
      "rewards/rejected": -7.589062690734863,
      "step": 270
    },
    {
      "epoch": 0.13145539906103287,
      "grad_norm": 1541.919005126956,
      "learning_rate": 4.827856025039123e-07,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.647265613079071,
      "logps/chosen": -447.79998779296875,
      "logps/rejected": -608.2000122070312,
      "loss": 2.7165,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 5.690625190734863,
      "rewards/margins": 10.362500190734863,
      "rewards/rejected": -4.656640529632568,
      "step": 280
    },
    {
      "epoch": 0.13615023474178403,
      "grad_norm": 1447.1139330985304,
      "learning_rate": 4.801773604590506e-07,
      "logits/chosen": -1.0105469226837158,
      "logits/rejected": -0.6304687261581421,
      "logps/chosen": -491.6000061035156,
      "logps/rejected": -603.7999877929688,
      "loss": 3.5316,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": 3.457812547683716,
      "rewards/margins": 9.696874618530273,
      "rewards/rejected": -6.231249809265137,
      "step": 290
    },
    {
      "epoch": 0.14084507042253522,
      "grad_norm": 1431.9839712451135,
      "learning_rate": 4.775691184141888e-07,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.7035156488418579,
      "logps/chosen": -461.79998779296875,
      "logps/rejected": -603.0,
      "loss": 3.0017,
      "rewards/accuracies": 0.589062511920929,
      "rewards/chosen": 4.237500190734863,
      "rewards/margins": 12.324999809265137,
      "rewards/rejected": -8.084375381469727,
      "step": 300
    },
    {
      "epoch": 0.14553990610328638,
      "grad_norm": 2912.4289912514673,
      "learning_rate": 4.749608763693271e-07,
      "logits/chosen": -1.0421874523162842,
      "logits/rejected": -0.682812511920929,
      "logps/chosen": -444.79998779296875,
      "logps/rejected": -589.2000122070312,
      "loss": 3.029,
      "rewards/accuracies": 0.60546875,
      "rewards/chosen": 4.885937690734863,
      "rewards/margins": 12.278124809265137,
      "rewards/rejected": -7.390625,
      "step": 310
    },
    {
      "epoch": 0.15023474178403756,
      "grad_norm": 2113.849045564183,
      "learning_rate": 4.7235263432446533e-07,
      "logits/chosen": -0.987109363079071,
      "logits/rejected": -0.8089843988418579,
      "logps/chosen": -440.3999938964844,
      "logps/rejected": -577.5999755859375,
      "loss": 2.9097,
      "rewards/accuracies": 0.589062511920929,
      "rewards/chosen": 4.817187309265137,
      "rewards/margins": 11.481249809265137,
      "rewards/rejected": -6.6640625,
      "step": 320
    },
    {
      "epoch": 0.15492957746478872,
      "grad_norm": 1609.6653946235178,
      "learning_rate": 4.6974439227960353e-07,
      "logits/chosen": -0.987500011920929,
      "logits/rejected": -0.687304675579071,
      "logps/chosen": -463.0,
      "logps/rejected": -615.7999877929688,
      "loss": 2.9833,
      "rewards/accuracies": 0.59765625,
      "rewards/chosen": 4.842577934265137,
      "rewards/margins": 12.4375,
      "rewards/rejected": -7.59375,
      "step": 330
    },
    {
      "epoch": 0.1596244131455399,
      "grad_norm": 1719.569487609078,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.692578136920929,
      "logps/chosen": -432.3999938964844,
      "logps/rejected": -539.0,
      "loss": 2.5545,
      "rewards/accuracies": 0.590624988079071,
      "rewards/chosen": 6.540625095367432,
      "rewards/margins": 9.818750381469727,
      "rewards/rejected": -3.291015625,
      "step": 340
    },
    {
      "epoch": 0.1643192488262911,
      "grad_norm": 1429.8898089139454,
      "learning_rate": 4.6452790818988004e-07,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.6578124761581421,
      "logps/chosen": -469.79998779296875,
      "logps/rejected": -611.7999877929688,
      "loss": 2.8116,
      "rewards/accuracies": 0.604687511920929,
      "rewards/chosen": 6.118750095367432,
      "rewards/margins": 13.012499809265137,
      "rewards/rejected": -6.890625,
      "step": 350
    },
    {
      "epoch": 0.16901408450704225,
      "grad_norm": 1838.4002415104974,
      "learning_rate": 4.6191966614501824e-07,
      "logits/chosen": -1.051171898841858,
      "logits/rejected": -0.756640613079071,
      "logps/chosen": -438.20001220703125,
      "logps/rejected": -588.4000244140625,
      "loss": 3.106,
      "rewards/accuracies": 0.6039062738418579,
      "rewards/chosen": 4.824999809265137,
      "rewards/margins": 10.631250381469727,
      "rewards/rejected": -5.807812690734863,
      "step": 360
    },
    {
      "epoch": 0.17370892018779344,
      "grad_norm": 1531.538488735521,
      "learning_rate": 4.593114241001565e-07,
      "logits/chosen": -1.078515648841858,
      "logits/rejected": -0.7203124761581421,
      "logps/chosen": -471.79998779296875,
      "logps/rejected": -613.4000244140625,
      "loss": 2.9722,
      "rewards/accuracies": 0.6039062738418579,
      "rewards/chosen": 4.030859470367432,
      "rewards/margins": 11.815625190734863,
      "rewards/rejected": -7.771874904632568,
      "step": 370
    },
    {
      "epoch": 0.1784037558685446,
      "grad_norm": 1548.2728370476664,
      "learning_rate": 4.5670318205529474e-07,
      "logits/chosen": -1.017578125,
      "logits/rejected": -0.679882824420929,
      "logps/chosen": -450.6000061035156,
      "logps/rejected": -609.5999755859375,
      "loss": 2.7828,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": 3.721484422683716,
      "rewards/margins": 12.524999618530273,
      "rewards/rejected": -8.801562309265137,
      "step": 380
    },
    {
      "epoch": 0.18309859154929578,
      "grad_norm": 1652.455143483272,
      "learning_rate": 4.54094940010433e-07,
      "logits/chosen": -0.9683593511581421,
      "logits/rejected": -0.6177734136581421,
      "logps/chosen": -457.6000061035156,
      "logps/rejected": -625.4000244140625,
      "loss": 3.9014,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": 5.4921875,
      "rewards/margins": 12.893750190734863,
      "rewards/rejected": -7.403124809265137,
      "step": 390
    },
    {
      "epoch": 0.18779342723004694,
      "grad_norm": 1451.7621804019343,
      "learning_rate": 4.514866979655712e-07,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.7152343988418579,
      "logps/chosen": -443.79998779296875,
      "logps/rejected": -596.4000244140625,
      "loss": 2.3804,
      "rewards/accuracies": 0.602343738079071,
      "rewards/chosen": 6.915625095367432,
      "rewards/margins": 14.056249618530273,
      "rewards/rejected": -7.131249904632568,
      "step": 400
    },
    {
      "epoch": 0.19248826291079812,
      "grad_norm": 1568.5034559063981,
      "learning_rate": 4.4887845592070945e-07,
      "logits/chosen": -1.037500023841858,
      "logits/rejected": -0.660351574420929,
      "logps/chosen": -437.3999938964844,
      "logps/rejected": -628.0,
      "loss": 3.3889,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": 6.471875190734863,
      "rewards/margins": 12.931249618530273,
      "rewards/rejected": -6.440625190734863,
      "step": 410
    },
    {
      "epoch": 0.19718309859154928,
      "grad_norm": 1673.3104199884774,
      "learning_rate": 4.462702138758477e-07,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.673632800579071,
      "logps/chosen": -470.79998779296875,
      "logps/rejected": -588.2000122070312,
      "loss": 3.3694,
      "rewards/accuracies": 0.600781261920929,
      "rewards/chosen": 3.71875,
      "rewards/margins": 11.149999618530273,
      "rewards/rejected": -7.442187309265137,
      "step": 420
    },
    {
      "epoch": 0.20187793427230047,
      "grad_norm": 1133.045149329716,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": -1.043359398841858,
      "logits/rejected": -0.6998046636581421,
      "logps/chosen": -454.3999938964844,
      "logps/rejected": -642.5999755859375,
      "loss": 2.5409,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": 4.6015625,
      "rewards/margins": 14.024999618530273,
      "rewards/rejected": -9.415624618530273,
      "step": 430
    },
    {
      "epoch": 0.20657276995305165,
      "grad_norm": 1284.0266994793435,
      "learning_rate": 4.4105372978612415e-07,
      "logits/chosen": -0.9214843511581421,
      "logits/rejected": -0.650585949420929,
      "logps/chosen": -476.0,
      "logps/rejected": -622.7999877929688,
      "loss": 2.7295,
      "rewards/accuracies": 0.6195312738418579,
      "rewards/chosen": 4.609375,
      "rewards/margins": 13.393750190734863,
      "rewards/rejected": -8.787500381469727,
      "step": 440
    },
    {
      "epoch": 0.2112676056338028,
      "grad_norm": 2053.751193674556,
      "learning_rate": 4.384454877412624e-07,
      "logits/chosen": -0.918749988079071,
      "logits/rejected": -0.563281238079071,
      "logps/chosen": -464.20001220703125,
      "logps/rejected": -711.4000244140625,
      "loss": 3.6895,
      "rewards/accuracies": 0.585156261920929,
      "rewards/chosen": 5.667187690734863,
      "rewards/margins": 13.75,
      "rewards/rejected": -8.09375,
      "step": 450
    },
    {
      "epoch": 0.215962441314554,
      "grad_norm": 1534.4421813142237,
      "learning_rate": 4.358372456964006e-07,
      "logits/chosen": -1.047265648841858,
      "logits/rejected": -0.680468738079071,
      "logps/chosen": -446.79998779296875,
      "logps/rejected": -610.7999877929688,
      "loss": 3.0695,
      "rewards/accuracies": 0.610156238079071,
      "rewards/chosen": 5.859375,
      "rewards/margins": 13.862500190734863,
      "rewards/rejected": -8.0,
      "step": 460
    },
    {
      "epoch": 0.22065727699530516,
      "grad_norm": 2848.1181608578786,
      "learning_rate": 4.3322900365153886e-07,
      "logits/chosen": -0.948437511920929,
      "logits/rejected": -0.687695324420929,
      "logps/chosen": -500.20001220703125,
      "logps/rejected": -604.4000244140625,
      "loss": 3.7691,
      "rewards/accuracies": 0.59765625,
      "rewards/chosen": 3.9632811546325684,
      "rewards/margins": 12.774999618530273,
      "rewards/rejected": -8.818750381469727,
      "step": 470
    },
    {
      "epoch": 0.22535211267605634,
      "grad_norm": 1415.1373959570624,
      "learning_rate": 4.306207616066771e-07,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.674023449420929,
      "logps/chosen": -439.20001220703125,
      "logps/rejected": -574.2000122070312,
      "loss": 3.0836,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": 4.996874809265137,
      "rewards/margins": 11.359375,
      "rewards/rejected": -6.365624904632568,
      "step": 480
    },
    {
      "epoch": 0.2300469483568075,
      "grad_norm": 1887.5717086772956,
      "learning_rate": 4.280125195618153e-07,
      "logits/chosen": -1.007421851158142,
      "logits/rejected": -0.6929687261581421,
      "logps/chosen": -447.3999938964844,
      "logps/rejected": -621.5999755859375,
      "loss": 3.1346,
      "rewards/accuracies": 0.6039062738418579,
      "rewards/chosen": 5.9140625,
      "rewards/margins": 13.420312881469727,
      "rewards/rejected": -7.511328220367432,
      "step": 490
    },
    {
      "epoch": 0.2347417840375587,
      "grad_norm": 2614.2313567047295,
      "learning_rate": 4.2540427751695357e-07,
      "logits/chosen": -1.017968773841858,
      "logits/rejected": -0.680859386920929,
      "logps/chosen": -456.6000061035156,
      "logps/rejected": -572.4000244140625,
      "loss": 3.4121,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 5.3984375,
      "rewards/margins": 11.534375190734863,
      "rewards/rejected": -6.128125190734863,
      "step": 500
    },
    {
      "epoch": 0.23943661971830985,
      "grad_norm": 1866.3074520020639,
      "learning_rate": 4.227960354720918e-07,
      "logits/chosen": -1.002343773841858,
      "logits/rejected": -0.6451171636581421,
      "logps/chosen": -442.6000061035156,
      "logps/rejected": -633.2000122070312,
      "loss": 2.7765,
      "rewards/accuracies": 0.594531238079071,
      "rewards/chosen": 5.426562309265137,
      "rewards/margins": 13.440625190734863,
      "rewards/rejected": -7.996874809265137,
      "step": 510
    },
    {
      "epoch": 0.24413145539906103,
      "grad_norm": 1385.6980655734326,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": -1.0496094226837158,
      "logits/rejected": -0.678906261920929,
      "logps/chosen": -461.0,
      "logps/rejected": -575.2000122070312,
      "loss": 3.4523,
      "rewards/accuracies": 0.6148437261581421,
      "rewards/chosen": 3.6292967796325684,
      "rewards/margins": 11.662500381469727,
      "rewards/rejected": -8.03125,
      "step": 520
    },
    {
      "epoch": 0.24882629107981222,
      "grad_norm": 1415.30011378921,
      "learning_rate": 4.1757955138236827e-07,
      "logits/chosen": -0.9769531488418579,
      "logits/rejected": -0.666796863079071,
      "logps/chosen": -444.0,
      "logps/rejected": -601.4000244140625,
      "loss": 2.6951,
      "rewards/accuracies": 0.6421874761581421,
      "rewards/chosen": 3.716796875,
      "rewards/margins": 13.787500381469727,
      "rewards/rejected": -10.059374809265137,
      "step": 530
    },
    {
      "epoch": 0.2535211267605634,
      "grad_norm": 1284.3346161326745,
      "learning_rate": 4.149713093375065e-07,
      "logits/chosen": -1.003515601158142,
      "logits/rejected": -0.715624988079071,
      "logps/chosen": -446.6000061035156,
      "logps/rejected": -562.4000244140625,
      "loss": 3.2813,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 4.331250190734863,
      "rewards/margins": 12.506250381469727,
      "rewards/rejected": -8.165624618530273,
      "step": 540
    },
    {
      "epoch": 0.25821596244131456,
      "grad_norm": 1672.9198056424855,
      "learning_rate": 4.123630672926447e-07,
      "logits/chosen": -1.034765601158142,
      "logits/rejected": -0.727343738079071,
      "logps/chosen": -430.0,
      "logps/rejected": -600.0,
      "loss": 2.8185,
      "rewards/accuracies": 0.62109375,
      "rewards/chosen": 5.142187595367432,
      "rewards/margins": 14.100000381469727,
      "rewards/rejected": -8.956250190734863,
      "step": 550
    },
    {
      "epoch": 0.26291079812206575,
      "grad_norm": 1497.7136354353509,
      "learning_rate": 4.09754825247783e-07,
      "logits/chosen": -0.9457031488418579,
      "logits/rejected": -0.675976574420929,
      "logps/chosen": -454.20001220703125,
      "logps/rejected": -590.5999755859375,
      "loss": 3.0893,
      "rewards/accuracies": 0.59765625,
      "rewards/chosen": 4.354101657867432,
      "rewards/margins": 13.193750381469727,
      "rewards/rejected": -8.84375,
      "step": 560
    },
    {
      "epoch": 0.2676056338028169,
      "grad_norm": 1235.2506438228415,
      "learning_rate": 4.0714658320292123e-07,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.6324218511581421,
      "logps/chosen": -453.0,
      "logps/rejected": -605.7999877929688,
      "loss": 2.5798,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 5.415625095367432,
      "rewards/margins": 16.193750381469727,
      "rewards/rejected": -10.774999618530273,
      "step": 570
    },
    {
      "epoch": 0.27230046948356806,
      "grad_norm": 1389.6657313063552,
      "learning_rate": 4.045383411580595e-07,
      "logits/chosen": -1.052343726158142,
      "logits/rejected": -0.656054675579071,
      "logps/chosen": -435.3999938964844,
      "logps/rejected": -598.4000244140625,
      "loss": 3.0205,
      "rewards/accuracies": 0.614062488079071,
      "rewards/chosen": 4.822461128234863,
      "rewards/margins": 14.715624809265137,
      "rewards/rejected": -9.90625,
      "step": 580
    },
    {
      "epoch": 0.27699530516431925,
      "grad_norm": 1438.3557402583242,
      "learning_rate": 4.019300991131977e-07,
      "logits/chosen": -1.028906226158142,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -437.79998779296875,
      "logps/rejected": -588.4000244140625,
      "loss": 2.9278,
      "rewards/accuracies": 0.6273437738418579,
      "rewards/chosen": 5.518750190734863,
      "rewards/margins": 14.774999618530273,
      "rewards/rejected": -9.251562118530273,
      "step": 590
    },
    {
      "epoch": 0.28169014084507044,
      "grad_norm": 1544.6312864069682,
      "learning_rate": 3.9932185706833594e-07,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.686718761920929,
      "logps/chosen": -460.3999938964844,
      "logps/rejected": -598.0,
      "loss": 2.6421,
      "rewards/accuracies": 0.617968738079071,
      "rewards/chosen": 5.451562404632568,
      "rewards/margins": 14.306249618530273,
      "rewards/rejected": -8.840624809265137,
      "step": 600
    },
    {
      "epoch": 0.2863849765258216,
      "grad_norm": 3074.1636988372115,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": -1.0320312976837158,
      "logits/rejected": -0.7250000238418579,
      "logps/chosen": -453.3999938964844,
      "logps/rejected": -593.0,
      "loss": 2.8659,
      "rewards/accuracies": 0.6148437261581421,
      "rewards/chosen": 5.188281059265137,
      "rewards/margins": 13.387499809265137,
      "rewards/rejected": -8.190625190734863,
      "step": 610
    },
    {
      "epoch": 0.29107981220657275,
      "grad_norm": 1431.0622969718042,
      "learning_rate": 3.941053729786124e-07,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.7359374761581421,
      "logps/chosen": -456.6000061035156,
      "logps/rejected": -607.0,
      "loss": 3.0604,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": 4.339062690734863,
      "rewards/margins": 15.100000381469727,
      "rewards/rejected": -10.753125190734863,
      "step": 620
    },
    {
      "epoch": 0.29577464788732394,
      "grad_norm": 1673.5605634503288,
      "learning_rate": 3.9149713093375064e-07,
      "logits/chosen": -1.027734398841858,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -454.0,
      "logps/rejected": -626.4000244140625,
      "loss": 2.625,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 6.378125190734863,
      "rewards/margins": 15.331250190734863,
      "rewards/rejected": -8.965624809265137,
      "step": 630
    },
    {
      "epoch": 0.3004694835680751,
      "grad_norm": 1315.9520542915832,
      "learning_rate": 3.888888888888889e-07,
      "logits/chosen": -1.0675780773162842,
      "logits/rejected": -0.673046886920929,
      "logps/chosen": -438.0,
      "logps/rejected": -637.5999755859375,
      "loss": 2.6736,
      "rewards/accuracies": 0.624218761920929,
      "rewards/chosen": 5.610937595367432,
      "rewards/margins": 16.168750762939453,
      "rewards/rejected": -10.574999809265137,
      "step": 640
    },
    {
      "epoch": 0.3051643192488263,
      "grad_norm": 1644.1234877855318,
      "learning_rate": 3.862806468440271e-07,
      "logits/chosen": -0.9691406488418579,
      "logits/rejected": -0.6246093511581421,
      "logps/chosen": -419.20001220703125,
      "logps/rejected": -559.7999877929688,
      "loss": 2.8067,
      "rewards/accuracies": 0.620312511920929,
      "rewards/chosen": 4.739062309265137,
      "rewards/margins": 13.125,
      "rewards/rejected": -8.393750190734863,
      "step": 650
    },
    {
      "epoch": 0.30985915492957744,
      "grad_norm": 1349.990147567295,
      "learning_rate": 3.8367240479916535e-07,
      "logits/chosen": -1.016015648841858,
      "logits/rejected": -0.7064453363418579,
      "logps/chosen": -439.0,
      "logps/rejected": -564.0,
      "loss": 2.9602,
      "rewards/accuracies": 0.614062488079071,
      "rewards/chosen": 5.306250095367432,
      "rewards/margins": 13.489062309265137,
      "rewards/rejected": -8.17822265625,
      "step": 660
    },
    {
      "epoch": 0.3145539906103286,
      "grad_norm": 1804.9583395315383,
      "learning_rate": 3.810641627543036e-07,
      "logits/chosen": -1.026953101158142,
      "logits/rejected": -0.690234363079071,
      "logps/chosen": -452.0,
      "logps/rejected": -604.7999877929688,
      "loss": 2.5166,
      "rewards/accuracies": 0.625781238079071,
      "rewards/chosen": 4.564062595367432,
      "rewards/margins": 15.731249809265137,
      "rewards/rejected": -11.178125381469727,
      "step": 670
    },
    {
      "epoch": 0.3192488262910798,
      "grad_norm": 1816.9821132395844,
      "learning_rate": 3.784559207094418e-07,
      "logits/chosen": -0.966015636920929,
      "logits/rejected": -0.6986328363418579,
      "logps/chosen": -470.6000061035156,
      "logps/rejected": -575.2000122070312,
      "loss": 3.5462,
      "rewards/accuracies": 0.58203125,
      "rewards/chosen": 3.1732420921325684,
      "rewards/margins": 11.403124809265137,
      "rewards/rejected": -8.234375,
      "step": 680
    },
    {
      "epoch": 0.323943661971831,
      "grad_norm": 1305.629141426007,
      "learning_rate": 3.7584767866458005e-07,
      "logits/chosen": -0.9808593988418579,
      "logits/rejected": -0.684765636920929,
      "logps/chosen": -461.0,
      "logps/rejected": -568.7999877929688,
      "loss": 3.4603,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 4.172656059265137,
      "rewards/margins": 12.268750190734863,
      "rewards/rejected": -8.084375381469727,
      "step": 690
    },
    {
      "epoch": 0.3286384976525822,
      "grad_norm": 2063.5958706893775,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": -0.964062511920929,
      "logits/rejected": -0.703906238079071,
      "logps/chosen": -486.6000061035156,
      "logps/rejected": -575.5999755859375,
      "loss": 3.8185,
      "rewards/accuracies": 0.616406261920929,
      "rewards/chosen": 3.426953077316284,
      "rewards/margins": 12.646875381469727,
      "rewards/rejected": -9.2265625,
      "step": 700
    },
    {
      "epoch": 0.3333333333333333,
      "grad_norm": 1950.4894177942947,
      "learning_rate": 3.706311945748565e-07,
      "logits/chosen": -0.9765625,
      "logits/rejected": -0.6953125,
      "logps/chosen": -500.3999938964844,
      "logps/rejected": -583.2000122070312,
      "loss": 3.6338,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 4.087500095367432,
      "rewards/margins": 12.046875,
      "rewards/rejected": -7.965624809265137,
      "step": 710
    },
    {
      "epoch": 0.3380281690140845,
      "grad_norm": 1590.6267482448668,
      "learning_rate": 3.6802295252999476e-07,
      "logits/chosen": -0.9750000238418579,
      "logits/rejected": -0.688671886920929,
      "logps/chosen": -469.6000061035156,
      "logps/rejected": -634.0,
      "loss": 2.9973,
      "rewards/accuracies": 0.6195312738418579,
      "rewards/chosen": 4.891406059265137,
      "rewards/margins": 15.300000190734863,
      "rewards/rejected": -10.399999618530273,
      "step": 720
    },
    {
      "epoch": 0.3427230046948357,
      "grad_norm": 1695.026940138068,
      "learning_rate": 3.65414710485133e-07,
      "logits/chosen": -1.017187476158142,
      "logits/rejected": -0.731640636920929,
      "logps/chosen": -463.79998779296875,
      "logps/rejected": -562.2000122070312,
      "loss": 3.1262,
      "rewards/accuracies": 0.624218761920929,
      "rewards/chosen": 3.669921875,
      "rewards/margins": 12.556249618530273,
      "rewards/rejected": -8.893750190734863,
      "step": 730
    },
    {
      "epoch": 0.3474178403755869,
      "grad_norm": 1296.9877484628305,
      "learning_rate": 3.6280646844027127e-07,
      "logits/chosen": -1.033203125,
      "logits/rejected": -0.6117187738418579,
      "logps/chosen": -441.3999938964844,
      "logps/rejected": -644.0,
      "loss": 2.6677,
      "rewards/accuracies": 0.6234375238418579,
      "rewards/chosen": 4.685937404632568,
      "rewards/margins": 15.34375,
      "rewards/rejected": -10.659375190734863,
      "step": 740
    },
    {
      "epoch": 0.352112676056338,
      "grad_norm": 1793.9342100547713,
      "learning_rate": 3.6019822639540947e-07,
      "logits/chosen": -1.0222656726837158,
      "logits/rejected": -0.7640625238418579,
      "logps/chosen": -474.79998779296875,
      "logps/rejected": -578.4000244140625,
      "loss": 3.0082,
      "rewards/accuracies": 0.6265624761581421,
      "rewards/chosen": 4.204687595367432,
      "rewards/margins": 14.443750381469727,
      "rewards/rejected": -10.237500190734863,
      "step": 750
    },
    {
      "epoch": 0.3568075117370892,
      "grad_norm": 1329.701083493045,
      "learning_rate": 3.575899843505477e-07,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.721484363079071,
      "logps/chosen": -451.0,
      "logps/rejected": -552.4000244140625,
      "loss": 3.3571,
      "rewards/accuracies": 0.6429687738418579,
      "rewards/chosen": 3.716796875,
      "rewards/margins": 13.321874618530273,
      "rewards/rejected": -9.606249809265137,
      "step": 760
    },
    {
      "epoch": 0.3615023474178404,
      "grad_norm": 2244.6770753983033,
      "learning_rate": 3.5498174230568597e-07,
      "logits/chosen": -1.01171875,
      "logits/rejected": -0.678515613079071,
      "logps/chosen": -445.20001220703125,
      "logps/rejected": -596.7999877929688,
      "loss": 2.7811,
      "rewards/accuracies": 0.633593738079071,
      "rewards/chosen": 4.6796875,
      "rewards/margins": 14.131250381469727,
      "rewards/rejected": -9.446874618530273,
      "step": 770
    },
    {
      "epoch": 0.36619718309859156,
      "grad_norm": 1952.6869341798013,
      "learning_rate": 3.5237350026082417e-07,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.7925781011581421,
      "logps/chosen": -457.0,
      "logps/rejected": -542.2000122070312,
      "loss": 3.1243,
      "rewards/accuracies": 0.600781261920929,
      "rewards/chosen": 3.8539061546325684,
      "rewards/margins": 11.556249618530273,
      "rewards/rejected": -7.690625190734863,
      "step": 780
    },
    {
      "epoch": 0.37089201877934275,
      "grad_norm": 1342.741466499115,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": -1.026953101158142,
      "logits/rejected": -0.665234386920929,
      "logps/chosen": -458.0,
      "logps/rejected": -613.2000122070312,
      "loss": 3.5876,
      "rewards/accuracies": 0.616406261920929,
      "rewards/chosen": 3.9312500953674316,
      "rewards/margins": 15.0625,
      "rewards/rejected": -11.137499809265137,
      "step": 790
    },
    {
      "epoch": 0.3755868544600939,
      "grad_norm": 2620.3254567243775,
      "learning_rate": 3.471570161711007e-07,
      "logits/chosen": -0.9312499761581421,
      "logits/rejected": -0.6439453363418579,
      "logps/chosen": -492.0,
      "logps/rejected": -619.5999755859375,
      "loss": 2.9573,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": 2.760058641433716,
      "rewards/margins": 14.037500381469727,
      "rewards/rejected": -11.268750190734863,
      "step": 800
    },
    {
      "epoch": 0.38028169014084506,
      "grad_norm": 1422.7764202284884,
      "learning_rate": 3.445487741262389e-07,
      "logits/chosen": -0.9847656488418579,
      "logits/rejected": -0.625781238079071,
      "logps/chosen": -441.79998779296875,
      "logps/rejected": -594.7999877929688,
      "loss": 2.6386,
      "rewards/accuracies": 0.5992187261581421,
      "rewards/chosen": 3.500781297683716,
      "rewards/margins": 15.112500190734863,
      "rewards/rejected": -11.621874809265137,
      "step": 810
    },
    {
      "epoch": 0.38497652582159625,
      "grad_norm": 1483.998095269015,
      "learning_rate": 3.4194053208137713e-07,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.6919921636581421,
      "logps/chosen": -454.0,
      "logps/rejected": -560.2000122070312,
      "loss": 2.5147,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 3.19921875,
      "rewards/margins": 13.6875,
      "rewards/rejected": -10.484375,
      "step": 820
    },
    {
      "epoch": 0.38967136150234744,
      "grad_norm": 1774.303048270542,
      "learning_rate": 3.393322900365154e-07,
      "logits/chosen": -1.03515625,
      "logits/rejected": -0.7630859613418579,
      "logps/chosen": -427.0,
      "logps/rejected": -568.0,
      "loss": 2.7163,
      "rewards/accuracies": 0.6148437261581421,
      "rewards/chosen": 2.9271483421325684,
      "rewards/margins": 13.518750190734863,
      "rewards/rejected": -10.596875190734863,
      "step": 830
    },
    {
      "epoch": 0.39436619718309857,
      "grad_norm": 1510.3371700857801,
      "learning_rate": 3.367240479916536e-07,
      "logits/chosen": -0.977734386920929,
      "logits/rejected": -0.685546875,
      "logps/chosen": -485.79998779296875,
      "logps/rejected": -609.5999755859375,
      "loss": 3.0495,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": 3.0003905296325684,
      "rewards/margins": 13.112500190734863,
      "rewards/rejected": -10.128125190734863,
      "step": 840
    },
    {
      "epoch": 0.39906103286384975,
      "grad_norm": 1263.2389792653637,
      "learning_rate": 3.3411580594679184e-07,
      "logits/chosen": -0.970703125,
      "logits/rejected": -0.613476574420929,
      "logps/chosen": -441.3999938964844,
      "logps/rejected": -594.0,
      "loss": 2.7586,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": 4.283203125,
      "rewards/margins": 15.134374618530273,
      "rewards/rejected": -10.831250190734863,
      "step": 850
    },
    {
      "epoch": 0.40375586854460094,
      "grad_norm": 1497.1784561712352,
      "learning_rate": 3.315075639019301e-07,
      "logits/chosen": -0.975781261920929,
      "logits/rejected": -0.5912109613418579,
      "logps/chosen": -434.6000061035156,
      "logps/rejected": -585.2000122070312,
      "loss": 2.5809,
      "rewards/accuracies": 0.63671875,
      "rewards/chosen": 4.454687595367432,
      "rewards/margins": 16.887500762939453,
      "rewards/rejected": -12.453125,
      "step": 860
    },
    {
      "epoch": 0.4084507042253521,
      "grad_norm": 2915.0676603137636,
      "learning_rate": 3.288993218570683e-07,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.7080078125,
      "logps/chosen": -443.0,
      "logps/rejected": -564.7999877929688,
      "loss": 3.217,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": 4.051562309265137,
      "rewards/margins": 12.524999618530273,
      "rewards/rejected": -8.4609375,
      "step": 870
    },
    {
      "epoch": 0.4131455399061033,
      "grad_norm": 1725.3090251334486,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": -1.035546898841858,
      "logits/rejected": -0.7269531488418579,
      "logps/chosen": -464.79998779296875,
      "logps/rejected": -587.7999877929688,
      "loss": 2.8278,
      "rewards/accuracies": 0.647656261920929,
      "rewards/chosen": 3.7734375,
      "rewards/margins": 14.274999618530273,
      "rewards/rejected": -10.509374618530273,
      "step": 880
    },
    {
      "epoch": 0.41784037558685444,
      "grad_norm": 1595.0427796055112,
      "learning_rate": 3.236828377673448e-07,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -489.0,
      "logps/rejected": -579.4000244140625,
      "loss": 3.3522,
      "rewards/accuracies": 0.6109374761581421,
      "rewards/chosen": 4.025000095367432,
      "rewards/margins": 13.399999618530273,
      "rewards/rejected": -9.378125190734863,
      "step": 890
    },
    {
      "epoch": 0.4225352112676056,
      "grad_norm": 1312.3328295730682,
      "learning_rate": 3.2107459572248305e-07,
      "logits/chosen": -1.0363280773162842,
      "logits/rejected": -0.6714843511581421,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -591.5999755859375,
      "loss": 2.4306,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 4.309374809265137,
      "rewards/margins": 13.759374618530273,
      "rewards/rejected": -9.448437690734863,
      "step": 900
    },
    {
      "epoch": 0.4272300469483568,
      "grad_norm": 1559.9693696557133,
      "learning_rate": 3.1846635367762125e-07,
      "logits/chosen": -1.035546898841858,
      "logits/rejected": -0.671093761920929,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -620.5999755859375,
      "loss": 3.0994,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 3.4515624046325684,
      "rewards/margins": 14.699999809265137,
      "rewards/rejected": -11.231249809265137,
      "step": 910
    },
    {
      "epoch": 0.431924882629108,
      "grad_norm": 1793.8734731634725,
      "learning_rate": 3.158581116327595e-07,
      "logits/chosen": -1.0148437023162842,
      "logits/rejected": -0.5859375,
      "logps/chosen": -458.79998779296875,
      "logps/rejected": -652.7999877929688,
      "loss": 3.14,
      "rewards/accuracies": 0.610156238079071,
      "rewards/chosen": 1.9591796398162842,
      "rewards/margins": 14.262499809265137,
      "rewards/rejected": -12.293749809265137,
      "step": 920
    },
    {
      "epoch": 0.43661971830985913,
      "grad_norm": 1346.8879851119766,
      "learning_rate": 3.1324986958789775e-07,
      "logits/chosen": -1.0792968273162842,
      "logits/rejected": -0.7320312261581421,
      "logps/chosen": -431.0,
      "logps/rejected": -564.0,
      "loss": 2.7165,
      "rewards/accuracies": 0.602343738079071,
      "rewards/chosen": 2.3648438453674316,
      "rewards/margins": 13.356249809265137,
      "rewards/rejected": -11.006250381469727,
      "step": 930
    },
    {
      "epoch": 0.4413145539906103,
      "grad_norm": 1356.9239637204528,
      "learning_rate": 3.1064162754303595e-07,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.628710925579071,
      "logps/chosen": -462.0,
      "logps/rejected": -628.4000244140625,
      "loss": 3.3591,
      "rewards/accuracies": 0.625781238079071,
      "rewards/chosen": 2.882031202316284,
      "rewards/margins": 14.571874618530273,
      "rewards/rejected": -11.709375381469727,
      "step": 940
    },
    {
      "epoch": 0.4460093896713615,
      "grad_norm": 1325.7890003411142,
      "learning_rate": 3.080333854981742e-07,
      "logits/chosen": -1.015234351158142,
      "logits/rejected": -0.739453136920929,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -558.4000244140625,
      "loss": 3.0561,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 3.684375047683716,
      "rewards/margins": 12.743749618530273,
      "rewards/rejected": -9.050000190734863,
      "step": 950
    },
    {
      "epoch": 0.4507042253521127,
      "grad_norm": 1495.742644621953,
      "learning_rate": 3.0542514345331246e-07,
      "logits/chosen": -1.001562476158142,
      "logits/rejected": -0.673632800579071,
      "logps/chosen": -460.6000061035156,
      "logps/rejected": -650.5999755859375,
      "loss": 2.9824,
      "rewards/accuracies": 0.6039062738418579,
      "rewards/chosen": 3.9693846702575684,
      "rewards/margins": 13.199999809265137,
      "rewards/rejected": -9.221875190734863,
      "step": 960
    },
    {
      "epoch": 0.45539906103286387,
      "grad_norm": 1902.584132111207,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": -1.044531226158142,
      "logits/rejected": -0.705078125,
      "logps/chosen": -484.0,
      "logps/rejected": -607.2000122070312,
      "loss": 3.9384,
      "rewards/accuracies": 0.617968738079071,
      "rewards/chosen": 2.143749952316284,
      "rewards/margins": 13.418749809265137,
      "rewards/rejected": -11.268750190734863,
      "step": 970
    },
    {
      "epoch": 0.460093896713615,
      "grad_norm": 1403.6302650837213,
      "learning_rate": 3.002086593635889e-07,
      "logits/chosen": -0.9644531011581421,
      "logits/rejected": -0.592968761920929,
      "logps/chosen": -455.6000061035156,
      "logps/rejected": -624.7999877929688,
      "loss": 2.4933,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": 3.1195311546325684,
      "rewards/margins": 14.346875190734863,
      "rewards/rejected": -11.228124618530273,
      "step": 980
    },
    {
      "epoch": 0.4647887323943662,
      "grad_norm": 1354.4612983563952,
      "learning_rate": 2.9760041731872716e-07,
      "logits/chosen": -1.024999976158142,
      "logits/rejected": -0.7572265863418579,
      "logps/chosen": -415.6000061035156,
      "logps/rejected": -584.0,
      "loss": 2.5658,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 3.7046875953674316,
      "rewards/margins": 14.671875,
      "rewards/rejected": -10.971875190734863,
      "step": 990
    },
    {
      "epoch": 0.4694835680751174,
      "grad_norm": 1488.0816712659262,
      "learning_rate": 2.9499217527386536e-07,
      "logits/chosen": -1.058203101158142,
      "logits/rejected": -0.6939452886581421,
      "logps/chosen": -439.6000061035156,
      "logps/rejected": -598.0,
      "loss": 2.6582,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 3.640625,
      "rewards/margins": 14.518750190734863,
      "rewards/rejected": -10.887499809265137,
      "step": 1000
    },
    {
      "epoch": 0.47417840375586856,
      "grad_norm": 1367.672954502692,
      "learning_rate": 2.923839332290036e-07,
      "logits/chosen": -1.064453125,
      "logits/rejected": -0.76953125,
      "logps/chosen": -434.6000061035156,
      "logps/rejected": -620.2000122070312,
      "loss": 2.8738,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": 3.729687452316284,
      "rewards/margins": 14.649999618530273,
      "rewards/rejected": -10.899999618530273,
      "step": 1010
    },
    {
      "epoch": 0.4788732394366197,
      "grad_norm": 2208.544077833186,
      "learning_rate": 2.8977569118414187e-07,
      "logits/chosen": -0.969921886920929,
      "logits/rejected": -0.6480468511581421,
      "logps/chosen": -434.3999938964844,
      "logps/rejected": -585.0,
      "loss": 2.5691,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": 4.40625,
      "rewards/margins": 15.506250381469727,
      "rewards/rejected": -11.106249809265137,
      "step": 1020
    },
    {
      "epoch": 0.4835680751173709,
      "grad_norm": 1198.5630850380069,
      "learning_rate": 2.8716744913928007e-07,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.724414050579071,
      "logps/chosen": -450.20001220703125,
      "logps/rejected": -568.2000122070312,
      "loss": 2.9202,
      "rewards/accuracies": 0.62890625,
      "rewards/chosen": 4.7109375,
      "rewards/margins": 14.393750190734863,
      "rewards/rejected": -9.690625190734863,
      "step": 1030
    },
    {
      "epoch": 0.48826291079812206,
      "grad_norm": 1469.1206522106563,
      "learning_rate": 2.845592070944183e-07,
      "logits/chosen": -0.973437488079071,
      "logits/rejected": -0.588574230670929,
      "logps/chosen": -449.79998779296875,
      "logps/rejected": -623.0,
      "loss": 2.8358,
      "rewards/accuracies": 0.6195312738418579,
      "rewards/chosen": 3.859375,
      "rewards/margins": 14.193750381469727,
      "rewards/rejected": -10.318750381469727,
      "step": 1040
    },
    {
      "epoch": 0.49295774647887325,
      "grad_norm": 1325.6011890463594,
      "learning_rate": 2.819509650495566e-07,
      "logits/chosen": -0.947265625,
      "logits/rejected": -0.6724609136581421,
      "logps/chosen": -444.79998779296875,
      "logps/rejected": -577.0,
      "loss": 2.6462,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": 4.029687404632568,
      "rewards/margins": 14.234375,
      "rewards/rejected": -10.196874618530273,
      "step": 1050
    },
    {
      "epoch": 0.49765258215962443,
      "grad_norm": 1374.8367212268588,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.714062511920929,
      "logps/chosen": -468.20001220703125,
      "logps/rejected": -586.4000244140625,
      "loss": 2.5722,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": 5.545312404632568,
      "rewards/margins": 15.175000190734863,
      "rewards/rejected": -9.635937690734863,
      "step": 1060
    },
    {
      "epoch": 0.5023474178403756,
      "grad_norm": 1689.3885806267656,
      "learning_rate": 2.7673448095983303e-07,
      "logits/chosen": -0.9437500238418579,
      "logits/rejected": -0.670703113079071,
      "logps/chosen": -458.0,
      "logps/rejected": -585.4000244140625,
      "loss": 3.6289,
      "rewards/accuracies": 0.6039062738418579,
      "rewards/chosen": 4.215624809265137,
      "rewards/margins": 13.5,
      "rewards/rejected": -9.28125,
      "step": 1070
    },
    {
      "epoch": 0.5070422535211268,
      "grad_norm": 1454.1431380614254,
      "learning_rate": 2.741262389149713e-07,
      "logits/chosen": -0.9820312261581421,
      "logits/rejected": -0.6919921636581421,
      "logps/chosen": -458.20001220703125,
      "logps/rejected": -605.2000122070312,
      "loss": 2.8711,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": 3.616406202316284,
      "rewards/margins": 17.524999618530273,
      "rewards/rejected": -13.918749809265137,
      "step": 1080
    },
    {
      "epoch": 0.5117370892018779,
      "grad_norm": 2281.534447492974,
      "learning_rate": 2.7151799687010953e-07,
      "logits/chosen": -1.0515625476837158,
      "logits/rejected": -0.727343738079071,
      "logps/chosen": -466.79998779296875,
      "logps/rejected": -611.5999755859375,
      "loss": 3.5195,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": 3.663281202316284,
      "rewards/margins": 14.503125190734863,
      "rewards/rejected": -10.831250190734863,
      "step": 1090
    },
    {
      "epoch": 0.5164319248826291,
      "grad_norm": 1371.9463148321481,
      "learning_rate": 2.6890975482524773e-07,
      "logits/chosen": -0.955859363079071,
      "logits/rejected": -0.647265613079071,
      "logps/chosen": -475.79998779296875,
      "logps/rejected": -606.5999755859375,
      "loss": 2.4466,
      "rewards/accuracies": 0.65234375,
      "rewards/chosen": 4.282812595367432,
      "rewards/margins": 15.793749809265137,
      "rewards/rejected": -11.506250381469727,
      "step": 1100
    },
    {
      "epoch": 0.5211267605633803,
      "grad_norm": 1906.9475183567156,
      "learning_rate": 2.66301512780386e-07,
      "logits/chosen": -1.0066406726837158,
      "logits/rejected": -0.6861327886581421,
      "logps/chosen": -435.6000061035156,
      "logps/rejected": -592.0,
      "loss": 3.6893,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": 4.573437690734863,
      "rewards/margins": 14.199999809265137,
      "rewards/rejected": -9.631250381469727,
      "step": 1110
    },
    {
      "epoch": 0.5258215962441315,
      "grad_norm": 1088.0603011017422,
      "learning_rate": 2.6369327073552424e-07,
      "logits/chosen": -0.95703125,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -440.0,
      "logps/rejected": -562.5999755859375,
      "loss": 2.9019,
      "rewards/accuracies": 0.63671875,
      "rewards/chosen": 5.40625,
      "rewards/margins": 14.193750381469727,
      "rewards/rejected": -8.787500381469727,
      "step": 1120
    },
    {
      "epoch": 0.5305164319248826,
      "grad_norm": 1445.7363628276426,
      "learning_rate": 2.6108502869066244e-07,
      "logits/chosen": -1.029687523841858,
      "logits/rejected": -0.639453113079071,
      "logps/chosen": -470.20001220703125,
      "logps/rejected": -624.7999877929688,
      "loss": 3.4923,
      "rewards/accuracies": 0.60546875,
      "rewards/chosen": 4.036718845367432,
      "rewards/margins": 13.246874809265137,
      "rewards/rejected": -9.193750381469727,
      "step": 1130
    },
    {
      "epoch": 0.5352112676056338,
      "grad_norm": 27154.531509212466,
      "learning_rate": 2.584767866458007e-07,
      "logits/chosen": -1.0031249523162842,
      "logits/rejected": -0.6761718988418579,
      "logps/chosen": -447.79998779296875,
      "logps/rejected": -582.0,
      "loss": 2.7987,
      "rewards/accuracies": 0.629687488079071,
      "rewards/chosen": 4.41015625,
      "rewards/margins": 14.356249809265137,
      "rewards/rejected": -9.949999809265137,
      "step": 1140
    },
    {
      "epoch": 0.539906103286385,
      "grad_norm": 1566.4355468328017,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": -0.971484363079071,
      "logits/rejected": -0.6156250238418579,
      "logps/chosen": -464.79998779296875,
      "logps/rejected": -610.0,
      "loss": 3.1721,
      "rewards/accuracies": 0.6265624761581421,
      "rewards/chosen": 4.428124904632568,
      "rewards/margins": 14.649999618530273,
      "rewards/rejected": -10.21875,
      "step": 1150
    },
    {
      "epoch": 0.5446009389671361,
      "grad_norm": 1907.9786550666458,
      "learning_rate": 2.5326030255607715e-07,
      "logits/chosen": -0.997265636920929,
      "logits/rejected": -0.684765636920929,
      "logps/chosen": -460.3999938964844,
      "logps/rejected": -589.2000122070312,
      "loss": 3.1058,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 5.824999809265137,
      "rewards/margins": 14.53125,
      "rewards/rejected": -8.696093559265137,
      "step": 1160
    },
    {
      "epoch": 0.5492957746478874,
      "grad_norm": 1364.9608033631719,
      "learning_rate": 2.506520605112154e-07,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.675000011920929,
      "logps/chosen": -448.79998779296875,
      "logps/rejected": -594.0,
      "loss": 3.4646,
      "rewards/accuracies": 0.600781261920929,
      "rewards/chosen": 5.990624904632568,
      "rewards/margins": 13.512499809265137,
      "rewards/rejected": -7.517578125,
      "step": 1170
    },
    {
      "epoch": 0.5539906103286385,
      "grad_norm": 2475.0598058774235,
      "learning_rate": 2.4804381846635365e-07,
      "logits/chosen": -0.93359375,
      "logits/rejected": -0.6527343988418579,
      "logps/chosen": -478.20001220703125,
      "logps/rejected": -591.2000122070312,
      "loss": 3.1651,
      "rewards/accuracies": 0.6226562261581421,
      "rewards/chosen": 5.682812690734863,
      "rewards/margins": 14.243749618530273,
      "rewards/rejected": -8.556249618530273,
      "step": 1180
    },
    {
      "epoch": 0.5586854460093896,
      "grad_norm": 1495.5284378352708,
      "learning_rate": 2.454355764214919e-07,
      "logits/chosen": -1.001953125,
      "logits/rejected": -0.658007800579071,
      "logps/chosen": -439.0,
      "logps/rejected": -617.4000244140625,
      "loss": 2.9566,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 4.481249809265137,
      "rewards/margins": 14.212499618530273,
      "rewards/rejected": -9.734375,
      "step": 1190
    },
    {
      "epoch": 0.5633802816901409,
      "grad_norm": 1227.687052177058,
      "learning_rate": 2.4282733437663016e-07,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.6800781488418579,
      "logps/chosen": -444.3999938964844,
      "logps/rejected": -565.7999877929688,
      "loss": 2.7307,
      "rewards/accuracies": 0.589062511920929,
      "rewards/chosen": 4.948437690734863,
      "rewards/margins": 13.206250190734863,
      "rewards/rejected": -8.253125190734863,
      "step": 1200
    },
    {
      "epoch": 0.568075117370892,
      "grad_norm": 1635.650256453477,
      "learning_rate": 2.4021909233176836e-07,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.6429687738418579,
      "logps/chosen": -434.3999938964844,
      "logps/rejected": -600.4000244140625,
      "loss": 2.7246,
      "rewards/accuracies": 0.6226562261581421,
      "rewards/chosen": 4.093359470367432,
      "rewards/margins": 14.75,
      "rewards/rejected": -10.653124809265137,
      "step": 1210
    },
    {
      "epoch": 0.5727699530516432,
      "grad_norm": 1421.698104022691,
      "learning_rate": 2.376108502869066e-07,
      "logits/chosen": -0.978515625,
      "logits/rejected": -0.5693359375,
      "logps/chosen": -464.6000061035156,
      "logps/rejected": -624.2000122070312,
      "loss": 3.1767,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 3.9976563453674316,
      "rewards/margins": 15.918749809265137,
      "rewards/rejected": -11.918749809265137,
      "step": 1220
    },
    {
      "epoch": 0.5774647887323944,
      "grad_norm": 4366.322889486067,
      "learning_rate": 2.3500260824204484e-07,
      "logits/chosen": -0.9917968511581421,
      "logits/rejected": -0.673632800579071,
      "logps/chosen": -489.3999938964844,
      "logps/rejected": -628.2000122070312,
      "loss": 3.1121,
      "rewards/accuracies": 0.604687511920929,
      "rewards/chosen": 2.59375,
      "rewards/margins": 14.865625381469727,
      "rewards/rejected": -12.265625,
      "step": 1230
    },
    {
      "epoch": 0.5821596244131455,
      "grad_norm": 1366.9823893250618,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": -1.0324218273162842,
      "logits/rejected": -0.6859375238418579,
      "logps/chosen": -448.6000061035156,
      "logps/rejected": -586.2000122070312,
      "loss": 2.7643,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 3.975781202316284,
      "rewards/margins": 14.512499809265137,
      "rewards/rejected": -10.524999618530273,
      "step": 1240
    },
    {
      "epoch": 0.5868544600938967,
      "grad_norm": 1599.2016110899367,
      "learning_rate": 2.2978612415232132e-07,
      "logits/chosen": -0.991406261920929,
      "logits/rejected": -0.6234375238418579,
      "logps/chosen": -478.6000061035156,
      "logps/rejected": -619.5999755859375,
      "loss": 2.9578,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": 3.587109327316284,
      "rewards/margins": 15.368749618530273,
      "rewards/rejected": -11.78125,
      "step": 1250
    },
    {
      "epoch": 0.5915492957746479,
      "grad_norm": 1276.1757108531826,
      "learning_rate": 2.2717788210745957e-07,
      "logits/chosen": -1.0046875476837158,
      "logits/rejected": -0.685546875,
      "logps/chosen": -451.6000061035156,
      "logps/rejected": -609.7999877929688,
      "loss": 3.3694,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": 3.66796875,
      "rewards/margins": 15.587499618530273,
      "rewards/rejected": -11.912500381469727,
      "step": 1260
    },
    {
      "epoch": 0.596244131455399,
      "grad_norm": 2235.1243679419404,
      "learning_rate": 2.245696400625978e-07,
      "logits/chosen": -0.9722656011581421,
      "logits/rejected": -0.6820312738418579,
      "logps/chosen": -477.3999938964844,
      "logps/rejected": -619.7999877929688,
      "loss": 3.3838,
      "rewards/accuracies": 0.629687488079071,
      "rewards/chosen": 3.8296875953674316,
      "rewards/margins": 14.556249618530273,
      "rewards/rejected": -10.725000381469727,
      "step": 1270
    },
    {
      "epoch": 0.6009389671361502,
      "grad_norm": 1402.2581478315037,
      "learning_rate": 2.2196139801773602e-07,
      "logits/chosen": -1.0304687023162842,
      "logits/rejected": -0.725781261920929,
      "logps/chosen": -442.6000061035156,
      "logps/rejected": -560.4000244140625,
      "loss": 2.7999,
      "rewards/accuracies": 0.620312511920929,
      "rewards/chosen": 3.921093702316284,
      "rewards/margins": 12.893750190734863,
      "rewards/rejected": -8.987500190734863,
      "step": 1280
    },
    {
      "epoch": 0.6056338028169014,
      "grad_norm": 2101.194208461549,
      "learning_rate": 2.1935315597287428e-07,
      "logits/chosen": -1.0085937976837158,
      "logits/rejected": -0.7300781011581421,
      "logps/chosen": -459.20001220703125,
      "logps/rejected": -569.2000122070312,
      "loss": 3.2613,
      "rewards/accuracies": 0.6195312738418579,
      "rewards/chosen": 4.310937404632568,
      "rewards/margins": 13.493749618530273,
      "rewards/rejected": -9.184374809265137,
      "step": 1290
    },
    {
      "epoch": 0.6103286384976526,
      "grad_norm": 1340.7990970749092,
      "learning_rate": 2.167449139280125e-07,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.735156238079071,
      "logps/chosen": -460.3999938964844,
      "logps/rejected": -605.2000122070312,
      "loss": 2.9729,
      "rewards/accuracies": 0.620312511920929,
      "rewards/chosen": 5.328125,
      "rewards/margins": 14.412500381469727,
      "rewards/rejected": -9.078125,
      "step": 1300
    },
    {
      "epoch": 0.6150234741784038,
      "grad_norm": 1071.3629012987517,
      "learning_rate": 2.1413667188315073e-07,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.6556640863418579,
      "logps/chosen": -437.0,
      "logps/rejected": -562.5999755859375,
      "loss": 2.8054,
      "rewards/accuracies": 0.6429687738418579,
      "rewards/chosen": 5.293749809265137,
      "rewards/margins": 16.21875,
      "rewards/rejected": -10.921875,
      "step": 1310
    },
    {
      "epoch": 0.6197183098591549,
      "grad_norm": 1443.1742505274058,
      "learning_rate": 2.1152842983828898e-07,
      "logits/chosen": -1.055078148841858,
      "logits/rejected": -0.790234386920929,
      "logps/chosen": -461.20001220703125,
      "logps/rejected": -577.2000122070312,
      "loss": 3.5987,
      "rewards/accuracies": 0.602343738079071,
      "rewards/chosen": 4.533593654632568,
      "rewards/margins": 13.334375381469727,
      "rewards/rejected": -8.800000190734863,
      "step": 1320
    },
    {
      "epoch": 0.6244131455399061,
      "grad_norm": 1774.0756615744851,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": -0.916015625,
      "logits/rejected": -0.7261718511581421,
      "logps/chosen": -480.79998779296875,
      "logps/rejected": -652.0,
      "loss": 3.4229,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 5.260937690734863,
      "rewards/margins": 16.734375,
      "rewards/rejected": -11.465624809265137,
      "step": 1330
    },
    {
      "epoch": 0.6291079812206573,
      "grad_norm": 1473.9513848298084,
      "learning_rate": 2.0631194574856543e-07,
      "logits/chosen": -0.956250011920929,
      "logits/rejected": -0.6910156011581421,
      "logps/chosen": -442.20001220703125,
      "logps/rejected": -531.4000244140625,
      "loss": 3.0268,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 5.287499904632568,
      "rewards/margins": 13.206250190734863,
      "rewards/rejected": -7.915625095367432,
      "step": 1340
    },
    {
      "epoch": 0.6338028169014085,
      "grad_norm": 2631.4966777182663,
      "learning_rate": 2.0370370370370369e-07,
      "logits/chosen": -1.0144531726837158,
      "logits/rejected": -0.668749988079071,
      "logps/chosen": -455.3999938964844,
      "logps/rejected": -631.7999877929688,
      "loss": 2.9182,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 5.064062595367432,
      "rewards/margins": 16.149999618530273,
      "rewards/rejected": -11.0625,
      "step": 1350
    },
    {
      "epoch": 0.6384976525821596,
      "grad_norm": 1641.4856233306718,
      "learning_rate": 2.010954616588419e-07,
      "logits/chosen": -1.0558593273162842,
      "logits/rejected": -0.776562511920929,
      "logps/chosen": -448.6000061035156,
      "logps/rejected": -565.0,
      "loss": 3.0185,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 5.032812595367432,
      "rewards/margins": 13.393750190734863,
      "rewards/rejected": -8.381250381469727,
      "step": 1360
    },
    {
      "epoch": 0.6431924882629108,
      "grad_norm": 1652.6110732556183,
      "learning_rate": 1.9848721961398017e-07,
      "logits/chosen": -0.951171875,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -454.79998779296875,
      "logps/rejected": -575.2000122070312,
      "loss": 2.648,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": 4.014062404632568,
      "rewards/margins": 12.581250190734863,
      "rewards/rejected": -8.556249618530273,
      "step": 1370
    },
    {
      "epoch": 0.647887323943662,
      "grad_norm": 1624.581515882958,
      "learning_rate": 1.958789775691184e-07,
      "logits/chosen": -1.0597655773162842,
      "logits/rejected": -0.760546863079071,
      "logps/chosen": -436.79998779296875,
      "logps/rejected": -591.5999755859375,
      "loss": 2.6278,
      "rewards/accuracies": 0.617968738079071,
      "rewards/chosen": 4.279687404632568,
      "rewards/margins": 14.556249618530273,
      "rewards/rejected": -10.262499809265137,
      "step": 1380
    },
    {
      "epoch": 0.6525821596244131,
      "grad_norm": 1588.1535760712807,
      "learning_rate": 1.9327073552425662e-07,
      "logits/chosen": -0.963671863079071,
      "logits/rejected": -0.6475585699081421,
      "logps/chosen": -432.79998779296875,
      "logps/rejected": -565.4000244140625,
      "loss": 2.9877,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 3.3492188453674316,
      "rewards/margins": 14.181249618530273,
      "rewards/rejected": -10.834375381469727,
      "step": 1390
    },
    {
      "epoch": 0.6572769953051644,
      "grad_norm": 1680.0182409798317,
      "learning_rate": 1.906624934793949e-07,
      "logits/chosen": -1.0617187023162842,
      "logits/rejected": -0.68359375,
      "logps/chosen": -457.0,
      "logps/rejected": -621.4000244140625,
      "loss": 3.254,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": 2.4117188453674316,
      "rewards/margins": 15.693750381469727,
      "rewards/rejected": -13.293749809265137,
      "step": 1400
    },
    {
      "epoch": 0.6619718309859155,
      "grad_norm": 1280.38794590205,
      "learning_rate": 1.8805425143453312e-07,
      "logits/chosen": -1.0285155773162842,
      "logits/rejected": -0.705078125,
      "logps/chosen": -433.20001220703125,
      "logps/rejected": -652.5999755859375,
      "loss": 3.1156,
      "rewards/accuracies": 0.614062488079071,
      "rewards/chosen": 3.5367188453674316,
      "rewards/margins": 16.15625,
      "rewards/rejected": -12.609375,
      "step": 1410
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 1400.3940888433954,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": -1.056249976158142,
      "logits/rejected": -0.7007812261581421,
      "logps/chosen": -445.79998779296875,
      "logps/rejected": -566.7999877929688,
      "loss": 2.6084,
      "rewards/accuracies": 0.624218761920929,
      "rewards/chosen": 3.7984375953674316,
      "rewards/margins": 14.068750381469727,
      "rewards/rejected": -10.268750190734863,
      "step": 1420
    },
    {
      "epoch": 0.6713615023474179,
      "grad_norm": 1128.1283944245477,
      "learning_rate": 1.828377673448096e-07,
      "logits/chosen": -1.0402343273162842,
      "logits/rejected": -0.691601574420929,
      "logps/chosen": -487.6000061035156,
      "logps/rejected": -596.4000244140625,
      "loss": 3.3679,
      "rewards/accuracies": 0.647656261920929,
      "rewards/chosen": 3.7088379859924316,
      "rewards/margins": 14.643750190734863,
      "rewards/rejected": -10.912500381469727,
      "step": 1430
    },
    {
      "epoch": 0.676056338028169,
      "grad_norm": 1261.0520738814687,
      "learning_rate": 1.8022952529994783e-07,
      "logits/chosen": -0.986328125,
      "logits/rejected": -0.717578113079071,
      "logps/chosen": -457.3999938964844,
      "logps/rejected": -581.0,
      "loss": 2.4932,
      "rewards/accuracies": 0.6273437738418579,
      "rewards/chosen": 5.192187309265137,
      "rewards/margins": 16.84375,
      "rewards/rejected": -11.662500381469727,
      "step": 1440
    },
    {
      "epoch": 0.6807511737089202,
      "grad_norm": 1621.0138231954081,
      "learning_rate": 1.7762128325508608e-07,
      "logits/chosen": -0.917187511920929,
      "logits/rejected": -0.612011730670929,
      "logps/chosen": -477.6000061035156,
      "logps/rejected": -628.7999877929688,
      "loss": 3.5834,
      "rewards/accuracies": 0.6117187738418579,
      "rewards/chosen": 4.045312404632568,
      "rewards/margins": 16.481250762939453,
      "rewards/rejected": -12.446874618530273,
      "step": 1450
    },
    {
      "epoch": 0.6854460093896714,
      "grad_norm": 1306.853234633484,
      "learning_rate": 1.750130412102243e-07,
      "logits/chosen": -1.019140601158142,
      "logits/rejected": -0.645703136920929,
      "logps/chosen": -495.3999938964844,
      "logps/rejected": -611.5999755859375,
      "loss": 3.3275,
      "rewards/accuracies": 0.6390625238418579,
      "rewards/chosen": 3.17578125,
      "rewards/margins": 14.850000381469727,
      "rewards/rejected": -11.675000190734863,
      "step": 1460
    },
    {
      "epoch": 0.6901408450704225,
      "grad_norm": 1724.3010632496987,
      "learning_rate": 1.7240479916536254e-07,
      "logits/chosen": -1.0460937023162842,
      "logits/rejected": -0.694091796875,
      "logps/chosen": -441.20001220703125,
      "logps/rejected": -629.5999755859375,
      "loss": 2.7783,
      "rewards/accuracies": 0.6265624761581421,
      "rewards/chosen": 3.598437547683716,
      "rewards/margins": 15.787500381469727,
      "rewards/rejected": -12.181249618530273,
      "step": 1470
    },
    {
      "epoch": 0.6948356807511737,
      "grad_norm": 1344.485511693929,
      "learning_rate": 1.697965571205008e-07,
      "logits/chosen": -0.995312511920929,
      "logits/rejected": -0.7095702886581421,
      "logps/chosen": -446.6000061035156,
      "logps/rejected": -567.2000122070312,
      "loss": 2.8973,
      "rewards/accuracies": 0.6171875,
      "rewards/chosen": 2.5882811546325684,
      "rewards/margins": 13.537500381469727,
      "rewards/rejected": -10.934374809265137,
      "step": 1480
    },
    {
      "epoch": 0.6995305164319249,
      "grad_norm": 2095.2000825174127,
      "learning_rate": 1.6718831507563902e-07,
      "logits/chosen": -1.0242187976837158,
      "logits/rejected": -0.7183593511581421,
      "logps/chosen": -453.79998779296875,
      "logps/rejected": -546.5999755859375,
      "loss": 2.6625,
      "rewards/accuracies": 0.632031261920929,
      "rewards/chosen": 3.520312547683716,
      "rewards/margins": 12.743749618530273,
      "rewards/rejected": -9.221875190734863,
      "step": 1490
    },
    {
      "epoch": 0.704225352112676,
      "grad_norm": 1272.0502479838722,
      "learning_rate": 1.6458007303077727e-07,
      "logits/chosen": -1.056640625,
      "logits/rejected": -0.7457031011581421,
      "logps/chosen": -433.3999938964844,
      "logps/rejected": -561.0,
      "loss": 2.9507,
      "rewards/accuracies": 0.6148437261581421,
      "rewards/chosen": 3.1117186546325684,
      "rewards/margins": 13.631250381469727,
      "rewards/rejected": -10.515625,
      "step": 1500
    },
    {
      "epoch": 0.7089201877934272,
      "grad_norm": 1561.9484676043603,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.75390625,
      "logps/chosen": -461.6000061035156,
      "logps/rejected": -610.2000122070312,
      "loss": 3.1314,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 3.1117186546325684,
      "rewards/margins": 14.081250190734863,
      "rewards/rejected": -10.959375381469727,
      "step": 1510
    },
    {
      "epoch": 0.7136150234741784,
      "grad_norm": 2759.9437778862534,
      "learning_rate": 1.5936358894105372e-07,
      "logits/chosen": -1.064062476158142,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -453.6000061035156,
      "logps/rejected": -585.0,
      "loss": 3.0774,
      "rewards/accuracies": 0.6109374761581421,
      "rewards/chosen": 3.893359422683716,
      "rewards/margins": 13.824999809265137,
      "rewards/rejected": -9.943750381469727,
      "step": 1520
    },
    {
      "epoch": 0.7183098591549296,
      "grad_norm": 1627.17515399697,
      "learning_rate": 1.5675534689619197e-07,
      "logits/chosen": -0.9996093511581421,
      "logits/rejected": -0.664843738079071,
      "logps/chosen": -462.20001220703125,
      "logps/rejected": -558.2000122070312,
      "loss": 2.8491,
      "rewards/accuracies": 0.61328125,
      "rewards/chosen": 3.3140625953674316,
      "rewards/margins": 13.65625,
      "rewards/rejected": -10.356249809265137,
      "step": 1530
    },
    {
      "epoch": 0.7230046948356808,
      "grad_norm": 1620.4264971536097,
      "learning_rate": 1.541471048513302e-07,
      "logits/chosen": -1.0234375,
      "logits/rejected": -0.681640625,
      "logps/chosen": -461.3999938964844,
      "logps/rejected": -598.5999755859375,
      "loss": 3.4056,
      "rewards/accuracies": 0.633593738079071,
      "rewards/chosen": 3.940380811691284,
      "rewards/margins": 14.837499618530273,
      "rewards/rejected": -10.909375190734863,
      "step": 1540
    },
    {
      "epoch": 0.7276995305164319,
      "grad_norm": 1504.0317365726369,
      "learning_rate": 1.5153886280646843e-07,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.702929675579071,
      "logps/chosen": -423.3999938964844,
      "logps/rejected": -549.0,
      "loss": 2.7939,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 4.1953125,
      "rewards/margins": 12.574999809265137,
      "rewards/rejected": -8.384374618530273,
      "step": 1550
    },
    {
      "epoch": 0.7323943661971831,
      "grad_norm": 1246.0630354054854,
      "learning_rate": 1.4893062076160668e-07,
      "logits/chosen": -0.9859374761581421,
      "logits/rejected": -0.619140625,
      "logps/chosen": -455.0,
      "logps/rejected": -603.4000244140625,
      "loss": 2.6526,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 5.364062309265137,
      "rewards/margins": 15.675000190734863,
      "rewards/rejected": -10.321874618530273,
      "step": 1560
    },
    {
      "epoch": 0.7370892018779343,
      "grad_norm": 1027.5300036621297,
      "learning_rate": 1.463223787167449e-07,
      "logits/chosen": -0.9945312738418579,
      "logits/rejected": -0.6640625,
      "logps/chosen": -459.6000061035156,
      "logps/rejected": -577.5999755859375,
      "loss": 2.8625,
      "rewards/accuracies": 0.616406261920929,
      "rewards/chosen": 5.1015625,
      "rewards/margins": 14.149999618530273,
      "rewards/rejected": -9.0625,
      "step": 1570
    },
    {
      "epoch": 0.7417840375586855,
      "grad_norm": 1654.8548979303134,
      "learning_rate": 1.4371413667188313e-07,
      "logits/chosen": -1.013671875,
      "logits/rejected": -0.627734363079071,
      "logps/chosen": -445.6000061035156,
      "logps/rejected": -621.5999755859375,
      "loss": 2.7124,
      "rewards/accuracies": 0.633593738079071,
      "rewards/chosen": 4.453125,
      "rewards/margins": 16.024999618530273,
      "rewards/rejected": -11.556249618530273,
      "step": 1580
    },
    {
      "epoch": 0.7464788732394366,
      "grad_norm": 1518.9960983136423,
      "learning_rate": 1.4110589462702139e-07,
      "logits/chosen": -0.9781249761581421,
      "logits/rejected": -0.699999988079071,
      "logps/chosen": -447.0,
      "logps/rejected": -560.2000122070312,
      "loss": 2.8653,
      "rewards/accuracies": 0.629687488079071,
      "rewards/chosen": 3.957812547683716,
      "rewards/margins": 12.581250190734863,
      "rewards/rejected": -8.637499809265137,
      "step": 1590
    },
    {
      "epoch": 0.7511737089201878,
      "grad_norm": 1790.8908727098801,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": -1.008203148841858,
      "logits/rejected": -0.654296875,
      "logps/chosen": -461.0,
      "logps/rejected": -614.2000122070312,
      "loss": 3.0565,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 3.817187547683716,
      "rewards/margins": 16.556249618530273,
      "rewards/rejected": -12.728124618530273,
      "step": 1600
    },
    {
      "epoch": 0.755868544600939,
      "grad_norm": 1656.5944068970352,
      "learning_rate": 1.3588941053729787e-07,
      "logits/chosen": -0.9957031011581421,
      "logits/rejected": -0.5972656011581421,
      "logps/chosen": -469.3999938964844,
      "logps/rejected": -616.2000122070312,
      "loss": 3.1971,
      "rewards/accuracies": 0.6382812261581421,
      "rewards/chosen": 3.418750047683716,
      "rewards/margins": 14.618749618530273,
      "rewards/rejected": -11.203125,
      "step": 1610
    },
    {
      "epoch": 0.7605633802816901,
      "grad_norm": 1431.4791078861954,
      "learning_rate": 1.332811684924361e-07,
      "logits/chosen": -0.961718738079071,
      "logits/rejected": -0.6919921636581421,
      "logps/chosen": -470.3999938964844,
      "logps/rejected": -602.0,
      "loss": 3.0214,
      "rewards/accuracies": 0.614062488079071,
      "rewards/chosen": 3.635937452316284,
      "rewards/margins": 14.587499618530273,
      "rewards/rejected": -10.949999809265137,
      "step": 1620
    },
    {
      "epoch": 0.7652582159624414,
      "grad_norm": 1407.6519816970804,
      "learning_rate": 1.3067292644757432e-07,
      "logits/chosen": -1.0539062023162842,
      "logits/rejected": -0.7060546875,
      "logps/chosen": -468.0,
      "logps/rejected": -640.7999877929688,
      "loss": 2.7867,
      "rewards/accuracies": 0.6273437738418579,
      "rewards/chosen": 3.736328125,
      "rewards/margins": 15.25,
      "rewards/rejected": -11.515625,
      "step": 1630
    },
    {
      "epoch": 0.7699530516431925,
      "grad_norm": 1348.0292867815167,
      "learning_rate": 1.2806468440271257e-07,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.6499999761581421,
      "logps/chosen": -440.20001220703125,
      "logps/rejected": -585.2000122070312,
      "loss": 2.4653,
      "rewards/accuracies": 0.6070312261581421,
      "rewards/chosen": 4.126562595367432,
      "rewards/margins": 15.9375,
      "rewards/rejected": -11.803125381469727,
      "step": 1640
    },
    {
      "epoch": 0.7746478873239436,
      "grad_norm": 1468.2193321032719,
      "learning_rate": 1.254564423578508e-07,
      "logits/chosen": -0.9554687738418579,
      "logits/rejected": -0.698437511920929,
      "logps/chosen": -445.6000061035156,
      "logps/rejected": -591.7999877929688,
      "loss": 2.5212,
      "rewards/accuracies": 0.62890625,
      "rewards/chosen": 3.606250047683716,
      "rewards/margins": 15.643750190734863,
      "rewards/rejected": -12.024999618530273,
      "step": 1650
    },
    {
      "epoch": 0.7793427230046949,
      "grad_norm": 1244.481592138495,
      "learning_rate": 1.2284820031298902e-07,
      "logits/chosen": -0.967578113079071,
      "logits/rejected": -0.6703125238418579,
      "logps/chosen": -483.20001220703125,
      "logps/rejected": -624.4000244140625,
      "loss": 2.3411,
      "rewards/accuracies": 0.65234375,
      "rewards/chosen": 3.6078124046325684,
      "rewards/margins": 16.643749237060547,
      "rewards/rejected": -13.037500381469727,
      "step": 1660
    },
    {
      "epoch": 0.784037558685446,
      "grad_norm": 2467.646058630298,
      "learning_rate": 1.2023995826812728e-07,
      "logits/chosen": -0.989453136920929,
      "logits/rejected": -0.66015625,
      "logps/chosen": -451.79998779296875,
      "logps/rejected": -595.2000122070312,
      "loss": 2.5093,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 3.6015625,
      "rewards/margins": 15.993749618530273,
      "rewards/rejected": -12.395312309265137,
      "step": 1670
    },
    {
      "epoch": 0.7887323943661971,
      "grad_norm": 1643.087251427477,
      "learning_rate": 1.176317162232655e-07,
      "logits/chosen": -1.039453148841858,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -446.79998779296875,
      "logps/rejected": -603.4000244140625,
      "loss": 2.776,
      "rewards/accuracies": 0.6148437261581421,
      "rewards/chosen": 3.611523389816284,
      "rewards/margins": 14.862500190734863,
      "rewards/rejected": -11.243749618530273,
      "step": 1680
    },
    {
      "epoch": 0.7934272300469484,
      "grad_norm": 1352.1095166711063,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": -1.0519530773162842,
      "logits/rejected": -0.734375,
      "logps/chosen": -447.79998779296875,
      "logps/rejected": -593.0,
      "loss": 3.0022,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 3.7007813453674316,
      "rewards/margins": 14.524999618530273,
      "rewards/rejected": -10.809374809265137,
      "step": 1690
    },
    {
      "epoch": 0.7981220657276995,
      "grad_norm": 1663.687424154431,
      "learning_rate": 1.1241523213354198e-07,
      "logits/chosen": -1.02734375,
      "logits/rejected": -0.722851574420929,
      "logps/chosen": -452.79998779296875,
      "logps/rejected": -615.2000122070312,
      "loss": 2.7187,
      "rewards/accuracies": 0.624218761920929,
      "rewards/chosen": 4.196875095367432,
      "rewards/margins": 15.824999809265137,
      "rewards/rejected": -11.628125190734863,
      "step": 1700
    },
    {
      "epoch": 0.8028169014084507,
      "grad_norm": 1617.2508639200043,
      "learning_rate": 1.0980699008868022e-07,
      "logits/chosen": -1.0734374523162842,
      "logits/rejected": -0.7601562738418579,
      "logps/chosen": -468.79998779296875,
      "logps/rejected": -572.2000122070312,
      "loss": 3.5428,
      "rewards/accuracies": 0.614062488079071,
      "rewards/chosen": 2.5503907203674316,
      "rewards/margins": 14.074999809265137,
      "rewards/rejected": -11.524999618530273,
      "step": 1710
    },
    {
      "epoch": 0.8075117370892019,
      "grad_norm": 1507.8341627612087,
      "learning_rate": 1.0719874804381846e-07,
      "logits/chosen": -1.1046874523162842,
      "logits/rejected": -0.7867187261581421,
      "logps/chosen": -443.0,
      "logps/rejected": -535.7999877929688,
      "loss": 2.9887,
      "rewards/accuracies": 0.6226562261581421,
      "rewards/chosen": 3.34375,
      "rewards/margins": 13.774999618530273,
      "rewards/rejected": -10.443750381469727,
      "step": 1720
    },
    {
      "epoch": 0.812206572769953,
      "grad_norm": 1757.649533935488,
      "learning_rate": 1.045905059989567e-07,
      "logits/chosen": -1.037109375,
      "logits/rejected": -0.699414074420929,
      "logps/chosen": -426.3999938964844,
      "logps/rejected": -530.5999755859375,
      "loss": 2.4663,
      "rewards/accuracies": 0.629687488079071,
      "rewards/chosen": 3.655468702316284,
      "rewards/margins": 14.756250381469727,
      "rewards/rejected": -11.112500190734863,
      "step": 1730
    },
    {
      "epoch": 0.8169014084507042,
      "grad_norm": 1890.6515598185326,
      "learning_rate": 1.0198226395409494e-07,
      "logits/chosen": -1.041015625,
      "logits/rejected": -0.712109386920929,
      "logps/chosen": -434.6000061035156,
      "logps/rejected": -567.5999755859375,
      "loss": 2.4001,
      "rewards/accuracies": 0.635937511920929,
      "rewards/chosen": 3.578125,
      "rewards/margins": 15.081250190734863,
      "rewards/rejected": -11.509374618530273,
      "step": 1740
    },
    {
      "epoch": 0.8215962441314554,
      "grad_norm": 1244.3303537359604,
      "learning_rate": 9.937402190923318e-08,
      "logits/chosen": -1.0574219226837158,
      "logits/rejected": -0.6703125238418579,
      "logps/chosen": -422.3999938964844,
      "logps/rejected": -620.7999877929688,
      "loss": 2.2965,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 4.2109375,
      "rewards/margins": 19.118749618530273,
      "rewards/rejected": -14.918749809265137,
      "step": 1750
    },
    {
      "epoch": 0.8262910798122066,
      "grad_norm": 1290.3641929164842,
      "learning_rate": 9.676577986437141e-08,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.746874988079071,
      "logps/chosen": -469.3999938964844,
      "logps/rejected": -589.5999755859375,
      "loss": 2.5106,
      "rewards/accuracies": 0.6351562738418579,
      "rewards/chosen": 3.891406297683716,
      "rewards/margins": 16.100000381469727,
      "rewards/rejected": -12.212499618530273,
      "step": 1760
    },
    {
      "epoch": 0.8309859154929577,
      "grad_norm": 1678.1562785560782,
      "learning_rate": 9.415753781950965e-08,
      "logits/chosen": -0.993359386920929,
      "logits/rejected": -0.690625011920929,
      "logps/chosen": -439.0,
      "logps/rejected": -588.0,
      "loss": 2.9217,
      "rewards/accuracies": 0.6328125,
      "rewards/chosen": 3.653125047683716,
      "rewards/margins": 15.068750381469727,
      "rewards/rejected": -11.415624618530273,
      "step": 1770
    },
    {
      "epoch": 0.8356807511737089,
      "grad_norm": 1227.6660820708619,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": -0.994921863079071,
      "logits/rejected": -0.6845703125,
      "logps/chosen": -446.79998779296875,
      "logps/rejected": -582.0,
      "loss": 2.6593,
      "rewards/accuracies": 0.639843761920929,
      "rewards/chosen": 3.6304688453674316,
      "rewards/margins": 13.862500190734863,
      "rewards/rejected": -10.228124618530273,
      "step": 1780
    },
    {
      "epoch": 0.8403755868544601,
      "grad_norm": 1445.7974626978519,
      "learning_rate": 8.894105372978613e-08,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.669140636920929,
      "logps/chosen": -457.6000061035156,
      "logps/rejected": -584.2000122070312,
      "loss": 2.8483,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 4.251562595367432,
      "rewards/margins": 16.049999237060547,
      "rewards/rejected": -11.806249618530273,
      "step": 1790
    },
    {
      "epoch": 0.8450704225352113,
      "grad_norm": 1326.532434045158,
      "learning_rate": 8.633281168492435e-08,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.679492175579071,
      "logps/chosen": -446.0,
      "logps/rejected": -581.5999755859375,
      "loss": 3.0347,
      "rewards/accuracies": 0.6234375238418579,
      "rewards/chosen": 3.1304688453674316,
      "rewards/margins": 14.65625,
      "rewards/rejected": -11.524999618530273,
      "step": 1800
    },
    {
      "epoch": 0.8497652582159625,
      "grad_norm": 1482.7086561315384,
      "learning_rate": 8.372456964006259e-08,
      "logits/chosen": -1.0343749523162842,
      "logits/rejected": -0.746874988079071,
      "logps/chosen": -444.20001220703125,
      "logps/rejected": -590.5999755859375,
      "loss": 3.03,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 3.176953077316284,
      "rewards/margins": 16.043750762939453,
      "rewards/rejected": -12.862500190734863,
      "step": 1810
    },
    {
      "epoch": 0.8544600938967136,
      "grad_norm": 2452.8873147663867,
      "learning_rate": 8.111632759520083e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.761914074420929,
      "logps/chosen": -486.79998779296875,
      "logps/rejected": -604.2000122070312,
      "loss": 3.3046,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 2.958789110183716,
      "rewards/margins": 14.449999809265137,
      "rewards/rejected": -11.484375,
      "step": 1820
    },
    {
      "epoch": 0.8591549295774648,
      "grad_norm": 1530.8433838276749,
      "learning_rate": 7.850808555033907e-08,
      "logits/chosen": -0.9296875,
      "logits/rejected": -0.6162109375,
      "logps/chosen": -450.3999938964844,
      "logps/rejected": -613.0,
      "loss": 2.7145,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": 2.6500000953674316,
      "rewards/margins": 15.306249618530273,
      "rewards/rejected": -12.65625,
      "step": 1830
    },
    {
      "epoch": 0.863849765258216,
      "grad_norm": 1598.4258179912727,
      "learning_rate": 7.58998435054773e-08,
      "logits/chosen": -1.0851562023162842,
      "logits/rejected": -0.771484375,
      "logps/chosen": -461.0,
      "logps/rejected": -586.7999877929688,
      "loss": 2.6132,
      "rewards/accuracies": 0.62109375,
      "rewards/chosen": 4.604687690734863,
      "rewards/margins": 15.237500190734863,
      "rewards/rejected": -10.621874809265137,
      "step": 1840
    },
    {
      "epoch": 0.8685446009389671,
      "grad_norm": 1550.4231596372506,
      "learning_rate": 7.329160146061554e-08,
      "logits/chosen": -1.012109398841858,
      "logits/rejected": -0.653124988079071,
      "logps/chosen": -436.3999938964844,
      "logps/rejected": -580.4000244140625,
      "loss": 2.8483,
      "rewards/accuracies": 0.608593761920929,
      "rewards/chosen": 3.909374952316284,
      "rewards/margins": 14.493749618530273,
      "rewards/rejected": -10.571874618530273,
      "step": 1850
    },
    {
      "epoch": 0.8732394366197183,
      "grad_norm": 1866.8288476530904,
      "learning_rate": 7.068335941575378e-08,
      "logits/chosen": -1.031640648841858,
      "logits/rejected": -0.6976562738418579,
      "logps/chosen": -448.0,
      "logps/rejected": -577.4000244140625,
      "loss": 3.0313,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 2.1390624046325684,
      "rewards/margins": 14.006250381469727,
      "rewards/rejected": -11.890625,
      "step": 1860
    },
    {
      "epoch": 0.8779342723004695,
      "grad_norm": 1500.7479361649953,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.637890636920929,
      "logps/chosen": -439.79998779296875,
      "logps/rejected": -602.5999755859375,
      "loss": 2.5384,
      "rewards/accuracies": 0.633593738079071,
      "rewards/chosen": 3.090625047683716,
      "rewards/margins": 17.325000762939453,
      "rewards/rejected": -14.199999809265137,
      "step": 1870
    },
    {
      "epoch": 0.8826291079812206,
      "grad_norm": 1271.434914382074,
      "learning_rate": 6.546687532603024e-08,
      "logits/chosen": -1.0078125,
      "logits/rejected": -0.6421874761581421,
      "logps/chosen": -447.0,
      "logps/rejected": -592.2000122070312,
      "loss": 2.7624,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 3.067578077316284,
      "rewards/margins": 14.368749618530273,
      "rewards/rejected": -11.293749809265137,
      "step": 1880
    },
    {
      "epoch": 0.8873239436619719,
      "grad_norm": 1397.0702597064248,
      "learning_rate": 6.285863328116848e-08,
      "logits/chosen": -1.08203125,
      "logits/rejected": -0.6796875,
      "logps/chosen": -408.79998779296875,
      "logps/rejected": -604.0,
      "loss": 2.6634,
      "rewards/accuracies": 0.62109375,
      "rewards/chosen": 3.073437452316284,
      "rewards/margins": 15.568750381469727,
      "rewards/rejected": -12.456250190734863,
      "step": 1890
    },
    {
      "epoch": 0.892018779342723,
      "grad_norm": 1262.8771091167048,
      "learning_rate": 6.025039123630672e-08,
      "logits/chosen": -1.0554687976837158,
      "logits/rejected": -0.7308593988418579,
      "logps/chosen": -457.6000061035156,
      "logps/rejected": -562.4000244140625,
      "loss": 2.9883,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": 2.946093797683716,
      "rewards/margins": 13.699999809265137,
      "rewards/rejected": -10.743749618530273,
      "step": 1900
    },
    {
      "epoch": 0.8967136150234741,
      "grad_norm": 1538.779966561391,
      "learning_rate": 5.764214919144496e-08,
      "logits/chosen": -1.0382812023162842,
      "logits/rejected": -0.7417968511581421,
      "logps/chosen": -460.20001220703125,
      "logps/rejected": -559.4000244140625,
      "loss": 3.4307,
      "rewards/accuracies": 0.6070312261581421,
      "rewards/chosen": 2.8416991233825684,
      "rewards/margins": 12.668749809265137,
      "rewards/rejected": -9.828125,
      "step": 1910
    },
    {
      "epoch": 0.9014084507042254,
      "grad_norm": 2870.709301161901,
      "learning_rate": 5.50339071465832e-08,
      "logits/chosen": -1.006250023841858,
      "logits/rejected": -0.6832031011581421,
      "logps/chosen": -460.6000061035156,
      "logps/rejected": -539.7999877929688,
      "loss": 3.2639,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 2.890625,
      "rewards/margins": 15.381250381469727,
      "rewards/rejected": -12.475000381469727,
      "step": 1920
    },
    {
      "epoch": 0.9061032863849765,
      "grad_norm": 1596.6147448643887,
      "learning_rate": 5.2425665101721436e-08,
      "logits/chosen": -1.04296875,
      "logits/rejected": -0.6820312738418579,
      "logps/chosen": -459.0,
      "logps/rejected": -630.2000122070312,
      "loss": 2.3996,
      "rewards/accuracies": 0.641406238079071,
      "rewards/chosen": 3.807812452316284,
      "rewards/margins": 16.368749618530273,
      "rewards/rejected": -12.556249618530273,
      "step": 1930
    },
    {
      "epoch": 0.9107981220657277,
      "grad_norm": 1280.1322871121254,
      "learning_rate": 4.9817423056859675e-08,
      "logits/chosen": -1.0226562023162842,
      "logits/rejected": -0.6792968511581421,
      "logps/chosen": -464.20001220703125,
      "logps/rejected": -611.2000122070312,
      "loss": 2.7596,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 4.216406345367432,
      "rewards/margins": 16.950000762939453,
      "rewards/rejected": -12.746874809265137,
      "step": 1940
    },
    {
      "epoch": 0.9154929577464789,
      "grad_norm": 1146.8581083859165,
      "learning_rate": 4.720918101199791e-08,
      "logits/chosen": -1.078125,
      "logits/rejected": -0.712109386920929,
      "logps/chosen": -461.20001220703125,
      "logps/rejected": -667.5999755859375,
      "loss": 2.8258,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 3.3960938453674316,
      "rewards/margins": 16.918750762939453,
      "rewards/rejected": -13.543749809265137,
      "step": 1950
    },
    {
      "epoch": 0.92018779342723,
      "grad_norm": 1230.8946830444777,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": -1.015625,
      "logits/rejected": -0.7242187261581421,
      "logps/chosen": -450.3999938964844,
      "logps/rejected": -606.0,
      "loss": 2.7315,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 3.7109375,
      "rewards/margins": 14.324999809265137,
      "rewards/rejected": -10.612500190734863,
      "step": 1960
    },
    {
      "epoch": 0.9248826291079812,
      "grad_norm": 1317.6764442794433,
      "learning_rate": 4.199269692227438e-08,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.6611328125,
      "logps/chosen": -445.20001220703125,
      "logps/rejected": -546.2000122070312,
      "loss": 3.3262,
      "rewards/accuracies": 0.604687511920929,
      "rewards/chosen": 3.7744140625,
      "rewards/margins": 12.78125,
      "rewards/rejected": -9.009374618530273,
      "step": 1970
    },
    {
      "epoch": 0.9295774647887324,
      "grad_norm": 1105.3966651597098,
      "learning_rate": 3.938445487741262e-08,
      "logits/chosen": -1.023046851158142,
      "logits/rejected": -0.761914074420929,
      "logps/chosen": -462.3999938964844,
      "logps/rejected": -589.5999755859375,
      "loss": 2.7506,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 3.964062452316284,
      "rewards/margins": 15.487500190734863,
      "rewards/rejected": -11.506250381469727,
      "step": 1980
    },
    {
      "epoch": 0.9342723004694836,
      "grad_norm": 1427.1675316150554,
      "learning_rate": 3.677621283255086e-08,
      "logits/chosen": -1.072265625,
      "logits/rejected": -0.689453125,
      "logps/chosen": -464.79998779296875,
      "logps/rejected": -587.5999755859375,
      "loss": 2.7455,
      "rewards/accuracies": 0.6382812261581421,
      "rewards/chosen": 4.060937404632568,
      "rewards/margins": 15.675000190734863,
      "rewards/rejected": -11.625,
      "step": 1990
    },
    {
      "epoch": 0.9389671361502347,
      "grad_norm": 1412.9555219220274,
      "learning_rate": 3.41679707876891e-08,
      "logits/chosen": -1.0128905773162842,
      "logits/rejected": -0.623242199420929,
      "logps/chosen": -454.0,
      "logps/rejected": -650.2000122070312,
      "loss": 2.7754,
      "rewards/accuracies": 0.6273437738418579,
      "rewards/chosen": 3.4457030296325684,
      "rewards/margins": 16.274999618530273,
      "rewards/rejected": -12.834375381469727,
      "step": 2000
    },
    {
      "epoch": 0.9436619718309859,
      "grad_norm": 1613.7506528007127,
      "learning_rate": 3.155972874282733e-08,
      "logits/chosen": -0.989062488079071,
      "logits/rejected": -0.6363281011581421,
      "logps/chosen": -432.6000061035156,
      "logps/rejected": -577.2000122070312,
      "loss": 2.6937,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 4.428124904632568,
      "rewards/margins": 13.9375,
      "rewards/rejected": -9.506250381469727,
      "step": 2010
    },
    {
      "epoch": 0.9483568075117371,
      "grad_norm": 1374.8938811570445,
      "learning_rate": 2.8951486697965573e-08,
      "logits/chosen": -1.014062523841858,
      "logits/rejected": -0.7328125238418579,
      "logps/chosen": -454.6000061035156,
      "logps/rejected": -594.0,
      "loss": 2.7465,
      "rewards/accuracies": 0.625781238079071,
      "rewards/chosen": 4.020312309265137,
      "rewards/margins": 14.612500190734863,
      "rewards/rejected": -10.59375,
      "step": 2020
    },
    {
      "epoch": 0.9530516431924883,
      "grad_norm": 1419.8993404319463,
      "learning_rate": 2.634324465310381e-08,
      "logits/chosen": -1.000390648841858,
      "logits/rejected": -0.724609375,
      "logps/chosen": -439.0,
      "logps/rejected": -585.2000122070312,
      "loss": 2.5755,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 3.735546827316284,
      "rewards/margins": 15.175000190734863,
      "rewards/rejected": -11.443750381469727,
      "step": 2030
    },
    {
      "epoch": 0.9577464788732394,
      "grad_norm": 1357.3827274328441,
      "learning_rate": 2.3735002608242045e-08,
      "logits/chosen": -0.9984375238418579,
      "logits/rejected": -0.712695300579071,
      "logps/chosen": -462.6000061035156,
      "logps/rejected": -566.0,
      "loss": 3.2529,
      "rewards/accuracies": 0.6273437738418579,
      "rewards/chosen": 3.4664063453674316,
      "rewards/margins": 15.274999618530273,
      "rewards/rejected": -11.818750381469727,
      "step": 2040
    },
    {
      "epoch": 0.9624413145539906,
      "grad_norm": 1273.0174043804745,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": -1.0183594226837158,
      "logits/rejected": -0.705078125,
      "logps/chosen": -449.3999938964844,
      "logps/rejected": -534.2000122070312,
      "loss": 2.5968,
      "rewards/accuracies": 0.6109374761581421,
      "rewards/chosen": 3.33203125,
      "rewards/margins": 13.056249618530273,
      "rewards/rejected": -9.721875190734863,
      "step": 2050
    },
    {
      "epoch": 0.9671361502347418,
      "grad_norm": 1106.8211536582048,
      "learning_rate": 1.8518518518518518e-08,
      "logits/chosen": -0.9878906011581421,
      "logits/rejected": -0.68359375,
      "logps/chosen": -471.0,
      "logps/rejected": -576.2000122070312,
      "loss": 2.4971,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 3.3980469703674316,
      "rewards/margins": 16.831249237060547,
      "rewards/rejected": -13.412500381469727,
      "step": 2060
    },
    {
      "epoch": 0.971830985915493,
      "grad_norm": 1563.0954062713447,
      "learning_rate": 1.5910276473656755e-08,
      "logits/chosen": -0.983593761920929,
      "logits/rejected": -0.6976562738418579,
      "logps/chosen": -444.6000061035156,
      "logps/rejected": -584.7999877929688,
      "loss": 3.1018,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": 3.8218750953674316,
      "rewards/margins": 16.134374618530273,
      "rewards/rejected": -12.331250190734863,
      "step": 2070
    },
    {
      "epoch": 0.9765258215962441,
      "grad_norm": 1284.8434256253668,
      "learning_rate": 1.3302034428794991e-08,
      "logits/chosen": -0.9937499761581421,
      "logits/rejected": -0.6279296875,
      "logps/chosen": -433.3999938964844,
      "logps/rejected": -544.5999755859375,
      "loss": 2.4778,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 3.399218797683716,
      "rewards/margins": 14.737500190734863,
      "rewards/rejected": -11.356249809265137,
      "step": 2080
    },
    {
      "epoch": 0.9812206572769953,
      "grad_norm": 1414.5543772282429,
      "learning_rate": 1.0693792383933229e-08,
      "logits/chosen": -1.005468726158142,
      "logits/rejected": -0.5785156488418579,
      "logps/chosen": -457.6000061035156,
      "logps/rejected": -601.2000122070312,
      "loss": 2.9064,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 3.951171875,
      "rewards/margins": 15.678125381469727,
      "rewards/rejected": -11.737500190734863,
      "step": 2090
    },
    {
      "epoch": 0.9859154929577465,
      "grad_norm": 1746.883277962999,
      "learning_rate": 8.085550339071465e-09,
      "logits/chosen": -1.07421875,
      "logits/rejected": -0.7386718988418579,
      "logps/chosen": -466.20001220703125,
      "logps/rejected": -630.7999877929688,
      "loss": 2.8683,
      "rewards/accuracies": 0.6304687261581421,
      "rewards/chosen": 3.913281202316284,
      "rewards/margins": 15.318750381469727,
      "rewards/rejected": -11.412500381469727,
      "step": 2100
    },
    {
      "epoch": 0.9906103286384976,
      "grad_norm": 1384.360818889628,
      "learning_rate": 5.4773082942097025e-09,
      "logits/chosen": -1.025781273841858,
      "logits/rejected": -0.6712890863418579,
      "logps/chosen": -445.20001220703125,
      "logps/rejected": -564.4000244140625,
      "loss": 2.304,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 3.950000047683716,
      "rewards/margins": 15.637499809265137,
      "rewards/rejected": -11.675000190734863,
      "step": 2110
    },
    {
      "epoch": 0.9953051643192489,
      "grad_norm": 1475.190949668735,
      "learning_rate": 2.8690662493479393e-09,
      "logits/chosen": -0.992968738079071,
      "logits/rejected": -0.737109363079071,
      "logps/chosen": -450.3999938964844,
      "logps/rejected": -576.0,
      "loss": 2.7137,
      "rewards/accuracies": 0.6351562738418579,
      "rewards/chosen": 3.7874999046325684,
      "rewards/margins": 15.375,
      "rewards/rejected": -11.574999809265137,
      "step": 2120
    },
    {
      "epoch": 1.0,
      "grad_norm": 1412.5237202780083,
      "learning_rate": 2.608242044861763e-10,
      "logits/chosen": -1.042578101158142,
      "logits/rejected": -0.6802734136581421,
      "logps/chosen": -437.0,
      "logps/rejected": -602.7999877929688,
      "loss": 2.6638,
      "rewards/accuracies": 0.6100813150405884,
      "rewards/chosen": 4.067968845367432,
      "rewards/margins": 15.637499809265137,
      "rewards/rejected": -11.568750381469727,
      "step": 2130
    },
    {
      "epoch": 1.0,
      "step": 2130,
      "total_flos": 0.0,
      "train_loss": 2.9490895759331788,
      "train_runtime": 4813.8824,
      "train_samples_per_second": 56.625,
      "train_steps_per_second": 0.442
    }
  ],
  "logging_steps": 10,
  "max_steps": 2130,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}